WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	//#include "../../../platform/x86/intel_ips.h"
		32	#include
		33
4560	Serge	34
3031	serge	35	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		36
		37	void getrawmonotonic(struct timespec *ts);
		38
4560	Serge	39	/**
		40	* RC6 is a special power stage which allows the GPU to enter an very
		41	* low-voltage mode when idle, using down to 0V while at this stage. This
		42	* stage is entered automatically when the GPU is idle when RC6 support is
		43	* enabled, and as soon as new workload arises GPU wakes up automatically as well.
		44	*
		45	* There are different RC6 modes available in Intel GPU, which differentiate
		46	* among each other with the latency required to enter and leave RC6 and
		47	* voltage consumed by the GPU in different states.
		48	*
		49	* The combination of the following flags define which states GPU is allowed
		50	* to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
		51	* RC6pp is deepest RC6. Their support by hardware varies according to the
		52	* GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
		53	* which brings the most power savings; deeper states save more power, but
		54	* require higher latency to switch to and wake up.
		55	*/
		56	#define INTEL_RC6_ENABLE (1<<0)
		57	#define INTEL_RC6p_ENABLE (1<<1)
		58	#define INTEL_RC6pp_ENABLE (1<<2)
		59
6084	serge	60	static void bxt_init_clock_gating(struct drm_device *dev)
5354	serge	61	{
		62	struct drm_i915_private *dev_priv = dev->dev_private;
		63
6084	serge	64	/* WaDisableSDEUnitClockGating:bxt */
5354	serge	65	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		66	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		67
		68	/*
6084	serge	69	* FIXME:
		70	* GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
5354	serge	71	*/
6084	serge	72	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		73	GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
5354	serge	74	}
		75
3031	serge	76	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		77	{
5060	serge	78	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	79	u32 tmp;
		80
		81	tmp = I915_READ(CLKCFG);
		82
		83	switch (tmp & CLKCFG_FSB_MASK) {
		84	case CLKCFG_FSB_533:
		85	dev_priv->fsb_freq = 533; /* 1334 /
		86	break;
		87	case CLKCFG_FSB_800:
		88	dev_priv->fsb_freq = 800; /* 2004 /
		89	break;
		90	case CLKCFG_FSB_667:
		91	dev_priv->fsb_freq = 667; /* 1674 /
		92	break;
		93	case CLKCFG_FSB_400:
		94	dev_priv->fsb_freq = 400; /* 1004 /
		95	break;
		96	}
		97
		98	switch (tmp & CLKCFG_MEM_MASK) {
		99	case CLKCFG_MEM_533:
		100	dev_priv->mem_freq = 533;
		101	break;
		102	case CLKCFG_MEM_667:
		103	dev_priv->mem_freq = 667;
		104	break;
		105	case CLKCFG_MEM_800:
		106	dev_priv->mem_freq = 800;
		107	break;
		108	}
		109
		110	/* detect pineview DDR3 setting */
		111	tmp = I915_READ(CSHRDDR3CTL);
		112	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		113	}
		114
		115	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		116	{
5060	serge	117	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	118	u16 ddrpll, csipll;
		119
		120	ddrpll = I915_READ16(DDRMPLL1);
		121	csipll = I915_READ16(CSIPLL0);
		122
		123	switch (ddrpll & 0xff) {
		124	case 0xc:
		125	dev_priv->mem_freq = 800;
		126	break;
		127	case 0x10:
		128	dev_priv->mem_freq = 1066;
		129	break;
		130	case 0x14:
		131	dev_priv->mem_freq = 1333;
		132	break;
		133	case 0x18:
		134	dev_priv->mem_freq = 1600;
		135	break;
		136	default:
		137	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		138	ddrpll & 0xff);
		139	dev_priv->mem_freq = 0;
		140	break;
		141	}
		142
		143	dev_priv->ips.r_t = dev_priv->mem_freq;
		144
		145	switch (csipll & 0x3ff) {
		146	case 0x00c:
		147	dev_priv->fsb_freq = 3200;
		148	break;
		149	case 0x00e:
		150	dev_priv->fsb_freq = 3733;
		151	break;
		152	case 0x010:
		153	dev_priv->fsb_freq = 4266;
		154	break;
		155	case 0x012:
		156	dev_priv->fsb_freq = 4800;
		157	break;
		158	case 0x014:
		159	dev_priv->fsb_freq = 5333;
		160	break;
		161	case 0x016:
		162	dev_priv->fsb_freq = 5866;
		163	break;
		164	case 0x018:
		165	dev_priv->fsb_freq = 6400;
		166	break;
		167	default:
		168	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		169	csipll & 0x3ff);
		170	dev_priv->fsb_freq = 0;
		171	break;
		172	}
		173
		174	if (dev_priv->fsb_freq == 3200) {
		175	dev_priv->ips.c_m = 0;
		176	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		177	dev_priv->ips.c_m = 1;
		178	} else {
		179	dev_priv->ips.c_m = 2;
		180	}
		181	}
		182
		183	static const struct cxsr_latency cxsr_latency_table[] = {
		184	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		185	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		186	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		187	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		188	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		189
		190	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		191	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		192	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		193	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		194	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		195
		196	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		197	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		198	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		199	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		200	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		201
		202	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		203	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		204	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		205	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		206	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		207
		208	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		209	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		210	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		211	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		212	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		213
		214	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		215	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		216	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		217	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		218	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		219	};
		220
		221	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		222	int is_ddr3,
		223	int fsb,
		224	int mem)
		225	{
		226	const struct cxsr_latency *latency;
		227	int i;
		228
		229	if (fsb == 0 \|\| mem == 0)
		230	return NULL;
		231
		232	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		233	latency = &cxsr_latency_table[i];
		234	if (is_desktop == latency->is_desktop &&
		235	is_ddr3 == latency->is_ddr3 &&
		236	fsb == latency->fsb_freq && mem == latency->mem_freq)
		237	return latency;
		238	}
		239
		240	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		241
		242	return NULL;
		243	}
		244
6084	serge	245	static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
		246	{
		247	u32 val;
		248
		249	mutex_lock(&dev_priv->rps.hw_lock);
		250
		251	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		252	if (enable)
		253	val &= ~FORCE_DDR_HIGH_FREQ;
		254	else
		255	val \|= FORCE_DDR_HIGH_FREQ;
		256	val &= ~FORCE_DDR_LOW_FREQ;
		257	val \|= FORCE_DDR_FREQ_REQ_ACK;
		258	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
		259
		260	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		261	FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
		262	DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
		263
		264	mutex_unlock(&dev_priv->rps.hw_lock);
		265	}
		266
		267	static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
		268	{
		269	u32 val;
		270
		271	mutex_lock(&dev_priv->rps.hw_lock);
		272
		273	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
		274	if (enable)
		275	val \|= DSP_MAXFIFO_PM5_ENABLE;
		276	else
		277	val &= ~DSP_MAXFIFO_PM5_ENABLE;
		278	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
		279
		280	mutex_unlock(&dev_priv->rps.hw_lock);
		281	}
		282
		283	#define FW_WM(value, plane) \
		284	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
		285
5060	serge	286	void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
3031	serge	287	{
5060	serge	288	struct drm_device *dev = dev_priv->dev;
		289	u32 val;
3031	serge	290
5060	serge	291	if (IS_VALLEYVIEW(dev)) {
		292	I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
6084	serge	293	POSTING_READ(FW_BLC_SELF_VLV);
		294	dev_priv->wm.vlv.cxsr = enable;
5060	serge	295	} else if (IS_G4X(dev) \|\| IS_CRESTLINE(dev)) {
		296	I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
6084	serge	297	POSTING_READ(FW_BLC_SELF);
5060	serge	298	} else if (IS_PINEVIEW(dev)) {
		299	val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
		300	val \|= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
		301	I915_WRITE(DSPFW3, val);
6084	serge	302	POSTING_READ(DSPFW3);
5060	serge	303	} else if (IS_I945G(dev) \|\| IS_I945GM(dev)) {
		304	val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
		305	_MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
		306	I915_WRITE(FW_BLC_SELF, val);
6084	serge	307	POSTING_READ(FW_BLC_SELF);
5060	serge	308	} else if (IS_I915GM(dev)) {
		309	val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
		310	_MASKED_BIT_DISABLE(INSTPM_SELF_EN);
		311	I915_WRITE(INSTPM, val);
6084	serge	312	POSTING_READ(INSTPM);
5060	serge	313	} else {
		314	return;
		315	}
		316
		317	DRM_DEBUG_KMS("memory self-refresh is %s\n",
		318	enable ? "enabled" : "disabled");
3031	serge	319	}
		320
6084	serge	321
3031	serge	322	/*
		323	* Latency for FIFO fetches is dependent on several factors:
		324	* - memory configuration (speed, channels)
		325	* - chipset
		326	* - current MCH state
		327	* It can be fairly high in some situations, so here we assume a fairly
		328	* pessimal value. It's a tradeoff between extra memory fetches (if we
		329	* set this value too high, the FIFO will fetch frequently to stay full)
		330	* and power consumption (set it too low to save power and we might see
		331	* FIFO underruns and display "flicker").
		332	*
		333	* A value of 5us seems to be a good balance; safe for very low end
		334	* platforms but not overly aggressive on lower latency configs.
		335	*/
5354	serge	336	static const int pessimal_latency_ns = 5000;
3031	serge	337
6084	serge	338	#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
		339	((((dsparb) >> (lo_shift)) & 0xff) \| ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
		340
		341	static int vlv_get_fifo_size(struct drm_device *dev,
		342	enum pipe pipe, int plane)
		343	{
		344	struct drm_i915_private *dev_priv = dev->dev_private;
		345	int sprite0_start, sprite1_start, size;
		346
		347	switch (pipe) {
		348	uint32_t dsparb, dsparb2, dsparb3;
		349	case PIPE_A:
		350	dsparb = I915_READ(DSPARB);
		351	dsparb2 = I915_READ(DSPARB2);
		352	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
		353	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
		354	break;
		355	case PIPE_B:
		356	dsparb = I915_READ(DSPARB);
		357	dsparb2 = I915_READ(DSPARB2);
		358	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
		359	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
		360	break;
		361	case PIPE_C:
		362	dsparb2 = I915_READ(DSPARB2);
		363	dsparb3 = I915_READ(DSPARB3);
		364	sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
		365	sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
		366	break;
		367	default:
		368	return 0;
		369	}
		370
		371	switch (plane) {
		372	case 0:
		373	size = sprite0_start;
		374	break;
		375	case 1:
		376	size = sprite1_start - sprite0_start;
		377	break;
		378	case 2:
		379	size = 512 - 1 - sprite1_start;
		380	break;
		381	default:
		382	return 0;
		383	}
		384
		385	DRM_DEBUG_KMS("Pipe %c %s %c FIFO size: %d\n",
		386	pipe_name(pipe), plane == 0 ? "primary" : "sprite",
		387	plane == 0 ? plane_name(pipe) : sprite_name(pipe, plane - 1),
		388	size);
		389
		390	return size;
		391	}
		392
3031	serge	393	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		394	{
		395	struct drm_i915_private *dev_priv = dev->dev_private;
		396	uint32_t dsparb = I915_READ(DSPARB);
		397	int size;
		398
		399	size = dsparb & 0x7f;
		400	if (plane)
		401	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		402
		403	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		404	plane ? "B" : "A", size);
		405
		406	return size;
		407	}
		408
4560	Serge	409	static int i830_get_fifo_size(struct drm_device *dev, int plane)
3031	serge	410	{
		411	struct drm_i915_private *dev_priv = dev->dev_private;
		412	uint32_t dsparb = I915_READ(DSPARB);
		413	int size;
		414
		415	size = dsparb & 0x1ff;
		416	if (plane)
		417	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		418	size >>= 1; /* Convert to cachelines */
		419
		420	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		421	plane ? "B" : "A", size);
		422
		423	return size;
		424	}
		425
		426	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		427	{
		428	struct drm_i915_private *dev_priv = dev->dev_private;
		429	uint32_t dsparb = I915_READ(DSPARB);
		430	int size;
		431
		432	size = dsparb & 0x7f;
		433	size >>= 2; /* Convert to cachelines */
		434
		435	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		436	plane ? "B" : "A",
		437	size);
		438
		439	return size;
		440	}
		441
		442	/* Pineview has different values for various configs */
		443	static const struct intel_watermark_params pineview_display_wm = {
5060	serge	444	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		445	.max_wm = PINEVIEW_MAX_WM,
		446	.default_wm = PINEVIEW_DFT_WM,
		447	.guard_size = PINEVIEW_GUARD_WM,
		448	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	449	};
		450	static const struct intel_watermark_params pineview_display_hplloff_wm = {
5060	serge	451	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		452	.max_wm = PINEVIEW_MAX_WM,
		453	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
		454	.guard_size = PINEVIEW_GUARD_WM,
		455	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	456	};
		457	static const struct intel_watermark_params pineview_cursor_wm = {
5060	serge	458	.fifo_size = PINEVIEW_CURSOR_FIFO,
		459	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		460	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		461	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		462	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	463	};
		464	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
5060	serge	465	.fifo_size = PINEVIEW_CURSOR_FIFO,
		466	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		467	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		468	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		469	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	470	};
		471	static const struct intel_watermark_params g4x_wm_info = {
5060	serge	472	.fifo_size = G4X_FIFO_SIZE,
		473	.max_wm = G4X_MAX_WM,
		474	.default_wm = G4X_MAX_WM,
		475	.guard_size = 2,
		476	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	477	};
		478	static const struct intel_watermark_params g4x_cursor_wm_info = {
5060	serge	479	.fifo_size = I965_CURSOR_FIFO,
		480	.max_wm = I965_CURSOR_MAX_WM,
		481	.default_wm = I965_CURSOR_DFT_WM,
		482	.guard_size = 2,
		483	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	484	};
		485	static const struct intel_watermark_params valleyview_wm_info = {
5060	serge	486	.fifo_size = VALLEYVIEW_FIFO_SIZE,
		487	.max_wm = VALLEYVIEW_MAX_WM,
		488	.default_wm = VALLEYVIEW_MAX_WM,
		489	.guard_size = 2,
		490	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	491	};
		492	static const struct intel_watermark_params valleyview_cursor_wm_info = {
5060	serge	493	.fifo_size = I965_CURSOR_FIFO,
		494	.max_wm = VALLEYVIEW_CURSOR_MAX_WM,
		495	.default_wm = I965_CURSOR_DFT_WM,
		496	.guard_size = 2,
		497	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	498	};
		499	static const struct intel_watermark_params i965_cursor_wm_info = {
5060	serge	500	.fifo_size = I965_CURSOR_FIFO,
		501	.max_wm = I965_CURSOR_MAX_WM,
		502	.default_wm = I965_CURSOR_DFT_WM,
		503	.guard_size = 2,
		504	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	505	};
		506	static const struct intel_watermark_params i945_wm_info = {
5060	serge	507	.fifo_size = I945_FIFO_SIZE,
		508	.max_wm = I915_MAX_WM,
		509	.default_wm = 1,
		510	.guard_size = 2,
		511	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	512	};
		513	static const struct intel_watermark_params i915_wm_info = {
5060	serge	514	.fifo_size = I915_FIFO_SIZE,
		515	.max_wm = I915_MAX_WM,
		516	.default_wm = 1,
		517	.guard_size = 2,
		518	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	519	};
5354	serge	520	static const struct intel_watermark_params i830_a_wm_info = {
5060	serge	521	.fifo_size = I855GM_FIFO_SIZE,
		522	.max_wm = I915_MAX_WM,
		523	.default_wm = 1,
		524	.guard_size = 2,
		525	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	526	};
5354	serge	527	static const struct intel_watermark_params i830_bc_wm_info = {
		528	.fifo_size = I855GM_FIFO_SIZE,
		529	.max_wm = I915_MAX_WM/2,
		530	.default_wm = 1,
		531	.guard_size = 2,
		532	.cacheline_size = I830_FIFO_LINE_SIZE,
		533	};
4560	Serge	534	static const struct intel_watermark_params i845_wm_info = {
5060	serge	535	.fifo_size = I830_FIFO_SIZE,
		536	.max_wm = I915_MAX_WM,
		537	.default_wm = 1,
		538	.guard_size = 2,
		539	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	540	};
		541
		542	/**
		543	* intel_calculate_wm - calculate watermark level
		544	* @clock_in_khz: pixel clock
		545	* @wm: chip FIFO params
		546	* @pixel_size: display pixel size
		547	* @latency_ns: memory latency for the platform
		548	*
		549	* Calculate the watermark level (the level at which the display plane will
		550	* start fetching from memory again). Each chip has a different display
		551	* FIFO size and allocation, so the caller needs to figure that out and pass
		552	* in the correct intel_watermark_params structure.
		553	*
		554	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		555	* on the pixel size. When it reaches the watermark level, it'll start
		556	* fetching FIFO line sized based chunks from memory until the FIFO fills
		557	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		558	* will occur, and a display engine hang could result.
		559	*/
		560	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		561	const struct intel_watermark_params *wm,
		562	int fifo_size,
		563	int pixel_size,
		564	unsigned long latency_ns)
		565	{
		566	long entries_required, wm_size;
		567
		568	/*
		569	* Note: we need to make sure we don't overflow for various clock &
		570	* latency values.
		571	* clocks go from a few thousand to several hundred thousand.
		572	* latency is usually a few thousand
		573	*/
		574	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		575	1000;
		576	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		577
		578	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		579
		580	wm_size = fifo_size - (entries_required + wm->guard_size);
		581
		582	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		583
		584	/* Don't promote wm_size to unsigned... */
		585	if (wm_size > (long)wm->max_wm)
		586	wm_size = wm->max_wm;
		587	if (wm_size <= 0)
		588	wm_size = wm->default_wm;
5354	serge	589
		590	/*
		591	* Bspec seems to indicate that the value shouldn't be lower than
		592	* 'burst size + 1'. Certainly 830 is quite unhappy with low values.
		593	* Lets go for 8 which is the burst size since certain platforms
		594	* already use a hardcoded 8 (which is what the spec says should be
		595	* done).
		596	*/
		597	if (wm_size <= 8)
		598	wm_size = 8;
		599
3031	serge	600	return wm_size;
		601	}
		602
		603	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		604	{
		605	struct drm_crtc crtc, enabled = NULL;
		606
5060	serge	607	for_each_crtc(dev, crtc) {
3243	Serge	608	if (intel_crtc_active(crtc)) {
3031	serge	609	if (enabled)
		610	return NULL;
		611	enabled = crtc;
		612	}
		613	}
		614
		615	return enabled;
		616	}
		617
4560	Serge	618	static void pineview_update_wm(struct drm_crtc *unused_crtc)
3031	serge	619	{
4560	Serge	620	struct drm_device *dev = unused_crtc->dev;
3031	serge	621	struct drm_i915_private *dev_priv = dev->dev_private;
		622	struct drm_crtc *crtc;
		623	const struct cxsr_latency *latency;
		624	u32 reg;
		625	unsigned long wm;
		626
		627	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		628	dev_priv->fsb_freq, dev_priv->mem_freq);
		629	if (!latency) {
		630	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
5060	serge	631	intel_set_memory_cxsr(dev_priv, false);
3031	serge	632	return;
		633	}
		634
		635	crtc = single_enabled_crtc(dev);
		636	if (crtc) {
6084	serge	637	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
		638	int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
		639	int clock = adjusted_mode->crtc_clock;
3031	serge	640
		641	/* Display SR */
		642	wm = intel_calculate_wm(clock, &pineview_display_wm,
		643	pineview_display_wm.fifo_size,
		644	pixel_size, latency->display_sr);
		645	reg = I915_READ(DSPFW1);
		646	reg &= ~DSPFW_SR_MASK;
6084	serge	647	reg \|= FW_WM(wm, SR);
3031	serge	648	I915_WRITE(DSPFW1, reg);
		649	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		650
		651	/* cursor SR */
		652	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		653	pineview_display_wm.fifo_size,
		654	pixel_size, latency->cursor_sr);
		655	reg = I915_READ(DSPFW3);
		656	reg &= ~DSPFW_CURSOR_SR_MASK;
6084	serge	657	reg \|= FW_WM(wm, CURSOR_SR);
3031	serge	658	I915_WRITE(DSPFW3, reg);
		659
		660	/* Display HPLL off SR */
		661	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		662	pineview_display_hplloff_wm.fifo_size,
		663	pixel_size, latency->display_hpll_disable);
		664	reg = I915_READ(DSPFW3);
		665	reg &= ~DSPFW_HPLL_SR_MASK;
6084	serge	666	reg \|= FW_WM(wm, HPLL_SR);
3031	serge	667	I915_WRITE(DSPFW3, reg);
		668
		669	/* cursor HPLL off SR */
		670	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		671	pineview_display_hplloff_wm.fifo_size,
		672	pixel_size, latency->cursor_hpll_disable);
		673	reg = I915_READ(DSPFW3);
		674	reg &= ~DSPFW_HPLL_CURSOR_MASK;
6084	serge	675	reg \|= FW_WM(wm, HPLL_CURSOR);
3031	serge	676	I915_WRITE(DSPFW3, reg);
		677	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		678
5060	serge	679	intel_set_memory_cxsr(dev_priv, true);
3031	serge	680	} else {
5060	serge	681	intel_set_memory_cxsr(dev_priv, false);
3031	serge	682	}
		683	}
		684
		685	static bool g4x_compute_wm0(struct drm_device *dev,
		686	int plane,
		687	const struct intel_watermark_params *display,
		688	int display_latency_ns,
		689	const struct intel_watermark_params *cursor,
		690	int cursor_latency_ns,
		691	int *plane_wm,
		692	int *cursor_wm)
		693	{
		694	struct drm_crtc *crtc;
4560	Serge	695	const struct drm_display_mode *adjusted_mode;
3031	serge	696	int htotal, hdisplay, clock, pixel_size;
		697	int line_time_us, line_count;
		698	int entries, tlb_miss;
		699
		700	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	701	if (!intel_crtc_active(crtc)) {
3031	serge	702	*cursor_wm = cursor->guard_size;
		703	*plane_wm = display->guard_size;
6084	serge	704	return false;
3031	serge	705	}
		706
6084	serge	707	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	708	clock = adjusted_mode->crtc_clock;
		709	htotal = adjusted_mode->crtc_htotal;
6084	serge	710	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		711	pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	712
		713	/* Use the small buffer method to calculate plane watermark */
		714	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		715	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		716	if (tlb_miss > 0)
		717	entries += tlb_miss;
		718	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		719	*plane_wm = entries + display->guard_size;
		720	if (*plane_wm > (int)display->max_wm)
		721	*plane_wm = display->max_wm;
		722
		723	/* Use the large buffer method to calculate cursor watermark */
5060	serge	724	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	725	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
6084	serge	726	entries = line_count * crtc->cursor->state->crtc_w * pixel_size;
3031	serge	727	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		728	if (tlb_miss > 0)
		729	entries += tlb_miss;
		730	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		731	*cursor_wm = entries + cursor->guard_size;
		732	if (*cursor_wm > (int)cursor->max_wm)
		733	*cursor_wm = (int)cursor->max_wm;
		734
		735	return true;
		736	}
		737
		738	/*
		739	* Check the wm result.
		740	*
		741	* If any calculated watermark values is larger than the maximum value that
		742	* can be programmed into the associated watermark register, that watermark
		743	* must be disabled.
		744	*/
		745	static bool g4x_check_srwm(struct drm_device *dev,
		746	int display_wm, int cursor_wm,
		747	const struct intel_watermark_params *display,
		748	const struct intel_watermark_params *cursor)
		749	{
		750	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		751	display_wm, cursor_wm);
		752
		753	if (display_wm > display->max_wm) {
		754	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		755	display_wm, display->max_wm);
		756	return false;
		757	}
		758
		759	if (cursor_wm > cursor->max_wm) {
		760	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		761	cursor_wm, cursor->max_wm);
		762	return false;
		763	}
		764
		765	if (!(display_wm \|\| cursor_wm)) {
		766	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		767	return false;
		768	}
		769
		770	return true;
		771	}
		772
		773	static bool g4x_compute_srwm(struct drm_device *dev,
		774	int plane,
		775	int latency_ns,
		776	const struct intel_watermark_params *display,
		777	const struct intel_watermark_params *cursor,
		778	int display_wm, int cursor_wm)
		779	{
		780	struct drm_crtc *crtc;
4560	Serge	781	const struct drm_display_mode *adjusted_mode;
3031	serge	782	int hdisplay, htotal, pixel_size, clock;
		783	unsigned long line_time_us;
		784	int line_count, line_size;
		785	int small, large;
		786	int entries;
		787
		788	if (!latency_ns) {
		789	display_wm = cursor_wm = 0;
		790	return false;
		791	}
		792
		793	crtc = intel_get_crtc_for_plane(dev, plane);
6084	serge	794	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	795	clock = adjusted_mode->crtc_clock;
		796	htotal = adjusted_mode->crtc_htotal;
6084	serge	797	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		798	pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	799
5060	serge	800	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	801	line_count = (latency_ns / line_time_us + 1000) / 1000;
		802	line_size = hdisplay * pixel_size;
		803
		804	/* Use the minimum of the small and large buffer method for primary */
		805	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		806	large = line_count * line_size;
		807
		808	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		809	*display_wm = entries + display->guard_size;
		810
		811	/* calculate the self-refresh watermark for display cursor */
6084	serge	812	entries = line_count * pixel_size * crtc->cursor->state->crtc_w;
3031	serge	813	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		814	*cursor_wm = entries + cursor->guard_size;
		815
		816	return g4x_check_srwm(dev,
		817	display_wm, cursor_wm,
		818	display, cursor);
		819	}
		820
6084	serge	821	#define FW_WM_VLV(value, plane) \
		822	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
		823
		824	static void vlv_write_wm_values(struct intel_crtc *crtc,
		825	const struct vlv_wm_values *wm)
3031	serge	826	{
6084	serge	827	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
		828	enum pipe pipe = crtc->pipe;
3031	serge	829
6084	serge	830	I915_WRITE(VLV_DDL(pipe),
		831	(wm->ddl[pipe].cursor << DDL_CURSOR_SHIFT) \|
		832	(wm->ddl[pipe].sprite[1] << DDL_SPRITE_SHIFT(1)) \|
		833	(wm->ddl[pipe].sprite[0] << DDL_SPRITE_SHIFT(0)) \|
		834	(wm->ddl[pipe].primary << DDL_PLANE_SHIFT));
3031	serge	835
6084	serge	836	I915_WRITE(DSPFW1,
		837	FW_WM(wm->sr.plane, SR) \|
		838	FW_WM(wm->pipe[PIPE_B].cursor, CURSORB) \|
		839	FW_WM_VLV(wm->pipe[PIPE_B].primary, PLANEB) \|
		840	FW_WM_VLV(wm->pipe[PIPE_A].primary, PLANEA));
		841	I915_WRITE(DSPFW2,
		842	FW_WM_VLV(wm->pipe[PIPE_A].sprite[1], SPRITEB) \|
		843	FW_WM(wm->pipe[PIPE_A].cursor, CURSORA) \|
		844	FW_WM_VLV(wm->pipe[PIPE_A].sprite[0], SPRITEA));
		845	I915_WRITE(DSPFW3,
		846	FW_WM(wm->sr.cursor, CURSOR_SR));
3031	serge	847
6084	serge	848	if (IS_CHERRYVIEW(dev_priv)) {
		849	I915_WRITE(DSPFW7_CHV,
		850	FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) \|
		851	FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
		852	I915_WRITE(DSPFW8_CHV,
		853	FW_WM_VLV(wm->pipe[PIPE_C].sprite[1], SPRITEF) \|
		854	FW_WM_VLV(wm->pipe[PIPE_C].sprite[0], SPRITEE));
		855	I915_WRITE(DSPFW9_CHV,
		856	FW_WM_VLV(wm->pipe[PIPE_C].primary, PLANEC) \|
		857	FW_WM(wm->pipe[PIPE_C].cursor, CURSORC));
		858	I915_WRITE(DSPHOWM,
		859	FW_WM(wm->sr.plane >> 9, SR_HI) \|
		860	FW_WM(wm->pipe[PIPE_C].sprite[1] >> 8, SPRITEF_HI) \|
		861	FW_WM(wm->pipe[PIPE_C].sprite[0] >> 8, SPRITEE_HI) \|
		862	FW_WM(wm->pipe[PIPE_C].primary >> 8, PLANEC_HI) \|
		863	FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) \|
		864	FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) \|
		865	FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) \|
		866	FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) \|
		867	FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) \|
		868	FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
		869	} else {
		870	I915_WRITE(DSPFW7,
		871	FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) \|
		872	FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
		873	I915_WRITE(DSPHOWM,
		874	FW_WM(wm->sr.plane >> 9, SR_HI) \|
		875	FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) \|
		876	FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) \|
		877	FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) \|
		878	FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) \|
		879	FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) \|
		880	FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
		881	}
3031	serge	882
6084	serge	883	/* zero (unused) WM1 watermarks */
		884	I915_WRITE(DSPFW4, 0);
		885	I915_WRITE(DSPFW5, 0);
		886	I915_WRITE(DSPFW6, 0);
		887	I915_WRITE(DSPHOWM1, 0);
3031	serge	888
6084	serge	889	POSTING_READ(DSPFW1);
3031	serge	890	}
		891
6084	serge	892	#undef FW_WM_VLV
3031	serge	893
6084	serge	894	enum vlv_wm_level {
		895	VLV_WM_LEVEL_PM2,
		896	VLV_WM_LEVEL_PM5,
		897	VLV_WM_LEVEL_DDR_DVFS,
		898	};
		899
		900	/* latency must be in 0.1us units. */
		901	static unsigned int vlv_wm_method2(unsigned int pixel_rate,
		902	unsigned int pipe_htotal,
		903	unsigned int horiz_pixels,
		904	unsigned int bytes_per_pixel,
		905	unsigned int latency)
3031	serge	906	{
6084	serge	907	unsigned int ret;
		908
		909	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		910	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		911	ret = DIV_ROUND_UP(ret, 64);
		912
		913	return ret;
		914	}
		915
		916	static void vlv_setup_wm_latency(struct drm_device *dev)
		917	{
3031	serge	918	struct drm_i915_private *dev_priv = dev->dev_private;
		919
6084	serge	920	/* all latencies in usec */
		921	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
3031	serge	922
6084	serge	923	dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
		924
		925	if (IS_CHERRYVIEW(dev_priv)) {
		926	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
		927	dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
		928
		929	dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
3031	serge	930	}
6084	serge	931	}
3031	serge	932
6084	serge	933	static uint16_t vlv_compute_wm_level(struct intel_plane *plane,
		934	struct intel_crtc *crtc,
		935	const struct intel_plane_state *state,
		936	int level)
		937	{
		938	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
		939	int clock, htotal, pixel_size, width, wm;
		940
		941	if (dev_priv->wm.pri_latency[level] == 0)
		942	return USHRT_MAX;
		943
		944	if (!state->visible)
		945	return 0;
		946
		947	pixel_size = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		948	clock = crtc->config->base.adjusted_mode.crtc_clock;
		949	htotal = crtc->config->base.adjusted_mode.crtc_htotal;
		950	width = crtc->config->pipe_src_w;
		951	if (WARN_ON(htotal == 0))
		952	htotal = 1;
		953
		954	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		955	/*
		956	* FIXME the formula gives values that are
		957	* too big for the cursor FIFO, and hence we
		958	* would never be able to use cursors. For
		959	* now just hardcode the watermark.
		960	*/
		961	wm = 63;
		962	} else {
		963	wm = vlv_wm_method2(clock, htotal, width, pixel_size,
		964	dev_priv->wm.pri_latency[level] * 10);
5354	serge	965	}
3031	serge	966
6084	serge	967	return min_t(int, wm, USHRT_MAX);
		968	}
5354	serge	969
6084	serge	970	static void vlv_compute_fifo(struct intel_crtc *crtc)
		971	{
		972	struct drm_device *dev = crtc->base.dev;
		973	struct vlv_wm_state *wm_state = &crtc->wm_state;
		974	struct intel_plane *plane;
		975	unsigned int total_rate = 0;
		976	const int fifo_size = 512 - 1;
		977	int fifo_extra, fifo_left = fifo_size;
		978
		979	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		980	struct intel_plane_state *state =
		981	to_intel_plane_state(plane->base.state);
		982
		983	if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
		984	continue;
		985
		986	if (state->visible) {
		987	wm_state->num_active_planes++;
		988	total_rate += drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		989	}
3031	serge	990	}
5354	serge	991
6084	serge	992	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		993	struct intel_plane_state *state =
		994	to_intel_plane_state(plane->base.state);
		995	unsigned int rate;
		996
		997	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		998	plane->wm.fifo_size = 63;
		999	continue;
		1000	}
		1001
		1002	if (!state->visible) {
		1003	plane->wm.fifo_size = 0;
		1004	continue;
		1005	}
		1006
		1007	rate = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		1008	plane->wm.fifo_size = fifo_size * rate / total_rate;
		1009	fifo_left -= plane->wm.fifo_size;
		1010	}
		1011
		1012	fifo_extra = DIV_ROUND_UP(fifo_left, wm_state->num_active_planes ?: 1);
		1013
		1014	/* spread the remainder evenly */
		1015	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1016	int plane_extra;
		1017
		1018	if (fifo_left == 0)
		1019	break;
		1020
		1021	if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
		1022	continue;
		1023
		1024	/* give it all to the first plane if none are active */
		1025	if (plane->wm.fifo_size == 0 &&
		1026	wm_state->num_active_planes)
		1027	continue;
		1028
		1029	plane_extra = min(fifo_extra, fifo_left);
		1030	plane->wm.fifo_size += plane_extra;
		1031	fifo_left -= plane_extra;
		1032	}
		1033
		1034	WARN_ON(fifo_left != 0);
3031	serge	1035	}
		1036
6084	serge	1037	static void vlv_invert_wms(struct intel_crtc *crtc)
		1038	{
		1039	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1040	int level;
3031	serge	1041
6084	serge	1042	for (level = 0; level < wm_state->num_levels; level++) {
		1043	struct drm_device *dev = crtc->base.dev;
		1044	const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
		1045	struct intel_plane *plane;
		1046
		1047	wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane;
		1048	wm_state->sr[level].cursor = 63 - wm_state->sr[level].cursor;
		1049
		1050	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1051	switch (plane->base.type) {
		1052	int sprite;
		1053	case DRM_PLANE_TYPE_CURSOR:
		1054	wm_state->wm[level].cursor = plane->wm.fifo_size -
		1055	wm_state->wm[level].cursor;
		1056	break;
		1057	case DRM_PLANE_TYPE_PRIMARY:
		1058	wm_state->wm[level].primary = plane->wm.fifo_size -
		1059	wm_state->wm[level].primary;
		1060	break;
		1061	case DRM_PLANE_TYPE_OVERLAY:
		1062	sprite = plane->plane;
		1063	wm_state->wm[level].sprite[sprite] = plane->wm.fifo_size -
		1064	wm_state->wm[level].sprite[sprite];
		1065	break;
		1066	}
		1067	}
		1068	}
		1069	}
		1070
		1071	static void vlv_compute_wm(struct intel_crtc *crtc)
3031	serge	1072	{
6084	serge	1073	struct drm_device *dev = crtc->base.dev;
		1074	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1075	struct intel_plane *plane;
		1076	int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
		1077	int level;
3031	serge	1078
6084	serge	1079	memset(wm_state, 0, sizeof(*wm_state));
3031	serge	1080
6084	serge	1081	wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed;
		1082	wm_state->num_levels = to_i915(dev)->wm.max_level + 1;
3031	serge	1083
6084	serge	1084	wm_state->num_active_planes = 0;
3031	serge	1085
6084	serge	1086	vlv_compute_fifo(crtc);
		1087
		1088	if (wm_state->num_active_planes != 1)
		1089	wm_state->cxsr = false;
		1090
		1091	if (wm_state->cxsr) {
		1092	for (level = 0; level < wm_state->num_levels; level++) {
		1093	wm_state->sr[level].plane = sr_fifo_size;
		1094	wm_state->sr[level].cursor = 63;
		1095	}
3243	Serge	1096	}
3031	serge	1097
6084	serge	1098	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1099	struct intel_plane_state *state =
		1100	to_intel_plane_state(plane->base.state);
3031	serge	1101
6084	serge	1102	if (!state->visible)
		1103	continue;
5060	serge	1104
6084	serge	1105	/* normal watermarks */
		1106	for (level = 0; level < wm_state->num_levels; level++) {
		1107	int wm = vlv_compute_wm_level(plane, crtc, state, level);
		1108	int max_wm = plane->base.type == DRM_PLANE_TYPE_CURSOR ? 63 : 511;
		1109
		1110	/* hack */
		1111	if (WARN_ON(level == 0 && wm > max_wm))
		1112	wm = max_wm;
		1113
		1114	if (wm > plane->wm.fifo_size)
		1115	break;
		1116
		1117	switch (plane->base.type) {
		1118	int sprite;
		1119	case DRM_PLANE_TYPE_CURSOR:
		1120	wm_state->wm[level].cursor = wm;
		1121	break;
		1122	case DRM_PLANE_TYPE_PRIMARY:
		1123	wm_state->wm[level].primary = wm;
		1124	break;
		1125	case DRM_PLANE_TYPE_OVERLAY:
		1126	sprite = plane->plane;
		1127	wm_state->wm[level].sprite[sprite] = wm;
		1128	break;
		1129	}
		1130	}
		1131
		1132	wm_state->num_levels = level;
		1133
		1134	if (!wm_state->cxsr)
		1135	continue;
		1136
		1137	/* maxfifo watermarks */
		1138	switch (plane->base.type) {
		1139	int sprite, level;
		1140	case DRM_PLANE_TYPE_CURSOR:
		1141	for (level = 0; level < wm_state->num_levels; level++)
		1142	wm_state->sr[level].cursor =
		1143	wm_state->wm[level].cursor;
		1144	break;
		1145	case DRM_PLANE_TYPE_PRIMARY:
		1146	for (level = 0; level < wm_state->num_levels; level++)
		1147	wm_state->sr[level].plane =
		1148	min(wm_state->sr[level].plane,
		1149	wm_state->wm[level].primary);
		1150	break;
		1151	case DRM_PLANE_TYPE_OVERLAY:
		1152	sprite = plane->plane;
		1153	for (level = 0; level < wm_state->num_levels; level++)
		1154	wm_state->sr[level].plane =
		1155	min(wm_state->sr[level].plane,
		1156	wm_state->wm[level].sprite[sprite]);
		1157	break;
		1158	}
		1159	}
		1160
		1161	/* clear any (partially) filled invalid levels */
		1162	for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) {
		1163	memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level]));
		1164	memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level]));
		1165	}
		1166
		1167	vlv_invert_wms(crtc);
3031	serge	1168	}
		1169
6084	serge	1170	#define VLV_FIFO(plane, value) \
		1171	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
		1172
		1173	static void vlv_pipe_set_fifo_size(struct intel_crtc *crtc)
5354	serge	1174	{
6084	serge	1175	struct drm_device *dev = crtc->base.dev;
		1176	struct drm_i915_private *dev_priv = to_i915(dev);
		1177	struct intel_plane *plane;
		1178	int sprite0_start = 0, sprite1_start = 0, fifo_size = 0;
5354	serge	1179
6084	serge	1180	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1181	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		1182	WARN_ON(plane->wm.fifo_size != 63);
		1183	continue;
		1184	}
5354	serge	1185
6084	serge	1186	if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
		1187	sprite0_start = plane->wm.fifo_size;
		1188	else if (plane->plane == 0)
		1189	sprite1_start = sprite0_start + plane->wm.fifo_size;
		1190	else
		1191	fifo_size = sprite1_start + plane->wm.fifo_size;
		1192	}
5354	serge	1193
6084	serge	1194	WARN_ON(fifo_size != 512 - 1);
5354	serge	1195
6084	serge	1196	DRM_DEBUG_KMS("Pipe %c FIFO split %d / %d / %d\n",
		1197	pipe_name(crtc->pipe), sprite0_start,
		1198	sprite1_start, fifo_size);
5354	serge	1199
6084	serge	1200	switch (crtc->pipe) {
		1201	uint32_t dsparb, dsparb2, dsparb3;
		1202	case PIPE_A:
		1203	dsparb = I915_READ(DSPARB);
		1204	dsparb2 = I915_READ(DSPARB2);
		1205
		1206	dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) \|
		1207	VLV_FIFO(SPRITEB, 0xff));
		1208	dsparb \|= (VLV_FIFO(SPRITEA, sprite0_start) \|
		1209	VLV_FIFO(SPRITEB, sprite1_start));
		1210
		1211	dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) \|
		1212	VLV_FIFO(SPRITEB_HI, 0x1));
		1213	dsparb2 \|= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) \|
		1214	VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
		1215
		1216	I915_WRITE(DSPARB, dsparb);
		1217	I915_WRITE(DSPARB2, dsparb2);
		1218	break;
		1219	case PIPE_B:
		1220	dsparb = I915_READ(DSPARB);
		1221	dsparb2 = I915_READ(DSPARB2);
		1222
		1223	dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) \|
		1224	VLV_FIFO(SPRITED, 0xff));
		1225	dsparb \|= (VLV_FIFO(SPRITEC, sprite0_start) \|
		1226	VLV_FIFO(SPRITED, sprite1_start));
		1227
		1228	dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) \|
		1229	VLV_FIFO(SPRITED_HI, 0xff));
		1230	dsparb2 \|= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) \|
		1231	VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
		1232
		1233	I915_WRITE(DSPARB, dsparb);
		1234	I915_WRITE(DSPARB2, dsparb2);
		1235	break;
		1236	case PIPE_C:
		1237	dsparb3 = I915_READ(DSPARB3);
		1238	dsparb2 = I915_READ(DSPARB2);
		1239
		1240	dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) \|
		1241	VLV_FIFO(SPRITEF, 0xff));
		1242	dsparb3 \|= (VLV_FIFO(SPRITEE, sprite0_start) \|
		1243	VLV_FIFO(SPRITEF, sprite1_start));
		1244
		1245	dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) \|
		1246	VLV_FIFO(SPRITEF_HI, 0xff));
		1247	dsparb2 \|= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) \|
		1248	VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
		1249
		1250	I915_WRITE(DSPARB3, dsparb3);
		1251	I915_WRITE(DSPARB2, dsparb2);
		1252	break;
		1253	default:
		1254	break;
5354	serge	1255	}
6084	serge	1256	}
5354	serge	1257
6084	serge	1258	#undef VLV_FIFO
5354	serge	1259
6084	serge	1260	static void vlv_merge_wm(struct drm_device *dev,
		1261	struct vlv_wm_values *wm)
		1262	{
		1263	struct intel_crtc *crtc;
		1264	int num_active_crtcs = 0;
5354	serge	1265
6084	serge	1266	wm->level = to_i915(dev)->wm.max_level;
		1267	wm->cxsr = true;
		1268
		1269	for_each_intel_crtc(dev, crtc) {
		1270	const struct vlv_wm_state *wm_state = &crtc->wm_state;
		1271
		1272	if (!crtc->active)
		1273	continue;
		1274
		1275	if (!wm_state->cxsr)
		1276	wm->cxsr = false;
		1277
		1278	num_active_crtcs++;
		1279	wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
		1280	}
		1281
		1282	if (num_active_crtcs != 1)
		1283	wm->cxsr = false;
		1284
		1285	if (num_active_crtcs > 1)
		1286	wm->level = VLV_WM_LEVEL_PM2;
		1287
		1288	for_each_intel_crtc(dev, crtc) {
		1289	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1290	enum pipe pipe = crtc->pipe;
		1291
		1292	if (!crtc->active)
		1293	continue;
		1294
		1295	wm->pipe[pipe] = wm_state->wm[wm->level];
		1296	if (wm->cxsr)
		1297	wm->sr = wm_state->sr[wm->level];
		1298
		1299	wm->ddl[pipe].primary = DDL_PRECISION_HIGH \| 2;
		1300	wm->ddl[pipe].sprite[0] = DDL_PRECISION_HIGH \| 2;
		1301	wm->ddl[pipe].sprite[1] = DDL_PRECISION_HIGH \| 2;
		1302	wm->ddl[pipe].cursor = DDL_PRECISION_HIGH \| 2;
		1303	}
5354	serge	1304	}
		1305
6084	serge	1306	static void vlv_update_wm(struct drm_crtc *crtc)
5354	serge	1307	{
		1308	struct drm_device *dev = crtc->dev;
		1309	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	1310	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1311	enum pipe pipe = intel_crtc->pipe;
		1312	struct vlv_wm_values wm = {};
5354	serge	1313
6084	serge	1314	vlv_compute_wm(intel_crtc);
		1315	vlv_merge_wm(dev, &wm);
5354	serge	1316
6084	serge	1317	if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) {
		1318	/* FIXME should be part of crtc atomic commit */
		1319	vlv_pipe_set_fifo_size(intel_crtc);
		1320	return;
5354	serge	1321	}
		1322
6084	serge	1323	if (wm.level < VLV_WM_LEVEL_DDR_DVFS &&
		1324	dev_priv->wm.vlv.level >= VLV_WM_LEVEL_DDR_DVFS)
		1325	chv_set_memory_dvfs(dev_priv, false);
		1326
		1327	if (wm.level < VLV_WM_LEVEL_PM5 &&
		1328	dev_priv->wm.vlv.level >= VLV_WM_LEVEL_PM5)
		1329	chv_set_memory_pm5(dev_priv, false);
		1330
		1331	if (!wm.cxsr && dev_priv->wm.vlv.cxsr)
		1332	intel_set_memory_cxsr(dev_priv, false);
		1333
		1334	/* FIXME should be part of crtc atomic commit */
		1335	vlv_pipe_set_fifo_size(intel_crtc);
		1336
		1337	vlv_write_wm_values(intel_crtc, &wm);
		1338
		1339	DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, "
		1340	"sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n",
		1341	pipe_name(pipe), wm.pipe[pipe].primary, wm.pipe[pipe].cursor,
		1342	wm.pipe[pipe].sprite[0], wm.pipe[pipe].sprite[1],
		1343	wm.sr.plane, wm.sr.cursor, wm.level, wm.cxsr);
		1344
		1345	if (wm.cxsr && !dev_priv->wm.vlv.cxsr)
		1346	intel_set_memory_cxsr(dev_priv, true);
		1347
		1348	if (wm.level >= VLV_WM_LEVEL_PM5 &&
		1349	dev_priv->wm.vlv.level < VLV_WM_LEVEL_PM5)
		1350	chv_set_memory_pm5(dev_priv, true);
		1351
		1352	if (wm.level >= VLV_WM_LEVEL_DDR_DVFS &&
		1353	dev_priv->wm.vlv.level < VLV_WM_LEVEL_DDR_DVFS)
		1354	chv_set_memory_dvfs(dev_priv, true);
		1355
		1356	dev_priv->wm.vlv = wm;
5354	serge	1357	}
		1358
6084	serge	1359	#define single_plane_enabled(mask) is_power_of_2(mask)
		1360
4560	Serge	1361	static void g4x_update_wm(struct drm_crtc *crtc)
3031	serge	1362	{
4560	Serge	1363	struct drm_device *dev = crtc->dev;
3031	serge	1364	static const int sr_latency_ns = 12000;
		1365	struct drm_i915_private *dev_priv = dev->dev_private;
		1366	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1367	int plane_sr, cursor_sr;
		1368	unsigned int enabled = 0;
5060	serge	1369	bool cxsr_enabled;
3031	serge	1370
3746	Serge	1371	if (g4x_compute_wm0(dev, PIPE_A,
5354	serge	1372	&g4x_wm_info, pessimal_latency_ns,
		1373	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1374	&planea_wm, &cursora_wm))
3746	Serge	1375	enabled \|= 1 << PIPE_A;
3031	serge	1376
3746	Serge	1377	if (g4x_compute_wm0(dev, PIPE_B,
5354	serge	1378	&g4x_wm_info, pessimal_latency_ns,
		1379	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1380	&planeb_wm, &cursorb_wm))
3746	Serge	1381	enabled \|= 1 << PIPE_B;
3031	serge	1382
		1383	if (single_plane_enabled(enabled) &&
		1384	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1385	sr_latency_ns,
		1386	&g4x_wm_info,
		1387	&g4x_cursor_wm_info,
3243	Serge	1388	&plane_sr, &cursor_sr)) {
5060	serge	1389	cxsr_enabled = true;
3243	Serge	1390	} else {
5060	serge	1391	cxsr_enabled = false;
		1392	intel_set_memory_cxsr(dev_priv, false);
3243	Serge	1393	plane_sr = cursor_sr = 0;
		1394	}
3031	serge	1395
5354	serge	1396	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
		1397	"B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
3031	serge	1398	planea_wm, cursora_wm,
		1399	planeb_wm, cursorb_wm,
		1400	plane_sr, cursor_sr);
		1401
		1402	I915_WRITE(DSPFW1,
6084	serge	1403	FW_WM(plane_sr, SR) \|
		1404	FW_WM(cursorb_wm, CURSORB) \|
		1405	FW_WM(planeb_wm, PLANEB) \|
		1406	FW_WM(planea_wm, PLANEA));
3031	serge	1407	I915_WRITE(DSPFW2,
3243	Serge	1408	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
6084	serge	1409	FW_WM(cursora_wm, CURSORA));
3031	serge	1410	/* HPLL off in SR has some issues on G4x... disable it */
		1411	I915_WRITE(DSPFW3,
3243	Serge	1412	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
6084	serge	1413	FW_WM(cursor_sr, CURSOR_SR));
5060	serge	1414
		1415	if (cxsr_enabled)
		1416	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1417	}
		1418
4560	Serge	1419	static void i965_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1420	{
4560	Serge	1421	struct drm_device *dev = unused_crtc->dev;
3031	serge	1422	struct drm_i915_private *dev_priv = dev->dev_private;
		1423	struct drm_crtc *crtc;
		1424	int srwm = 1;
		1425	int cursor_sr = 16;
5060	serge	1426	bool cxsr_enabled;
3031	serge	1427
		1428	/* Calc sr entries for one plane configs */
		1429	crtc = single_enabled_crtc(dev);
		1430	if (crtc) {
		1431	/* self-refresh has much higher latency */
		1432	static const int sr_latency_ns = 12000;
6084	serge	1433	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1434	int clock = adjusted_mode->crtc_clock;
		1435	int htotal = adjusted_mode->crtc_htotal;
6084	serge	1436	int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		1437	int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	1438	unsigned long line_time_us;
		1439	int entries;
		1440
5060	serge	1441	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1442
		1443	/* Use ns/us then divide to preserve precision */
		1444	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1445	pixel_size * hdisplay;
		1446	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1447	srwm = I965_FIFO_SIZE - entries;
		1448	if (srwm < 0)
		1449	srwm = 1;
		1450	srwm &= 0x1ff;
		1451	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1452	entries, srwm);
		1453
		1454	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
6084	serge	1455	pixel_size * crtc->cursor->state->crtc_w;
3031	serge	1456	entries = DIV_ROUND_UP(entries,
		1457	i965_cursor_wm_info.cacheline_size);
		1458	cursor_sr = i965_cursor_wm_info.fifo_size -
		1459	(entries + i965_cursor_wm_info.guard_size);
		1460
		1461	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1462	cursor_sr = i965_cursor_wm_info.max_wm;
		1463
		1464	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1465	"cursor %d\n", srwm, cursor_sr);
		1466
5060	serge	1467	cxsr_enabled = true;
3031	serge	1468	} else {
5060	serge	1469	cxsr_enabled = false;
3031	serge	1470	/* Turn off self refresh if both pipes are enabled */
5060	serge	1471	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1472	}
		1473
		1474	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1475	srwm);
		1476
		1477	/* 965 has limitations... */
6084	serge	1478	I915_WRITE(DSPFW1, FW_WM(srwm, SR) \|
		1479	FW_WM(8, CURSORB) \|
		1480	FW_WM(8, PLANEB) \|
		1481	FW_WM(8, PLANEA));
		1482	I915_WRITE(DSPFW2, FW_WM(8, CURSORA) \|
		1483	FW_WM(8, PLANEC_OLD));
3031	serge	1484	/* update cursor SR watermark */
6084	serge	1485	I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
5060	serge	1486
		1487	if (cxsr_enabled)
		1488	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1489	}
		1490
6084	serge	1491	#undef FW_WM
		1492
4560	Serge	1493	static void i9xx_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1494	{
4560	Serge	1495	struct drm_device *dev = unused_crtc->dev;
3031	serge	1496	struct drm_i915_private *dev_priv = dev->dev_private;
		1497	const struct intel_watermark_params *wm_info;
		1498	uint32_t fwater_lo;
		1499	uint32_t fwater_hi;
		1500	int cwm, srwm = 1;
		1501	int fifo_size;
		1502	int planea_wm, planeb_wm;
		1503	struct drm_crtc crtc, enabled = NULL;
		1504
		1505	if (IS_I945GM(dev))
		1506	wm_info = &i945_wm_info;
		1507	else if (!IS_GEN2(dev))
		1508	wm_info = &i915_wm_info;
		1509	else
5354	serge	1510	wm_info = &i830_a_wm_info;
3031	serge	1511
		1512	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1513	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1514	if (intel_crtc_active(crtc)) {
4560	Serge	1515	const struct drm_display_mode *adjusted_mode;
6084	serge	1516	int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
3243	Serge	1517	if (IS_GEN2(dev))
		1518	cpp = 4;
		1519
6084	serge	1520	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1521	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1522	wm_info, fifo_size, cpp,
5354	serge	1523	pessimal_latency_ns);
3031	serge	1524	enabled = crtc;
5354	serge	1525	} else {
3031	serge	1526	planea_wm = fifo_size - wm_info->guard_size;
5354	serge	1527	if (planea_wm > (long)wm_info->max_wm)
		1528	planea_wm = wm_info->max_wm;
		1529	}
3031	serge	1530
5354	serge	1531	if (IS_GEN2(dev))
		1532	wm_info = &i830_bc_wm_info;
		1533
3031	serge	1534	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1535	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1536	if (intel_crtc_active(crtc)) {
4560	Serge	1537	const struct drm_display_mode *adjusted_mode;
6084	serge	1538	int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
3243	Serge	1539	if (IS_GEN2(dev))
		1540	cpp = 4;
		1541
6084	serge	1542	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1543	planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1544	wm_info, fifo_size, cpp,
5354	serge	1545	pessimal_latency_ns);
3031	serge	1546	if (enabled == NULL)
		1547	enabled = crtc;
		1548	else
		1549	enabled = NULL;
5354	serge	1550	} else {
3031	serge	1551	planeb_wm = fifo_size - wm_info->guard_size;
5354	serge	1552	if (planeb_wm > (long)wm_info->max_wm)
		1553	planeb_wm = wm_info->max_wm;
		1554	}
3031	serge	1555
		1556	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1557
5060	serge	1558	if (IS_I915GM(dev) && enabled) {
		1559	struct drm_i915_gem_object *obj;
		1560
6084	serge	1561	obj = intel_fb_obj(enabled->primary->state->fb);
5060	serge	1562
		1563	/* self-refresh seems busted with untiled */
		1564	if (obj->tiling_mode == I915_TILING_NONE)
		1565	enabled = NULL;
		1566	}
		1567
3031	serge	1568	/*
		1569	* Overlay gets an aggressive default since video jitter is bad.
		1570	*/
		1571	cwm = 2;
		1572
		1573	/* Play safe and disable self-refresh before adjusting watermarks. */
5060	serge	1574	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1575
		1576	/* Calc sr entries for one plane configs */
		1577	if (HAS_FW_BLC(dev) && enabled) {
		1578	/* self-refresh has much higher latency */
		1579	static const int sr_latency_ns = 6000;
6084	serge	1580	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode;
4560	Serge	1581	int clock = adjusted_mode->crtc_clock;
		1582	int htotal = adjusted_mode->crtc_htotal;
6084	serge	1583	int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
		1584	int pixel_size = enabled->primary->state->fb->bits_per_pixel / 8;
3031	serge	1585	unsigned long line_time_us;
		1586	int entries;
		1587
5060	serge	1588	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1589
		1590	/* Use ns/us then divide to preserve precision */
		1591	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1592	pixel_size * hdisplay;
		1593	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1594	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1595	srwm = wm_info->fifo_size - entries;
		1596	if (srwm < 0)
		1597	srwm = 1;
		1598
		1599	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1600	I915_WRITE(FW_BLC_SELF,
		1601	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1602	else if (IS_I915GM(dev))
		1603	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1604	}
		1605
		1606	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1607	planea_wm, planeb_wm, cwm, srwm);
		1608
		1609	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1610	fwater_hi = (cwm & 0x1f);
		1611
		1612	/* Set request length to 8 cachelines per fetch */
		1613	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1614	fwater_hi = fwater_hi \| (1 << 8);
		1615
		1616	I915_WRITE(FW_BLC, fwater_lo);
		1617	I915_WRITE(FW_BLC2, fwater_hi);
		1618
5060	serge	1619	if (enabled)
		1620	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1621	}
		1622
4560	Serge	1623	static void i845_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1624	{
4560	Serge	1625	struct drm_device *dev = unused_crtc->dev;
3031	serge	1626	struct drm_i915_private *dev_priv = dev->dev_private;
		1627	struct drm_crtc *crtc;
4560	Serge	1628	const struct drm_display_mode *adjusted_mode;
3031	serge	1629	uint32_t fwater_lo;
		1630	int planea_wm;
		1631
		1632	crtc = single_enabled_crtc(dev);
		1633	if (crtc == NULL)
		1634	return;
		1635
6084	serge	1636	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1637	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
		1638	&i845_wm_info,
3031	serge	1639	dev_priv->display.get_fifo_size(dev, 0),
5354	serge	1640	4, pessimal_latency_ns);
3031	serge	1641	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1642	fwater_lo \|= (3<<8) \| planea_wm;
		1643
		1644	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1645
		1646	I915_WRITE(FW_BLC, fwater_lo);
		1647	}
		1648
6084	serge	1649	uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
3031	serge	1650	{
4104	Serge	1651	uint32_t pixel_rate;
		1652
6084	serge	1653	pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
4104	Serge	1654
		1655	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
		1656	* adjust the pixel_rate here. */
		1657
6084	serge	1658	if (pipe_config->pch_pfit.enabled) {
4104	Serge	1659	uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
6084	serge	1660	uint32_t pfit_size = pipe_config->pch_pfit.size;
4104	Serge	1661
6084	serge	1662	pipe_w = pipe_config->pipe_src_w;
		1663	pipe_h = pipe_config->pipe_src_h;
		1664
4104	Serge	1665	pfit_w = (pfit_size >> 16) & 0xFFFF;
		1666	pfit_h = pfit_size & 0xFFFF;
		1667	if (pipe_w < pfit_w)
		1668	pipe_w = pfit_w;
		1669	if (pipe_h < pfit_h)
		1670	pipe_h = pfit_h;
		1671
		1672	pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
		1673	pfit_w * pfit_h);
		1674	}
		1675
		1676	return pixel_rate;
		1677	}
		1678
		1679	/* latency must be in 0.1us units. */
		1680	static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		1681	uint32_t latency)
		1682	{
		1683	uint64_t ret;
		1684
		1685	if (WARN(latency == 0, "Latency value missing\n"))
		1686	return UINT_MAX;
		1687
		1688	ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
		1689	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
		1690
		1691	return ret;
		1692	}
		1693
		1694	/* latency must be in 0.1us units. */
		1695	static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		1696	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
		1697	uint32_t latency)
		1698	{
		1699	uint32_t ret;
		1700
		1701	if (WARN(latency == 0, "Latency value missing\n"))
		1702	return UINT_MAX;
		1703
		1704	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		1705	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		1706	ret = DIV_ROUND_UP(ret, 64) + 2;
		1707	return ret;
		1708	}
		1709
		1710	static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
		1711	uint8_t bytes_per_pixel)
		1712	{
		1713	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
		1714	}
		1715
5354	serge	1716	struct skl_pipe_wm_parameters {
		1717	bool active;
		1718	uint32_t pipe_htotal;
		1719	uint32_t pixel_rate; /* in KHz */
		1720	struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
		1721	};
		1722
4560	Serge	1723	struct ilk_wm_maximums {
4104	Serge	1724	uint16_t pri;
		1725	uint16_t spr;
		1726	uint16_t cur;
		1727	uint16_t fbc;
		1728	};
		1729
		1730	/* used in computing the new watermarks state */
		1731	struct intel_wm_config {
		1732	unsigned int num_pipes_active;
		1733	bool sprites_enabled;
		1734	bool sprites_scaled;
		1735	};
		1736
		1737	/*
		1738	* For both WM_PIPE and WM_LP.
		1739	* mem_value must be in 0.1us units.
		1740	*/
6084	serge	1741	static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
		1742	const struct intel_plane_state *pstate,
4104	Serge	1743	uint32_t mem_value,
		1744	bool is_lp)
		1745	{
6084	serge	1746	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
4104	Serge	1747	uint32_t method1, method2;
		1748
6084	serge	1749	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1750	return 0;
		1751
6084	serge	1752	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value);
4104	Serge	1753
		1754	if (!is_lp)
		1755	return method1;
		1756
6084	serge	1757	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1758	cstate->base.adjusted_mode.crtc_htotal,
		1759	drm_rect_width(&pstate->dst),
		1760	bpp,
4104	Serge	1761	mem_value);
		1762
		1763	return min(method1, method2);
		1764	}
		1765
		1766	/*
		1767	* For both WM_PIPE and WM_LP.
		1768	* mem_value must be in 0.1us units.
		1769	*/
6084	serge	1770	static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
		1771	const struct intel_plane_state *pstate,
4104	Serge	1772	uint32_t mem_value)
		1773	{
6084	serge	1774	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
4104	Serge	1775	uint32_t method1, method2;
		1776
6084	serge	1777	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1778	return 0;
		1779
6084	serge	1780	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value);
		1781	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1782	cstate->base.adjusted_mode.crtc_htotal,
		1783	drm_rect_width(&pstate->dst),
		1784	bpp,
4104	Serge	1785	mem_value);
		1786	return min(method1, method2);
		1787	}
		1788
		1789	/*
		1790	* For both WM_PIPE and WM_LP.
		1791	* mem_value must be in 0.1us units.
		1792	*/
6084	serge	1793	static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
		1794	const struct intel_plane_state *pstate,
4104	Serge	1795	uint32_t mem_value)
		1796	{
6660	serge	1797	/*
		1798	* We treat the cursor plane as always-on for the purposes of watermark
		1799	* calculation. Until we have two-stage watermark programming merged,
		1800	* this is necessary to avoid flickering.
		1801	*/
		1802	int cpp = 4;
		1803	int width = pstate->visible ? pstate->base.crtc_w : 64;
6084	serge	1804
6660	serge	1805	if (!cstate->base.active)
4104	Serge	1806	return 0;
		1807
6084	serge	1808	return ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1809	cstate->base.adjusted_mode.crtc_htotal,
6660	serge	1810	width, cpp, mem_value);
4104	Serge	1811	}
		1812
		1813	/* Only for WM_LP. */
6084	serge	1814	static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
		1815	const struct intel_plane_state *pstate,
4104	Serge	1816	uint32_t pri_val)
		1817	{
6084	serge	1818	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
		1819
		1820	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1821	return 0;
		1822
6084	serge	1823	return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->dst), bpp);
4104	Serge	1824	}
		1825
		1826	static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
		1827	{
4560	Serge	1828	if (INTEL_INFO(dev)->gen >= 8)
		1829	return 3072;
		1830	else if (INTEL_INFO(dev)->gen >= 7)
4104	Serge	1831	return 768;
		1832	else
		1833	return 512;
		1834	}
		1835
5060	serge	1836	static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
		1837	int level, bool is_sprite)
		1838	{
		1839	if (INTEL_INFO(dev)->gen >= 8)
		1840	/* BDW primary/sprite plane watermarks */
		1841	return level == 0 ? 255 : 2047;
		1842	else if (INTEL_INFO(dev)->gen >= 7)
		1843	/* IVB/HSW primary/sprite plane watermarks */
		1844	return level == 0 ? 127 : 1023;
		1845	else if (!is_sprite)
		1846	/* ILK/SNB primary plane watermarks */
		1847	return level == 0 ? 127 : 511;
		1848	else
		1849	/* ILK/SNB sprite plane watermarks */
		1850	return level == 0 ? 63 : 255;
		1851	}
		1852
		1853	static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
		1854	int level)
		1855	{
		1856	if (INTEL_INFO(dev)->gen >= 7)
		1857	return level == 0 ? 63 : 255;
		1858	else
		1859	return level == 0 ? 31 : 63;
		1860	}
		1861
		1862	static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
		1863	{
		1864	if (INTEL_INFO(dev)->gen >= 8)
		1865	return 31;
		1866	else
		1867	return 15;
		1868	}
		1869
4104	Serge	1870	/* Calculate the maximum primary/sprite plane watermark */
		1871	static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
		1872	int level,
		1873	const struct intel_wm_config *config,
		1874	enum intel_ddb_partitioning ddb_partitioning,
		1875	bool is_sprite)
		1876	{
		1877	unsigned int fifo_size = ilk_display_fifo_size(dev);
		1878
		1879	/* if sprites aren't enabled, sprites get nothing */
		1880	if (is_sprite && !config->sprites_enabled)
		1881	return 0;
		1882
		1883	/* HSW allows LP1+ watermarks even with multiple pipes */
		1884	if (level == 0 \|\| config->num_pipes_active > 1) {
		1885	fifo_size /= INTEL_INFO(dev)->num_pipes;
		1886
		1887	/*
		1888	* For some reason the non self refresh
		1889	* FIFO size is only half of the self
		1890	* refresh FIFO size on ILK/SNB.
		1891	*/
		1892	if (INTEL_INFO(dev)->gen <= 6)
		1893	fifo_size /= 2;
		1894	}
		1895
		1896	if (config->sprites_enabled) {
		1897	/* level 0 is always calculated with 1:1 split */
		1898	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
		1899	if (is_sprite)
		1900	fifo_size *= 5;
		1901	fifo_size /= 6;
		1902	} else {
		1903	fifo_size /= 2;
		1904	}
		1905	}
		1906
		1907	/* clamp to max that the registers can hold */
5060	serge	1908	return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
4104	Serge	1909	}
		1910
		1911	/* Calculate the maximum cursor plane watermark */
		1912	static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
		1913	int level,
		1914	const struct intel_wm_config *config)
		1915	{
		1916	/* HSW LP1+ watermarks w/ multiple pipes */
		1917	if (level > 0 && config->num_pipes_active > 1)
		1918	return 64;
		1919
		1920	/* otherwise just report max that registers can hold */
5060	serge	1921	return ilk_cursor_wm_reg_max(dev, level);
4539	Serge	1922	}
4104	Serge	1923
5060	serge	1924	static void ilk_compute_wm_maximums(const struct drm_device *dev,
6084	serge	1925	int level,
		1926	const struct intel_wm_config *config,
		1927	enum intel_ddb_partitioning ddb_partitioning,
4560	Serge	1928	struct ilk_wm_maximums *max)
4104	Serge	1929	{
		1930	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
		1931	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
		1932	max->cur = ilk_cursor_wm_max(dev, level, config);
5060	serge	1933	max->fbc = ilk_fbc_wm_reg_max(dev);
4539	Serge	1934	}
4104	Serge	1935
5060	serge	1936	static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
		1937	int level,
		1938	struct ilk_wm_maximums *max)
		1939	{
		1940	max->pri = ilk_plane_wm_reg_max(dev, level, false);
		1941	max->spr = ilk_plane_wm_reg_max(dev, level, true);
		1942	max->cur = ilk_cursor_wm_reg_max(dev, level);
		1943	max->fbc = ilk_fbc_wm_reg_max(dev);
		1944	}
		1945
4560	Serge	1946	static bool ilk_validate_wm_level(int level,
		1947	const struct ilk_wm_maximums *max,
6084	serge	1948	struct intel_wm_level *result)
4104	Serge	1949	{
		1950	bool ret;
		1951
		1952	/* already determined to be invalid? */
		1953	if (!result->enable)
		1954	return false;
		1955
		1956	result->enable = result->pri_val <= max->pri &&
		1957	result->spr_val <= max->spr &&
		1958	result->cur_val <= max->cur;
		1959
		1960	ret = result->enable;
		1961
		1962	/*
		1963	* HACK until we can pre-compute everything,
		1964	* and thus fail gracefully if LP0 watermarks
		1965	* are exceeded...
		1966	*/
		1967	if (level == 0 && !result->enable) {
		1968	if (result->pri_val > max->pri)
		1969	DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
		1970	level, result->pri_val, max->pri);
		1971	if (result->spr_val > max->spr)
		1972	DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
		1973	level, result->spr_val, max->spr);
		1974	if (result->cur_val > max->cur)
		1975	DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
		1976	level, result->cur_val, max->cur);
		1977
		1978	result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
		1979	result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
		1980	result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
		1981	result->enable = true;
		1982	}
		1983
		1984	return ret;
		1985	}
		1986
5060	serge	1987	static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
6084	serge	1988	const struct intel_crtc *intel_crtc,
4104	Serge	1989	int level,
6084	serge	1990	struct intel_crtc_state *cstate,
4104	Serge	1991	struct intel_wm_level *result)
		1992	{
6084	serge	1993	struct intel_plane *intel_plane;
4104	Serge	1994	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
		1995	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
		1996	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
		1997
		1998	/* WM1+ latency values stored in 0.5us units */
		1999	if (level > 0) {
		2000	pri_latency *= 5;
		2001	spr_latency *= 5;
		2002	cur_latency *= 5;
		2003	}
		2004
6084	serge	2005	for_each_intel_plane_on_crtc(dev_priv->dev, intel_crtc, intel_plane) {
		2006	struct intel_plane_state *pstate =
		2007	to_intel_plane_state(intel_plane->base.state);
		2008
		2009	switch (intel_plane->base.type) {
		2010	case DRM_PLANE_TYPE_PRIMARY:
		2011	result->pri_val = ilk_compute_pri_wm(cstate, pstate,
		2012	pri_latency,
		2013	level);
		2014	result->fbc_val = ilk_compute_fbc_wm(cstate, pstate,
		2015	result->pri_val);
		2016	break;
		2017	case DRM_PLANE_TYPE_OVERLAY:
		2018	result->spr_val = ilk_compute_spr_wm(cstate, pstate,
		2019	spr_latency);
		2020	break;
		2021	case DRM_PLANE_TYPE_CURSOR:
		2022	result->cur_val = ilk_compute_cur_wm(cstate, pstate,
		2023	cur_latency);
		2024	break;
		2025	}
		2026	}
		2027
4104	Serge	2028	result->enable = true;
		2029	}
		2030
		2031	static uint32_t
		2032	hsw_compute_linetime_wm(struct drm_device dev, struct drm_crtc crtc)
		2033	{
3031	serge	2034	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2035	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6084	serge	2036	const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
4104	Serge	2037	u32 linetime, ips_linetime;
3031	serge	2038
6084	serge	2039	if (!intel_crtc->active)
4104	Serge	2040	return 0;
3031	serge	2041
		2042	/* The WM are computed with base on how long it takes to fill a single
		2043	* row at the given clock rate, multiplied by 8.
		2044	* */
6084	serge	2045	linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
		2046	adjusted_mode->crtc_clock);
		2047	ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
		2048	dev_priv->cdclk_freq);
3031	serge	2049
4104	Serge	2050	return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) \|
		2051	PIPE_WM_LINETIME_TIME(linetime);
		2052	}
		2053
5354	serge	2054	static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
4104	Serge	2055	{
		2056	struct drm_i915_private *dev_priv = dev->dev_private;
		2057
5354	serge	2058	if (IS_GEN9(dev)) {
		2059	uint32_t val;
		2060	int ret, i;
		2061	int level, max_level = ilk_wm_max_level(dev);
		2062
		2063	/* read the first set of memory latencies[0:3] */
		2064	val = 0; /* data0 to be programmed to 0 for first set */
		2065	mutex_lock(&dev_priv->rps.hw_lock);
		2066	ret = sandybridge_pcode_read(dev_priv,
		2067	GEN9_PCODE_READ_MEM_LATENCY,
		2068	&val);
		2069	mutex_unlock(&dev_priv->rps.hw_lock);
		2070
		2071	if (ret) {
		2072	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2073	return;
		2074	}
		2075
		2076	wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2077	wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2078	GEN9_MEM_LATENCY_LEVEL_MASK;
		2079	wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2080	GEN9_MEM_LATENCY_LEVEL_MASK;
		2081	wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2082	GEN9_MEM_LATENCY_LEVEL_MASK;
		2083
		2084	/* read the second set of memory latencies[4:7] */
		2085	val = 1; /* data0 to be programmed to 1 for second set */
		2086	mutex_lock(&dev_priv->rps.hw_lock);
		2087	ret = sandybridge_pcode_read(dev_priv,
		2088	GEN9_PCODE_READ_MEM_LATENCY,
		2089	&val);
		2090	mutex_unlock(&dev_priv->rps.hw_lock);
		2091	if (ret) {
		2092	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2093	return;
		2094	}
		2095
		2096	wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2097	wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2098	GEN9_MEM_LATENCY_LEVEL_MASK;
		2099	wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2100	GEN9_MEM_LATENCY_LEVEL_MASK;
		2101	wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2102	GEN9_MEM_LATENCY_LEVEL_MASK;
		2103
		2104	/*
6084	serge	2105	* WaWmMemoryReadLatency:skl
		2106	*
5354	serge	2107	* punit doesn't take into account the read latency so we need
		2108	* to add 2us to the various latency levels we retrieve from
		2109	* the punit.
		2110	* - W0 is a bit special in that it's the only level that
		2111	* can't be disabled if we want to have display working, so
		2112	* we always add 2us there.
		2113	* - For levels >=1, punit returns 0us latency when they are
		2114	* disabled, so we respect that and don't add 2us then
		2115	*
		2116	* Additionally, if a level n (n > 1) has a 0us latency, all
		2117	* levels m (m >= n) need to be disabled. We make sure to
		2118	* sanitize the values out of the punit to satisfy this
		2119	* requirement.
		2120	*/
		2121	wm[0] += 2;
		2122	for (level = 1; level <= max_level; level++)
		2123	if (wm[level] != 0)
		2124	wm[level] += 2;
		2125	else {
		2126	for (i = level + 1; i <= max_level; i++)
		2127	wm[i] = 0;
		2128
		2129	break;
		2130	}
		2131	} else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2132	uint64_t sskpd = I915_READ64(MCH_SSKPD);
		2133
		2134	wm[0] = (sskpd >> 56) & 0xFF;
		2135	if (wm[0] == 0)
		2136	wm[0] = sskpd & 0xF;
		2137	wm[1] = (sskpd >> 4) & 0xFF;
		2138	wm[2] = (sskpd >> 12) & 0xFF;
		2139	wm[3] = (sskpd >> 20) & 0x1FF;
		2140	wm[4] = (sskpd >> 32) & 0x1FF;
		2141	} else if (INTEL_INFO(dev)->gen >= 6) {
		2142	uint32_t sskpd = I915_READ(MCH_SSKPD);
		2143
		2144	wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		2145	wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		2146	wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		2147	wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
		2148	} else if (INTEL_INFO(dev)->gen >= 5) {
		2149	uint32_t mltr = I915_READ(MLTR_ILK);
		2150
		2151	/* ILK primary LP0 latency is 700 ns */
		2152	wm[0] = 7;
		2153	wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		2154	wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
		2155	}
		2156	}
		2157
		2158	static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2159	{
		2160	/* ILK sprite LP0 latency is 1300 ns */
		2161	if (INTEL_INFO(dev)->gen == 5)
		2162	wm[0] = 13;
		2163	}
		2164
		2165	static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2166	{
		2167	/* ILK cursor LP0 latency is 1300 ns */
		2168	if (INTEL_INFO(dev)->gen == 5)
		2169	wm[0] = 13;
		2170
		2171	/* WaDoubleCursorLP3Latency:ivb */
		2172	if (IS_IVYBRIDGE(dev))
		2173	wm[3] *= 2;
		2174	}
		2175
5060	serge	2176	int ilk_wm_max_level(const struct drm_device *dev)
4560	Serge	2177	{
		2178	/* how many WM levels are we expecting */
6084	serge	2179	if (INTEL_INFO(dev)->gen >= 9)
5354	serge	2180	return 7;
		2181	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
4560	Serge	2182	return 4;
		2183	else if (INTEL_INFO(dev)->gen >= 6)
		2184	return 3;
		2185	else
		2186	return 2;
		2187	}
		2188
4104	Serge	2189	static void intel_print_wm_latency(struct drm_device *dev,
		2190	const char *name,
5354	serge	2191	const uint16_t wm[8])
4104	Serge	2192	{
4560	Serge	2193	int level, max_level = ilk_wm_max_level(dev);
4104	Serge	2194
		2195	for (level = 0; level <= max_level; level++) {
		2196	unsigned int latency = wm[level];
		2197
		2198	if (latency == 0) {
		2199	DRM_ERROR("%s WM%d latency not provided\n",
		2200	name, level);
		2201	continue;
		2202	}
		2203
5354	serge	2204	/*
		2205	* - latencies are in us on gen9.
		2206	* - before then, WM1+ latency values are in 0.5us units
		2207	*/
		2208	if (IS_GEN9(dev))
		2209	latency *= 10;
		2210	else if (level > 0)
4104	Serge	2211	latency *= 5;
		2212
		2213	DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
		2214	name, level, wm[level],
		2215	latency / 10, latency % 10);
		2216	}
		2217	}
		2218
5060	serge	2219	static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
		2220	uint16_t wm[5], uint16_t min)
4104	Serge	2221	{
5060	serge	2222	int level, max_level = ilk_wm_max_level(dev_priv->dev);
		2223
		2224	if (wm[0] >= min)
		2225	return false;
		2226
		2227	wm[0] = max(wm[0], min);
		2228	for (level = 1; level <= max_level; level++)
		2229	wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
		2230
		2231	return true;
		2232	}
		2233
		2234	static void snb_wm_latency_quirk(struct drm_device *dev)
		2235	{
4104	Serge	2236	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2237	bool changed;
4104	Serge	2238
5060	serge	2239	/*
		2240	* The BIOS provided WM memory latency values are often
		2241	* inadequate for high resolution displays. Adjust them.
		2242	*/
		2243	changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) \|
		2244	ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) \|
		2245	ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
		2246
		2247	if (!changed)
		2248	return;
		2249
		2250	DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
		2251	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2252	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2253	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
		2254	}
		2255
		2256	static void ilk_setup_wm_latency(struct drm_device *dev)
		2257	{
		2258	struct drm_i915_private *dev_priv = dev->dev_private;
		2259
4104	Serge	2260	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
		2261
		2262	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
		2263	sizeof(dev_priv->wm.pri_latency));
		2264	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
		2265	sizeof(dev_priv->wm.pri_latency));
		2266
		2267	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
		2268	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
		2269
		2270	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2271	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2272	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
5060	serge	2273
		2274	if (IS_GEN6(dev))
		2275	snb_wm_latency_quirk(dev);
4104	Serge	2276	}
		2277
5354	serge	2278	static void skl_setup_wm_latency(struct drm_device *dev)
		2279	{
		2280	struct drm_i915_private *dev_priv = dev->dev_private;
		2281
		2282	intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
		2283	intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
		2284	}
		2285
5060	serge	2286	static void ilk_compute_wm_config(struct drm_device *dev,
		2287	struct intel_wm_config *config)
		2288	{
		2289	struct intel_crtc *intel_crtc;
		2290
		2291	/* Compute the currently _active_ config */
		2292	for_each_intel_crtc(dev, intel_crtc) {
		2293	const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
		2294
		2295	if (!wm->pipe_enabled)
		2296	continue;
		2297
		2298	config->sprites_enabled \|= wm->sprites_enabled;
		2299	config->sprites_scaled \|= wm->sprites_scaled;
		2300	config->num_pipes_active++;
4104	Serge	2301	}
4560	Serge	2302	}
4104	Serge	2303
4560	Serge	2304	/* Compute new watermarks for the pipe */
6084	serge	2305	static bool intel_compute_pipe_wm(struct intel_crtc_state *cstate,
4560	Serge	2306	struct intel_pipe_wm *pipe_wm)
		2307	{
6084	serge	2308	struct drm_crtc *crtc = cstate->base.crtc;
4560	Serge	2309	struct drm_device *dev = crtc->dev;
5060	serge	2310	const struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	2311	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2312	struct intel_plane *intel_plane;
		2313	struct intel_plane_state *sprstate = NULL;
4560	Serge	2314	int level, max_level = ilk_wm_max_level(dev);
		2315	/* LP0 watermark maximums depend on this pipe alone */
		2316	struct intel_wm_config config = {
		2317	.num_pipes_active = 1,
		2318	};
		2319	struct ilk_wm_maximums max;
4104	Serge	2320
6084	serge	2321	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		2322	if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY) {
		2323	sprstate = to_intel_plane_state(intel_plane->base.state);
		2324	break;
		2325	}
		2326	}
4560	Serge	2327
6084	serge	2328	config.sprites_enabled = sprstate->visible;
		2329	config.sprites_scaled = sprstate->visible &&
		2330	(drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 \|\|
		2331	drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16);
		2332
		2333	pipe_wm->pipe_enabled = cstate->base.active;
		2334	pipe_wm->sprites_enabled = sprstate->visible;
		2335	pipe_wm->sprites_scaled = config.sprites_scaled;
		2336
4560	Serge	2337	/* ILK/SNB: LP2+ watermarks only w/o sprites */
6084	serge	2338	if (INTEL_INFO(dev)->gen <= 6 && sprstate->visible)
4560	Serge	2339	max_level = 1;
		2340
		2341	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
6084	serge	2342	if (config.sprites_scaled)
4560	Serge	2343	max_level = 0;
		2344
6084	serge	2345	ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate, &pipe_wm->wm[0]);
4560	Serge	2346
		2347	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2348	pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
		2349
5060	serge	2350	/* LP0 watermarks always use 1/2 DDB partitioning */
		2351	ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
		2352
4560	Serge	2353	/* At least LP0 must be valid */
5060	serge	2354	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
		2355	return false;
		2356
		2357	ilk_compute_wm_reg_maximums(dev, 1, &max);
		2358
		2359	for (level = 1; level <= max_level; level++) {
		2360	struct intel_wm_level wm = {};
		2361
6084	serge	2362	ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate, &wm);
5060	serge	2363
		2364	/*
		2365	* Disable any watermark level that exceeds the
		2366	* register maximums since such watermarks are
		2367	* always invalid.
		2368	*/
		2369	if (!ilk_validate_wm_level(level, &max, &wm))
		2370	break;
		2371
		2372	pipe_wm->wm[level] = wm;
		2373	}
		2374
		2375	return true;
4104	Serge	2376	}
		2377
4560	Serge	2378	/*
		2379	* Merge the watermarks from all active pipes for a specific level.
		2380	*/
		2381	static void ilk_merge_wm_level(struct drm_device *dev,
		2382	int level,
		2383	struct intel_wm_level *ret_wm)
4104	Serge	2384	{
4560	Serge	2385	const struct intel_crtc *intel_crtc;
4104	Serge	2386
5060	serge	2387	ret_wm->enable = true;
4104	Serge	2388
5060	serge	2389	for_each_intel_crtc(dev, intel_crtc) {
		2390	const struct intel_pipe_wm *active = &intel_crtc->wm.active;
		2391	const struct intel_wm_level *wm = &active->wm[level];
		2392
		2393	if (!active->pipe_enabled)
		2394	continue;
		2395
		2396	/*
		2397	* The watermark values may have been used in the past,
		2398	* so we must maintain them in the registers for some
		2399	* time even if the level is now disabled.
		2400	*/
4560	Serge	2401	if (!wm->enable)
5060	serge	2402	ret_wm->enable = false;
4104	Serge	2403
4560	Serge	2404	ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
		2405	ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
		2406	ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
		2407	ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
		2408	}
		2409	}
		2410
		2411	/*
		2412	* Merge all low power watermarks for all active pipes.
		2413	*/
		2414	static void ilk_wm_merge(struct drm_device *dev,
		2415	const struct intel_wm_config *config,
		2416	const struct ilk_wm_maximums *max,
		2417	struct intel_pipe_wm *merged)
		2418	{
6084	serge	2419	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2420	int level, max_level = ilk_wm_max_level(dev);
5060	serge	2421	int last_enabled_level = max_level;
4560	Serge	2422
		2423	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
		2424	if ((INTEL_INFO(dev)->gen <= 6 \|\| IS_IVYBRIDGE(dev)) &&
		2425	config->num_pipes_active > 1)
		2426	return;
		2427
		2428	/* ILK: FBC WM must be disabled always */
		2429	merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
		2430
		2431	/* merge each WM1+ level */
4104	Serge	2432	for (level = 1; level <= max_level; level++) {
4560	Serge	2433	struct intel_wm_level *wm = &merged->wm[level];
		2434
		2435	ilk_merge_wm_level(dev, level, wm);
		2436
5060	serge	2437	if (level > last_enabled_level)
		2438	wm->enable = false;
		2439	else if (!ilk_validate_wm_level(level, max, wm))
		2440	/* make sure all following levels get disabled */
		2441	last_enabled_level = level - 1;
4560	Serge	2442
		2443	/*
		2444	* The spec says it is preferred to disable
		2445	* FBC WMs instead of disabling a WM level.
		2446	*/
		2447	if (wm->fbc_val > max->fbc) {
5060	serge	2448	if (wm->enable)
6084	serge	2449	merged->fbc_wm_enabled = false;
4560	Serge	2450	wm->fbc_val = 0;
4104	Serge	2451	}
		2452	}
		2453
4560	Serge	2454	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
		2455	/*
		2456	* FIXME this is racy. FBC might get enabled later.
		2457	* What we should check here is whether FBC can be
		2458	* enabled sometime later.
		2459	*/
6084	serge	2460	if (IS_GEN5(dev) && !merged->fbc_wm_enabled &&
		2461	intel_fbc_enabled(dev_priv)) {
4560	Serge	2462	for (level = 2; level <= max_level; level++) {
		2463	struct intel_wm_level *wm = &merged->wm[level];
		2464
		2465	wm->enable = false;
		2466	}
		2467	}
		2468	}
		2469
		2470	static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
		2471	{
		2472	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
		2473	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
		2474	}
		2475
		2476	/* The value we need to program into the WM_LPx latency field */
		2477	static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
		2478	{
		2479	struct drm_i915_private *dev_priv = dev->dev_private;
		2480
		2481	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2482	return 2 * level;
		2483	else
		2484	return dev_priv->wm.pri_latency[level];
		2485	}
		2486
		2487	static void ilk_compute_wm_results(struct drm_device *dev,
		2488	const struct intel_pipe_wm *merged,
		2489	enum intel_ddb_partitioning partitioning,
		2490	struct ilk_wm_values *results)
		2491	{
		2492	struct intel_crtc *intel_crtc;
		2493	int level, wm_lp;
		2494
		2495	results->enable_fbc_wm = merged->fbc_wm_enabled;
		2496	results->partitioning = partitioning;
		2497
		2498	/* LP1+ register values */
4104	Serge	2499	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2500	const struct intel_wm_level *r;
		2501
4560	Serge	2502	level = ilk_wm_lp_to_level(wm_lp, merged);
		2503
		2504	r = &merged->wm[level];
4104	Serge	2505
5060	serge	2506	/*
		2507	* Maintain the watermark values even if the level is
		2508	* disabled. Doing otherwise could cause underruns.
		2509	*/
		2510	results->wm_lp[wm_lp - 1] =
4560	Serge	2511	(ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) \|
		2512	(r->pri_val << WM1_LP_SR_SHIFT) \|
		2513	r->cur_val;
		2514
5060	serge	2515	if (r->enable)
		2516	results->wm_lp[wm_lp - 1] \|= WM1_LP_SR_EN;
		2517
4560	Serge	2518	if (INTEL_INFO(dev)->gen >= 8)
		2519	results->wm_lp[wm_lp - 1] \|=
		2520	r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
		2521	else
		2522	results->wm_lp[wm_lp - 1] \|=
		2523	r->fbc_val << WM1_LP_FBC_SHIFT;
		2524
5060	serge	2525	/*
		2526	* Always set WM1S_LP_EN when spr_val != 0, even if the
		2527	* level is disabled. Doing otherwise could cause underruns.
		2528	*/
4560	Serge	2529	if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
		2530	WARN_ON(wm_lp != 1);
		2531	results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN \| r->spr_val;
		2532	} else
6084	serge	2533	results->wm_lp_spr[wm_lp - 1] = r->spr_val;
4104	Serge	2534	}
		2535
4560	Serge	2536	/* LP0 register values */
5060	serge	2537	for_each_intel_crtc(dev, intel_crtc) {
4560	Serge	2538	enum pipe pipe = intel_crtc->pipe;
		2539	const struct intel_wm_level *r =
		2540	&intel_crtc->wm.active.wm[0];
4104	Serge	2541
4560	Serge	2542	if (WARN_ON(!r->enable))
		2543	continue;
		2544
		2545	results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
		2546
		2547	results->wm_pipe[pipe] =
		2548	(r->pri_val << WM0_PIPE_PLANE_SHIFT) \|
		2549	(r->spr_val << WM0_PIPE_SPRITE_SHIFT) \|
		2550	r->cur_val;
4104	Serge	2551	}
		2552	}
		2553
		2554	/* Find the result with the highest level enabled. Check for enable_fbc_wm in
		2555	* case both are at the same level. Prefer r1 in case they're the same. */
4560	Serge	2556	static struct intel_pipe_wm ilk_find_best_result(struct drm_device dev,
		2557	struct intel_pipe_wm *r1,
		2558	struct intel_pipe_wm *r2)
4104	Serge	2559	{
4560	Serge	2560	int level, max_level = ilk_wm_max_level(dev);
		2561	int level1 = 0, level2 = 0;
4104	Serge	2562
4560	Serge	2563	for (level = 1; level <= max_level; level++) {
		2564	if (r1->wm[level].enable)
		2565	level1 = level;
		2566	if (r2->wm[level].enable)
		2567	level2 = level;
4104	Serge	2568	}
		2569
4560	Serge	2570	if (level1 == level2) {
		2571	if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
4104	Serge	2572	return r2;
		2573	else
		2574	return r1;
4560	Serge	2575	} else if (level1 > level2) {
4104	Serge	2576	return r1;
		2577	} else {
		2578	return r2;
		2579	}
		2580	}
		2581
4560	Serge	2582	/* dirty bits used to track which watermarks need changes */
		2583	#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
		2584	#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
		2585	#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
		2586	#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) \| WM_DIRTY_LP(2) \| WM_DIRTY_LP(3))
		2587	#define WM_DIRTY_FBC (1 << 24)
		2588	#define WM_DIRTY_DDB (1 << 25)
		2589
5354	serge	2590	static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
4560	Serge	2591	const struct ilk_wm_values *old,
		2592	const struct ilk_wm_values *new)
		2593	{
		2594	unsigned int dirty = 0;
		2595	enum pipe pipe;
		2596	int wm_lp;
		2597
5354	serge	2598	for_each_pipe(dev_priv, pipe) {
4560	Serge	2599	if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
		2600	dirty \|= WM_DIRTY_LINETIME(pipe);
		2601	/* Must disable LP1+ watermarks too */
		2602	dirty \|= WM_DIRTY_LP_ALL;
		2603	}
		2604
		2605	if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
		2606	dirty \|= WM_DIRTY_PIPE(pipe);
		2607	/* Must disable LP1+ watermarks too */
		2608	dirty \|= WM_DIRTY_LP_ALL;
		2609	}
		2610	}
		2611
		2612	if (old->enable_fbc_wm != new->enable_fbc_wm) {
		2613	dirty \|= WM_DIRTY_FBC;
		2614	/* Must disable LP1+ watermarks too */
		2615	dirty \|= WM_DIRTY_LP_ALL;
		2616	}
		2617
		2618	if (old->partitioning != new->partitioning) {
		2619	dirty \|= WM_DIRTY_DDB;
		2620	/* Must disable LP1+ watermarks too */
		2621	dirty \|= WM_DIRTY_LP_ALL;
		2622	}
		2623
		2624	/* LP1+ watermarks already deemed dirty, no need to continue */
		2625	if (dirty & WM_DIRTY_LP_ALL)
		2626	return dirty;
		2627
		2628	/* Find the lowest numbered LP1+ watermark in need of an update... */
		2629	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2630	if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] \|\|
		2631	old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
		2632	break;
		2633	}
		2634
		2635	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
		2636	for (; wm_lp <= 3; wm_lp++)
		2637	dirty \|= WM_DIRTY_LP(wm_lp);
		2638
		2639	return dirty;
		2640	}
		2641
		2642	static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
		2643	unsigned int dirty)
		2644	{
		2645	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2646	bool changed = false;
		2647
		2648	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
		2649	previous->wm_lp[2] &= ~WM1_LP_SR_EN;
		2650	I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
		2651	changed = true;
		2652	}
		2653	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
		2654	previous->wm_lp[1] &= ~WM1_LP_SR_EN;
		2655	I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
		2656	changed = true;
		2657	}
		2658	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
		2659	previous->wm_lp[0] &= ~WM1_LP_SR_EN;
		2660	I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
		2661	changed = true;
		2662	}
		2663
		2664	/*
		2665	* Don't touch WM1S_LP_EN here.
		2666	* Doing so could cause underruns.
		2667	*/
		2668
		2669	return changed;
		2670	}
		2671
4104	Serge	2672	/*
		2673	* The spec says we shouldn't write when we don't need, because every write
		2674	* causes WMs to be re-evaluated, expending some power.
6084	serge	2675	*/
4560	Serge	2676	static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
		2677	struct ilk_wm_values *results)
4104	Serge	2678	{
4560	Serge	2679	struct drm_device *dev = dev_priv->dev;
		2680	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2681	unsigned int dirty;
4104	Serge	2682	uint32_t val;
3031	serge	2683
5354	serge	2684	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
4560	Serge	2685	if (!dirty)
4104	Serge	2686	return;
		2687
4560	Serge	2688	_ilk_disable_lp_wm(dev_priv, dirty);
4104	Serge	2689
4560	Serge	2690	if (dirty & WM_DIRTY_PIPE(PIPE_A))
4104	Serge	2691	I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
4560	Serge	2692	if (dirty & WM_DIRTY_PIPE(PIPE_B))
4104	Serge	2693	I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
4560	Serge	2694	if (dirty & WM_DIRTY_PIPE(PIPE_C))
4104	Serge	2695	I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
		2696
4560	Serge	2697	if (dirty & WM_DIRTY_LINETIME(PIPE_A))
4104	Serge	2698	I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
4560	Serge	2699	if (dirty & WM_DIRTY_LINETIME(PIPE_B))
4104	Serge	2700	I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
4560	Serge	2701	if (dirty & WM_DIRTY_LINETIME(PIPE_C))
4104	Serge	2702	I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
		2703
4560	Serge	2704	if (dirty & WM_DIRTY_DDB) {
		2705	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
6084	serge	2706	val = I915_READ(WM_MISC);
4560	Serge	2707	if (results->partitioning == INTEL_DDB_PART_1_2)
6084	serge	2708	val &= ~WM_MISC_DATA_PARTITION_5_6;
		2709	else
		2710	val \|= WM_MISC_DATA_PARTITION_5_6;
		2711	I915_WRITE(WM_MISC, val);
4560	Serge	2712	} else {
		2713	val = I915_READ(DISP_ARB_CTL2);
		2714	if (results->partitioning == INTEL_DDB_PART_1_2)
		2715	val &= ~DISP_DATA_PARTITION_5_6;
		2716	else
		2717	val \|= DISP_DATA_PARTITION_5_6;
		2718	I915_WRITE(DISP_ARB_CTL2, val);
		2719	}
4104	Serge	2720	}
		2721
4560	Serge	2722	if (dirty & WM_DIRTY_FBC) {
4104	Serge	2723	val = I915_READ(DISP_ARB_CTL);
		2724	if (results->enable_fbc_wm)
		2725	val &= ~DISP_FBC_WM_DIS;
		2726	else
		2727	val \|= DISP_FBC_WM_DIS;
		2728	I915_WRITE(DISP_ARB_CTL, val);
		2729	}
		2730
4560	Serge	2731	if (dirty & WM_DIRTY_LP(1) &&
		2732	previous->wm_lp_spr[0] != results->wm_lp_spr[0])
4104	Serge	2733	I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
4560	Serge	2734
		2735	if (INTEL_INFO(dev)->gen >= 7) {
		2736	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
6084	serge	2737	I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
4560	Serge	2738	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
6084	serge	2739	I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
4560	Serge	2740	}
4104	Serge	2741
4560	Serge	2742	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
4104	Serge	2743	I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
4560	Serge	2744	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
4104	Serge	2745	I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
4560	Serge	2746	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
4104	Serge	2747	I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
4560	Serge	2748
		2749	dev_priv->wm.hw = *results;
3031	serge	2750	}
		2751
4560	Serge	2752	static bool ilk_disable_lp_wm(struct drm_device *dev)
4104	Serge	2753	{
		2754	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2755
		2756	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
		2757	}
		2758
5354	serge	2759	/*
		2760	* On gen9, we need to allocate Display Data Buffer (DDB) portions to the
		2761	* different active planes.
		2762	*/
		2763
		2764	#define SKL_DDB_SIZE 896 /* in blocks */
6084	serge	2765	#define BXT_DDB_SIZE 512
5354	serge	2766
		2767	static void
		2768	skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
		2769	struct drm_crtc *for_crtc,
		2770	const struct intel_wm_config *config,
		2771	const struct skl_pipe_wm_parameters *params,
		2772	struct skl_ddb_entry alloc / out */)
		2773	{
		2774	struct drm_crtc *crtc;
		2775	unsigned int pipe_size, ddb_size;
		2776	int nth_active_pipe;
		2777
		2778	if (!params->active) {
		2779	alloc->start = 0;
		2780	alloc->end = 0;
		2781	return;
		2782	}
		2783
6084	serge	2784	if (IS_BROXTON(dev))
		2785	ddb_size = BXT_DDB_SIZE;
		2786	else
		2787	ddb_size = SKL_DDB_SIZE;
5354	serge	2788
		2789	ddb_size -= 4; /* 4 blocks for bypass path allocation */
		2790
		2791	nth_active_pipe = 0;
		2792	for_each_crtc(dev, crtc) {
6084	serge	2793	if (!to_intel_crtc(crtc)->active)
5354	serge	2794	continue;
		2795
		2796	if (crtc == for_crtc)
		2797	break;
		2798
		2799	nth_active_pipe++;
		2800	}
		2801
		2802	pipe_size = ddb_size / config->num_pipes_active;
		2803	alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
		2804	alloc->end = alloc->start + pipe_size;
		2805	}
		2806
		2807	static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
		2808	{
		2809	if (config->num_pipes_active == 1)
		2810	return 32;
		2811
		2812	return 8;
		2813	}
		2814
		2815	static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
		2816	{
		2817	entry->start = reg & 0x3ff;
		2818	entry->end = (reg >> 16) & 0x3ff;
		2819	if (entry->end)
		2820	entry->end += 1;
		2821	}
		2822
		2823	void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
		2824	struct skl_ddb_allocation ddb / out */)
		2825	{
		2826	enum pipe pipe;
		2827	int plane;
		2828	u32 val;
		2829
6084	serge	2830	memset(ddb, 0, sizeof(*ddb));
		2831
5354	serge	2832	for_each_pipe(dev_priv, pipe) {
6084	serge	2833	if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_PIPE(pipe)))
		2834	continue;
		2835
		2836	for_each_plane(dev_priv, pipe, plane) {
5354	serge	2837	val = I915_READ(PLANE_BUF_CFG(pipe, plane));
		2838	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
		2839	val);
		2840	}
		2841
		2842	val = I915_READ(CUR_BUF_CFG(pipe));
6084	serge	2843	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][PLANE_CURSOR],
		2844	val);
5354	serge	2845	}
		2846	}
		2847
		2848	static unsigned int
6084	serge	2849	skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p, int y)
5354	serge	2850	{
6084	serge	2851
		2852	/* for planar format */
		2853	if (p->y_bytes_per_pixel) {
		2854	if (y) /* y-plane data rate */
		2855	return p->horiz_pixels * p->vert_pixels * p->y_bytes_per_pixel;
		2856	else /* uv-plane data rate */
		2857	return (p->horiz_pixels/2) * (p->vert_pixels/2) * p->bytes_per_pixel;
		2858	}
		2859
		2860	/* for packed formats */
5354	serge	2861	return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
		2862	}
		2863
		2864	/*
		2865	* We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
		2866	* a 8192x4096@32bpp framebuffer:
		2867	* 3 * 4096 * 8192 * 4 < 2^32
		2868	*/
		2869	static unsigned int
		2870	skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
		2871	const struct skl_pipe_wm_parameters *params)
		2872	{
		2873	unsigned int total_data_rate = 0;
		2874	int plane;
		2875
		2876	for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
		2877	const struct intel_plane_wm_parameters *p;
		2878
		2879	p = ¶ms->plane[plane];
		2880	if (!p->enabled)
		2881	continue;
		2882
6084	serge	2883	total_data_rate += skl_plane_relative_data_rate(p, 0); /* packed/uv */
		2884	if (p->y_bytes_per_pixel) {
		2885	total_data_rate += skl_plane_relative_data_rate(p, 1); /* y-plane */
		2886	}
5354	serge	2887	}
		2888
		2889	return total_data_rate;
		2890	}
		2891
		2892	static void
		2893	skl_allocate_pipe_ddb(struct drm_crtc *crtc,
		2894	const struct intel_wm_config *config,
		2895	const struct skl_pipe_wm_parameters *params,
		2896	struct skl_ddb_allocation ddb / out */)
		2897	{
		2898	struct drm_device *dev = crtc->dev;
6084	serge	2899	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	2900	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2901	enum pipe pipe = intel_crtc->pipe;
		2902	struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
		2903	uint16_t alloc_size, start, cursor_blocks;
6084	serge	2904	uint16_t minimum[I915_MAX_PLANES];
		2905	uint16_t y_minimum[I915_MAX_PLANES];
5354	serge	2906	unsigned int total_data_rate;
		2907	int plane;
		2908
		2909	skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
		2910	alloc_size = skl_ddb_entry_size(alloc);
		2911	if (alloc_size == 0) {
		2912	memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
6084	serge	2913	memset(&ddb->plane[pipe][PLANE_CURSOR], 0,
		2914	sizeof(ddb->plane[pipe][PLANE_CURSOR]));
5354	serge	2915	return;
		2916	}
		2917
		2918	cursor_blocks = skl_cursor_allocation(config);
6084	serge	2919	ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks;
		2920	ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
5354	serge	2921
		2922	alloc_size -= cursor_blocks;
		2923	alloc->end -= cursor_blocks;
		2924
6084	serge	2925	/* 1. Allocate the mininum required blocks for each active plane */
		2926	for_each_plane(dev_priv, pipe, plane) {
		2927	const struct intel_plane_wm_parameters *p;
		2928
		2929	p = ¶ms->plane[plane];
		2930	if (!p->enabled)
		2931	continue;
		2932
		2933	minimum[plane] = 8;
		2934	alloc_size -= minimum[plane];
		2935	y_minimum[plane] = p->y_bytes_per_pixel ? 8 : 0;
		2936	alloc_size -= y_minimum[plane];
		2937	}
		2938
		2939	/*
		2940	* 2. Distribute the remaining space in proportion to the amount of
		2941	* data each plane needs to fetch from memory.
5354	serge	2942	*
		2943	* FIXME: we may not allocate every single block here.
6084	serge	2944	*/
5354	serge	2945	total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
		2946
		2947	start = alloc->start;
		2948	for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
		2949	const struct intel_plane_wm_parameters *p;
6084	serge	2950	unsigned int data_rate, y_data_rate;
		2951	uint16_t plane_blocks, y_plane_blocks = 0;
5354	serge	2952
		2953	p = ¶ms->plane[plane];
		2954	if (!p->enabled)
		2955	continue;
		2956
6084	serge	2957	data_rate = skl_plane_relative_data_rate(p, 0);
5354	serge	2958
		2959	/*
6084	serge	2960	* allocation for (packed formats) or (uv-plane part of planar format):
5354	serge	2961	* promote the expression to 64 bits to avoid overflowing, the
		2962	* result is < available as data_rate / total_data_rate < 1
		2963	*/
6084	serge	2964	plane_blocks = minimum[plane];
		2965	plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
		2966	total_data_rate);
5354	serge	2967
		2968	ddb->plane[pipe][plane].start = start;
		2969	ddb->plane[pipe][plane].end = start + plane_blocks;
		2970
		2971	start += plane_blocks;
6084	serge	2972
		2973	/*
		2974	* allocation for y_plane part of planar format:
		2975	*/
		2976	if (p->y_bytes_per_pixel) {
		2977	y_data_rate = skl_plane_relative_data_rate(p, 1);
		2978	y_plane_blocks = y_minimum[plane];
		2979	y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
		2980	total_data_rate);
		2981
		2982	ddb->y_plane[pipe][plane].start = start;
		2983	ddb->y_plane[pipe][plane].end = start + y_plane_blocks;
		2984
		2985	start += y_plane_blocks;
		2986	}
		2987
5354	serge	2988	}
		2989
		2990	}
		2991
6084	serge	2992	static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
5354	serge	2993	{
		2994	/* TODO: Take into account the scalers once we support them */
6084	serge	2995	return config->base.adjusted_mode.crtc_clock;
5354	serge	2996	}
		2997
		2998	/*
		2999	* The max latency should be 257 (max the punit can code is 255 and we add 2us
		3000	* for the read latency) and bytes_per_pixel should always be <= 8, so that
		3001	* should allow pixel_rate up to ~2 GHz which seems sufficient since max
		3002	* 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
		3003	*/
		3004	static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		3005	uint32_t latency)
		3006	{
		3007	uint32_t wm_intermediate_val, ret;
		3008
		3009	if (latency == 0)
		3010	return UINT_MAX;
		3011
6084	serge	3012	wm_intermediate_val = latency * pixel_rate * bytes_per_pixel / 512;
5354	serge	3013	ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
		3014
		3015	return ret;
		3016	}
		3017
		3018	static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		3019	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
6084	serge	3020	uint64_t tiling, uint32_t latency)
5354	serge	3021	{
6084	serge	3022	uint32_t ret;
		3023	uint32_t plane_bytes_per_line, plane_blocks_per_line;
		3024	uint32_t wm_intermediate_val;
5354	serge	3025
		3026	if (latency == 0)
		3027	return UINT_MAX;
		3028
		3029	plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
6084	serge	3030
		3031	if (tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3032	tiling == I915_FORMAT_MOD_Yf_TILED) {
		3033	plane_bytes_per_line *= 4;
		3034	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
		3035	plane_blocks_per_line /= 4;
		3036	} else {
		3037	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
		3038	}
		3039
5354	serge	3040	wm_intermediate_val = latency * pixel_rate;
		3041	ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
6084	serge	3042	plane_blocks_per_line;
5354	serge	3043
		3044	return ret;
		3045	}
		3046
		3047	static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
		3048	const struct intel_crtc *intel_crtc)
		3049	{
		3050	struct drm_device *dev = intel_crtc->base.dev;
		3051	struct drm_i915_private *dev_priv = dev->dev_private;
		3052	const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3053	enum pipe pipe = intel_crtc->pipe;
		3054
		3055	if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
		3056	sizeof(new_ddb->plane[pipe])))
		3057	return true;
		3058
6084	serge	3059	if (memcmp(&new_ddb->plane[pipe][PLANE_CURSOR], &cur_ddb->plane[pipe][PLANE_CURSOR],
		3060	sizeof(new_ddb->plane[pipe][PLANE_CURSOR])))
5354	serge	3061	return true;
		3062
		3063	return false;
		3064	}
		3065
		3066	static void skl_compute_wm_global_parameters(struct drm_device *dev,
		3067	struct intel_wm_config *config)
		3068	{
		3069	struct drm_crtc *crtc;
		3070	struct drm_plane *plane;
		3071
		3072	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
6084	serge	3073	config->num_pipes_active += to_intel_crtc(crtc)->active;
5354	serge	3074
		3075	/* FIXME: I don't think we need those two global parameters on SKL */
		3076	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		3077	struct intel_plane *intel_plane = to_intel_plane(plane);
		3078
		3079	config->sprites_enabled \|= intel_plane->wm.enabled;
		3080	config->sprites_scaled \|= intel_plane->wm.scaled;
		3081	}
		3082	}
		3083
		3084	static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
		3085	struct skl_pipe_wm_parameters *p)
		3086	{
		3087	struct drm_device *dev = crtc->dev;
		3088	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3089	enum pipe pipe = intel_crtc->pipe;
		3090	struct drm_plane *plane;
6084	serge	3091	struct drm_framebuffer *fb;
5354	serge	3092	int i = 1; /* Index for sprite planes start */
		3093
6084	serge	3094	p->active = intel_crtc->active;
5354	serge	3095	if (p->active) {
6084	serge	3096	p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
		3097	p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config);
5354	serge	3098
6084	serge	3099	fb = crtc->primary->state->fb;
		3100	/* For planar: Bpp is for uv plane, y_Bpp is for y plane */
		3101	if (fb) {
		3102	p->plane[0].enabled = true;
		3103	p->plane[0].bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
		3104	drm_format_plane_cpp(fb->pixel_format, 1) :
		3105	drm_format_plane_cpp(fb->pixel_format, 0);
		3106	p->plane[0].y_bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
		3107	drm_format_plane_cpp(fb->pixel_format, 0) : 0;
		3108	p->plane[0].tiling = fb->modifier[0];
		3109	} else {
		3110	p->plane[0].enabled = false;
		3111	p->plane[0].bytes_per_pixel = 0;
		3112	p->plane[0].y_bytes_per_pixel = 0;
		3113	p->plane[0].tiling = DRM_FORMAT_MOD_NONE;
		3114	}
		3115	p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w;
		3116	p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h;
		3117	p->plane[0].rotation = crtc->primary->state->rotation;
5354	serge	3118
6084	serge	3119	fb = crtc->cursor->state->fb;
		3120	p->plane[PLANE_CURSOR].y_bytes_per_pixel = 0;
		3121	if (fb) {
		3122	p->plane[PLANE_CURSOR].enabled = true;
		3123	p->plane[PLANE_CURSOR].bytes_per_pixel = fb->bits_per_pixel / 8;
		3124	p->plane[PLANE_CURSOR].horiz_pixels = crtc->cursor->state->crtc_w;
		3125	p->plane[PLANE_CURSOR].vert_pixels = crtc->cursor->state->crtc_h;
		3126	} else {
		3127	p->plane[PLANE_CURSOR].enabled = false;
		3128	p->plane[PLANE_CURSOR].bytes_per_pixel = 0;
		3129	p->plane[PLANE_CURSOR].horiz_pixels = 64;
		3130	p->plane[PLANE_CURSOR].vert_pixels = 64;
		3131	}
5354	serge	3132	}
		3133
		3134	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		3135	struct intel_plane *intel_plane = to_intel_plane(plane);
		3136
6084	serge	3137	if (intel_plane->pipe == pipe &&
		3138	plane->type == DRM_PLANE_TYPE_OVERLAY)
5354	serge	3139	p->plane[i++] = intel_plane->wm;
		3140	}
		3141	}
		3142
6084	serge	3143	static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
		3144	struct skl_pipe_wm_parameters *p,
5354	serge	3145	struct intel_plane_wm_parameters *p_params,
		3146	uint16_t ddb_allocation,
6084	serge	3147	int level,
5354	serge	3148	uint16_t out_blocks, / out */
		3149	uint8_t out_lines / out */)
		3150	{
6084	serge	3151	uint32_t latency = dev_priv->wm.skl_latency[level];
		3152	uint32_t method1, method2;
		3153	uint32_t plane_bytes_per_line, plane_blocks_per_line;
		3154	uint32_t res_blocks, res_lines;
		3155	uint32_t selected_result;
		3156	uint8_t bytes_per_pixel;
5354	serge	3157
6084	serge	3158	if (latency == 0 \|\| !p->active \|\| !p_params->enabled)
5354	serge	3159	return false;
		3160
6084	serge	3161	bytes_per_pixel = p_params->y_bytes_per_pixel ?
		3162	p_params->y_bytes_per_pixel :
		3163	p_params->bytes_per_pixel;
5354	serge	3164	method1 = skl_wm_method1(p->pixel_rate,
6084	serge	3165	bytes_per_pixel,
		3166	latency);
5354	serge	3167	method2 = skl_wm_method2(p->pixel_rate,
		3168	p->pipe_htotal,
		3169	p_params->horiz_pixels,
6084	serge	3170	bytes_per_pixel,
		3171	p_params->tiling,
		3172	latency);
5354	serge	3173
6084	serge	3174	plane_bytes_per_line = p_params->horiz_pixels * bytes_per_pixel;
		3175	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
5354	serge	3176
6084	serge	3177	if (p_params->tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3178	p_params->tiling == I915_FORMAT_MOD_Yf_TILED) {
		3179	uint32_t min_scanlines = 4;
		3180	uint32_t y_tile_minimum;
		3181	if (intel_rotation_90_or_270(p_params->rotation)) {
		3182	switch (p_params->bytes_per_pixel) {
		3183	case 1:
		3184	min_scanlines = 16;
		3185	break;
		3186	case 2:
		3187	min_scanlines = 8;
		3188	break;
		3189	case 8:
		3190	WARN(1, "Unsupported pixel depth for rotation");
		3191	}
		3192	}
		3193	y_tile_minimum = plane_blocks_per_line * min_scanlines;
		3194	selected_result = max(method2, y_tile_minimum);
		3195	} else {
		3196	if ((ddb_allocation / plane_blocks_per_line) >= 1)
		3197	selected_result = min(method1, method2);
		3198	else
		3199	selected_result = method1;
		3200	}
5354	serge	3201
6084	serge	3202	res_blocks = selected_result + 1;
		3203	res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
5354	serge	3204
6084	serge	3205	if (level >= 1 && level <= 7) {
		3206	if (p_params->tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3207	p_params->tiling == I915_FORMAT_MOD_Yf_TILED)
		3208	res_lines += 4;
		3209	else
		3210	res_blocks++;
		3211	}
		3212
		3213	if (res_blocks >= ddb_allocation \|\| res_lines > 31)
5354	serge	3214	return false;
		3215
		3216	*out_blocks = res_blocks;
		3217	*out_lines = res_lines;
		3218
		3219	return true;
		3220	}
		3221
		3222	static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
		3223	struct skl_ddb_allocation *ddb,
		3224	struct skl_pipe_wm_parameters *p,
		3225	enum pipe pipe,
		3226	int level,
		3227	int num_planes,
		3228	struct skl_wm_level *result)
		3229	{
		3230	uint16_t ddb_blocks;
		3231	int i;
		3232
		3233	for (i = 0; i < num_planes; i++) {
		3234	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
		3235
6084	serge	3236	result->plane_en[i] = skl_compute_plane_wm(dev_priv,
		3237	p, &p->plane[i],
5354	serge	3238	ddb_blocks,
6084	serge	3239	level,
5354	serge	3240	&result->plane_res_b[i],
		3241	&result->plane_res_l[i]);
		3242	}
		3243
6084	serge	3244	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][PLANE_CURSOR]);
		3245	result->plane_en[PLANE_CURSOR] = skl_compute_plane_wm(dev_priv, p,
		3246	&p->plane[PLANE_CURSOR],
		3247	ddb_blocks, level,
		3248	&result->plane_res_b[PLANE_CURSOR],
		3249	&result->plane_res_l[PLANE_CURSOR]);
5354	serge	3250	}
		3251
		3252	static uint32_t
		3253	skl_compute_linetime_wm(struct drm_crtc crtc, struct skl_pipe_wm_parameters p)
		3254	{
6084	serge	3255	if (!to_intel_crtc(crtc)->active)
5354	serge	3256	return 0;
		3257
6084	serge	3258	if (WARN_ON(p->pixel_rate == 0))
		3259	return 0;
		3260
5354	serge	3261	return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
		3262	}
		3263
		3264	static void skl_compute_transition_wm(struct drm_crtc *crtc,
		3265	struct skl_pipe_wm_parameters *params,
		3266	struct skl_wm_level trans_wm / out */)
		3267	{
		3268	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3269	int i;
		3270
		3271	if (!params->active)
		3272	return;
		3273
		3274	/* Until we know more, just disable transition WMs */
		3275	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3276	trans_wm->plane_en[i] = false;
6084	serge	3277	trans_wm->plane_en[PLANE_CURSOR] = false;
5354	serge	3278	}
		3279
		3280	static void skl_compute_pipe_wm(struct drm_crtc *crtc,
		3281	struct skl_ddb_allocation *ddb,
		3282	struct skl_pipe_wm_parameters *params,
		3283	struct skl_pipe_wm *pipe_wm)
		3284	{
		3285	struct drm_device *dev = crtc->dev;
		3286	const struct drm_i915_private *dev_priv = dev->dev_private;
		3287	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3288	int level, max_level = ilk_wm_max_level(dev);
		3289
		3290	for (level = 0; level <= max_level; level++) {
		3291	skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
		3292	level, intel_num_planes(intel_crtc),
		3293	&pipe_wm->wm[level]);
		3294	}
		3295	pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
		3296
		3297	skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
		3298	}
		3299
		3300	static void skl_compute_wm_results(struct drm_device *dev,
		3301	struct skl_pipe_wm_parameters *p,
		3302	struct skl_pipe_wm *p_wm,
		3303	struct skl_wm_values *r,
		3304	struct intel_crtc *intel_crtc)
		3305	{
		3306	int level, max_level = ilk_wm_max_level(dev);
		3307	enum pipe pipe = intel_crtc->pipe;
		3308	uint32_t temp;
		3309	int i;
		3310
		3311	for (level = 0; level <= max_level; level++) {
		3312	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3313	temp = 0;
		3314
		3315	temp \|= p_wm->wm[level].plane_res_l[i] <<
		3316	PLANE_WM_LINES_SHIFT;
		3317	temp \|= p_wm->wm[level].plane_res_b[i];
		3318	if (p_wm->wm[level].plane_en[i])
		3319	temp \|= PLANE_WM_EN;
		3320
		3321	r->plane[pipe][i][level] = temp;
		3322	}
		3323
		3324	temp = 0;
		3325
6084	serge	3326	temp \|= p_wm->wm[level].plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
		3327	temp \|= p_wm->wm[level].plane_res_b[PLANE_CURSOR];
5354	serge	3328
6084	serge	3329	if (p_wm->wm[level].plane_en[PLANE_CURSOR])
5354	serge	3330	temp \|= PLANE_WM_EN;
		3331
6084	serge	3332	r->plane[pipe][PLANE_CURSOR][level] = temp;
5354	serge	3333
		3334	}
		3335
		3336	/* transition WMs */
		3337	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3338	temp = 0;
		3339	temp \|= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
		3340	temp \|= p_wm->trans_wm.plane_res_b[i];
		3341	if (p_wm->trans_wm.plane_en[i])
		3342	temp \|= PLANE_WM_EN;
		3343
		3344	r->plane_trans[pipe][i] = temp;
		3345	}
		3346
		3347	temp = 0;
6084	serge	3348	temp \|= p_wm->trans_wm.plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
		3349	temp \|= p_wm->trans_wm.plane_res_b[PLANE_CURSOR];
		3350	if (p_wm->trans_wm.plane_en[PLANE_CURSOR])
5354	serge	3351	temp \|= PLANE_WM_EN;
		3352
6084	serge	3353	r->plane_trans[pipe][PLANE_CURSOR] = temp;
5354	serge	3354
		3355	r->wm_linetime[pipe] = p_wm->linetime;
		3356	}
		3357
		3358	static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
		3359	const struct skl_ddb_entry *entry)
		3360	{
		3361	if (entry->end)
		3362	I915_WRITE(reg, (entry->end - 1) << 16 \| entry->start);
		3363	else
		3364	I915_WRITE(reg, 0);
		3365	}
		3366
		3367	static void skl_write_wm_values(struct drm_i915_private *dev_priv,
		3368	const struct skl_wm_values *new)
		3369	{
		3370	struct drm_device *dev = dev_priv->dev;
		3371	struct intel_crtc *crtc;
		3372
		3373	list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
		3374	int i, level, max_level = ilk_wm_max_level(dev);
		3375	enum pipe pipe = crtc->pipe;
		3376
		3377	if (!new->dirty[pipe])
		3378	continue;
		3379
		3380	I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
		3381
		3382	for (level = 0; level <= max_level; level++) {
		3383	for (i = 0; i < intel_num_planes(crtc); i++)
		3384	I915_WRITE(PLANE_WM(pipe, i, level),
		3385	new->plane[pipe][i][level]);
		3386	I915_WRITE(CUR_WM(pipe, level),
6084	serge	3387	new->plane[pipe][PLANE_CURSOR][level]);
5354	serge	3388	}
		3389	for (i = 0; i < intel_num_planes(crtc); i++)
		3390	I915_WRITE(PLANE_WM_TRANS(pipe, i),
		3391	new->plane_trans[pipe][i]);
6084	serge	3392	I915_WRITE(CUR_WM_TRANS(pipe),
		3393	new->plane_trans[pipe][PLANE_CURSOR]);
5354	serge	3394
6084	serge	3395	for (i = 0; i < intel_num_planes(crtc); i++) {
5354	serge	3396	skl_ddb_entry_write(dev_priv,
		3397	PLANE_BUF_CFG(pipe, i),
		3398	&new->ddb.plane[pipe][i]);
6084	serge	3399	skl_ddb_entry_write(dev_priv,
		3400	PLANE_NV12_BUF_CFG(pipe, i),
		3401	&new->ddb.y_plane[pipe][i]);
		3402	}
5354	serge	3403
		3404	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
6084	serge	3405	&new->ddb.plane[pipe][PLANE_CURSOR]);
5354	serge	3406	}
		3407	}
		3408
		3409	/*
		3410	* When setting up a new DDB allocation arrangement, we need to correctly
		3411	* sequence the times at which the new allocations for the pipes are taken into
		3412	* account or we'll have pipes fetching from space previously allocated to
		3413	* another pipe.
		3414	*
		3415	* Roughly the sequence looks like:
		3416	* 1. re-allocate the pipe(s) with the allocation being reduced and not
		3417	* overlapping with a previous light-up pipe (another way to put it is:
		3418	* pipes with their new allocation strickly included into their old ones).
		3419	* 2. re-allocate the other pipes that get their allocation reduced
		3420	* 3. allocate the pipes having their allocation increased
		3421	*
		3422	* Steps 1. and 2. are here to take care of the following case:
		3423	* - Initially DDB looks like this:
		3424	* \| B \| C \|
		3425	* - enable pipe A.
		3426	* - pipe B has a reduced DDB allocation that overlaps with the old pipe C
		3427	* allocation
		3428	* \| A \| B \| C \|
		3429	*
		3430	* We need to sequence the re-allocation: C, B, A (and not B, C, A).
		3431	*/
		3432
		3433	static void
		3434	skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
		3435	{
		3436	int plane;
		3437
		3438	DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
		3439
6084	serge	3440	for_each_plane(dev_priv, pipe, plane) {
5354	serge	3441	I915_WRITE(PLANE_SURF(pipe, plane),
		3442	I915_READ(PLANE_SURF(pipe, plane)));
		3443	}
		3444	I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
		3445	}
		3446
		3447	static bool
		3448	skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
		3449	const struct skl_ddb_allocation *new,
		3450	enum pipe pipe)
		3451	{
		3452	uint16_t old_size, new_size;
		3453
		3454	old_size = skl_ddb_entry_size(&old->pipe[pipe]);
		3455	new_size = skl_ddb_entry_size(&new->pipe[pipe]);
		3456
		3457	return old_size != new_size &&
		3458	new->pipe[pipe].start >= old->pipe[pipe].start &&
		3459	new->pipe[pipe].end <= old->pipe[pipe].end;
		3460	}
		3461
		3462	static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
		3463	struct skl_wm_values *new_values)
		3464	{
		3465	struct drm_device *dev = dev_priv->dev;
		3466	struct skl_ddb_allocation cur_ddb, new_ddb;
6084	serge	3467	bool reallocated[I915_MAX_PIPES] = {};
5354	serge	3468	struct intel_crtc *crtc;
		3469	enum pipe pipe;
		3470
		3471	new_ddb = &new_values->ddb;
		3472	cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3473
		3474	/*
		3475	* First pass: flush the pipes with the new allocation contained into
		3476	* the old space.
		3477	*
		3478	* We'll wait for the vblank on those pipes to ensure we can safely
		3479	* re-allocate the freed space without this pipe fetching from it.
		3480	*/
		3481	for_each_intel_crtc(dev, crtc) {
		3482	if (!crtc->active)
		3483	continue;
		3484
		3485	pipe = crtc->pipe;
		3486
		3487	if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
		3488	continue;
		3489
		3490	skl_wm_flush_pipe(dev_priv, pipe, 1);
		3491	intel_wait_for_vblank(dev, pipe);
		3492
		3493	reallocated[pipe] = true;
		3494	}
		3495
		3496
		3497	/*
		3498	* Second pass: flush the pipes that are having their allocation
		3499	* reduced, but overlapping with a previous allocation.
		3500	*
		3501	* Here as well we need to wait for the vblank to make sure the freed
		3502	* space is not used anymore.
		3503	*/
		3504	for_each_intel_crtc(dev, crtc) {
		3505	if (!crtc->active)
		3506	continue;
		3507
		3508	pipe = crtc->pipe;
		3509
		3510	if (reallocated[pipe])
		3511	continue;
		3512
		3513	if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
		3514	skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
		3515	skl_wm_flush_pipe(dev_priv, pipe, 2);
		3516	intel_wait_for_vblank(dev, pipe);
6084	serge	3517	reallocated[pipe] = true;
5354	serge	3518	}
		3519	}
		3520
		3521	/*
		3522	* Third pass: flush the pipes that got more space allocated.
		3523	*
		3524	* We don't need to actively wait for the update here, next vblank
		3525	* will just get more DDB space with the correct WM values.
		3526	*/
		3527	for_each_intel_crtc(dev, crtc) {
		3528	if (!crtc->active)
		3529	continue;
		3530
		3531	pipe = crtc->pipe;
		3532
		3533	/*
		3534	* At this point, only the pipes more space than before are
		3535	* left to re-allocate.
		3536	*/
		3537	if (reallocated[pipe])
		3538	continue;
		3539
		3540	skl_wm_flush_pipe(dev_priv, pipe, 3);
		3541	}
		3542	}
		3543
		3544	static bool skl_update_pipe_wm(struct drm_crtc *crtc,
		3545	struct skl_pipe_wm_parameters *params,
		3546	struct intel_wm_config *config,
		3547	struct skl_ddb_allocation ddb, / out */
		3548	struct skl_pipe_wm pipe_wm / out */)
		3549	{
		3550	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3551
		3552	skl_compute_wm_pipe_parameters(crtc, params);
		3553	skl_allocate_pipe_ddb(crtc, config, params, ddb);
		3554	skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
		3555
		3556	if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
		3557	return false;
		3558
		3559	intel_crtc->wm.skl_active = *pipe_wm;
6084	serge	3560
5354	serge	3561	return true;
		3562	}
		3563
		3564	static void skl_update_other_pipe_wm(struct drm_device *dev,
		3565	struct drm_crtc *crtc,
		3566	struct intel_wm_config *config,
		3567	struct skl_wm_values *r)
		3568	{
		3569	struct intel_crtc *intel_crtc;
		3570	struct intel_crtc *this_crtc = to_intel_crtc(crtc);
		3571
		3572	/*
		3573	* If the WM update hasn't changed the allocation for this_crtc (the
		3574	* crtc we are currently computing the new WM values for), other
		3575	* enabled crtcs will keep the same allocation and we don't need to
		3576	* recompute anything for them.
		3577	*/
		3578	if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
		3579	return;
		3580
		3581	/*
		3582	* Otherwise, because of this_crtc being freshly enabled/disabled, the
		3583	* other active pipes need new DDB allocation and WM values.
		3584	*/
		3585	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
		3586	base.head) {
		3587	struct skl_pipe_wm_parameters params = {};
		3588	struct skl_pipe_wm pipe_wm = {};
		3589	bool wm_changed;
		3590
		3591	if (this_crtc->pipe == intel_crtc->pipe)
		3592	continue;
		3593
		3594	if (!intel_crtc->active)
		3595	continue;
		3596
		3597	wm_changed = skl_update_pipe_wm(&intel_crtc->base,
		3598	¶ms, config,
		3599	&r->ddb, &pipe_wm);
		3600
		3601	/*
		3602	* If we end up re-computing the other pipe WM values, it's
		3603	* because it was really needed, so we expect the WM values to
		3604	* be different.
6084	serge	3605	*/
5354	serge	3606	WARN_ON(!wm_changed);
		3607
		3608	skl_compute_wm_results(dev, ¶ms, &pipe_wm, r, intel_crtc);
		3609	r->dirty[intel_crtc->pipe] = true;
		3610	}
		3611	}
		3612
6084	serge	3613	static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe)
		3614	{
		3615	watermarks->wm_linetime[pipe] = 0;
		3616	memset(watermarks->plane[pipe], 0,
		3617	sizeof(uint32_t) * 8 * I915_MAX_PLANES);
		3618	memset(watermarks->plane_trans[pipe],
		3619	0, sizeof(uint32_t) * I915_MAX_PLANES);
		3620	watermarks->plane_trans[pipe][PLANE_CURSOR] = 0;
		3621
		3622	/* Clear ddb entries for pipe */
		3623	memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry));
		3624	memset(&watermarks->ddb.plane[pipe], 0,
		3625	sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
		3626	memset(&watermarks->ddb.y_plane[pipe], 0,
		3627	sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
		3628	memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0,
		3629	sizeof(struct skl_ddb_entry));
		3630
		3631	}
		3632
5354	serge	3633	static void skl_update_wm(struct drm_crtc *crtc)
		3634	{
		3635	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3636	struct drm_device *dev = crtc->dev;
		3637	struct drm_i915_private *dev_priv = dev->dev_private;
		3638	struct skl_pipe_wm_parameters params = {};
		3639	struct skl_wm_values *results = &dev_priv->wm.skl_results;
		3640	struct skl_pipe_wm pipe_wm = {};
		3641	struct intel_wm_config config = {};
		3642
		3643
6084	serge	3644	/* Clear all dirty flags */
		3645	memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES);
		3646
		3647	skl_clear_wm(results, intel_crtc->pipe);
		3648
5354	serge	3649	skl_compute_wm_global_parameters(dev, &config);
		3650
		3651	if (!skl_update_pipe_wm(crtc, ¶ms, &config,
		3652	&results->ddb, &pipe_wm))
		3653	return;
		3654
		3655	skl_compute_wm_results(dev, ¶ms, &pipe_wm, results, intel_crtc);
		3656	results->dirty[intel_crtc->pipe] = true;
		3657
		3658	skl_update_other_pipe_wm(dev, crtc, &config, results);
		3659	skl_write_wm_values(dev_priv, results);
		3660	skl_flush_wm_values(dev_priv, results);
		3661
		3662	/* store the new configuration */
		3663	dev_priv->wm.skl_hw = *results;
		3664	}
		3665
		3666	static void
		3667	skl_update_sprite_wm(struct drm_plane plane, struct drm_crtc crtc,
		3668	uint32_t sprite_width, uint32_t sprite_height,
		3669	int pixel_size, bool enabled, bool scaled)
		3670	{
		3671	struct intel_plane *intel_plane = to_intel_plane(plane);
6084	serge	3672	struct drm_framebuffer *fb = plane->state->fb;
5354	serge	3673
		3674	intel_plane->wm.enabled = enabled;
		3675	intel_plane->wm.scaled = scaled;
		3676	intel_plane->wm.horiz_pixels = sprite_width;
		3677	intel_plane->wm.vert_pixels = sprite_height;
6084	serge	3678	intel_plane->wm.tiling = DRM_FORMAT_MOD_NONE;
5354	serge	3679
6084	serge	3680	/* For planar: Bpp is for UV plane, y_Bpp is for Y plane */
		3681	intel_plane->wm.bytes_per_pixel =
		3682	(fb && fb->pixel_format == DRM_FORMAT_NV12) ?
		3683	drm_format_plane_cpp(plane->state->fb->pixel_format, 1) : pixel_size;
		3684	intel_plane->wm.y_bytes_per_pixel =
		3685	(fb && fb->pixel_format == DRM_FORMAT_NV12) ?
		3686	drm_format_plane_cpp(plane->state->fb->pixel_format, 0) : 0;
		3687
		3688	/*
		3689	* Framebuffer can be NULL on plane disable, but it does not
		3690	* matter for watermarks if we assume no tiling in that case.
		3691	*/
		3692	if (fb)
		3693	intel_plane->wm.tiling = fb->modifier[0];
		3694	intel_plane->wm.rotation = plane->state->rotation;
		3695
5354	serge	3696	skl_update_wm(crtc);
		3697	}
		3698
4560	Serge	3699	static void ilk_update_wm(struct drm_crtc *crtc)
		3700	{
		3701	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6084	serge	3702	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
4560	Serge	3703	struct drm_device *dev = crtc->dev;
		3704	struct drm_i915_private *dev_priv = dev->dev_private;
		3705	struct ilk_wm_maximums max;
		3706	struct ilk_wm_values results = {};
4104	Serge	3707	enum intel_ddb_partitioning partitioning;
4560	Serge	3708	struct intel_pipe_wm pipe_wm = {};
		3709	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
		3710	struct intel_wm_config config = {};
4104	Serge	3711
6084	serge	3712	WARN_ON(cstate->base.active != intel_crtc->active);
4104	Serge	3713
6084	serge	3714	intel_compute_pipe_wm(cstate, &pipe_wm);
4560	Serge	3715
		3716	if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
		3717	return;
		3718
		3719	intel_crtc->wm.active = pipe_wm;
		3720
5060	serge	3721	ilk_compute_wm_config(dev, &config);
		3722
4560	Serge	3723	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
		3724	ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
		3725
		3726	/* 5/6 split only in single pipe config on IVB+ */
		3727	if (INTEL_INFO(dev)->gen >= 7 &&
		3728	config.num_pipes_active == 1 && config.sprites_enabled) {
		3729	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
		3730	ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
		3731
		3732	best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
4104	Serge	3733	} else {
4560	Serge	3734	best_lp_wm = &lp_wm_1_2;
4104	Serge	3735	}
		3736
4560	Serge	3737	partitioning = (best_lp_wm == &lp_wm_1_2) ?
4104	Serge	3738	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
		3739
4560	Serge	3740	ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
		3741
		3742	ilk_write_wm_values(dev_priv, &results);
4104	Serge	3743	}
		3744
5060	serge	3745	static void
		3746	ilk_update_sprite_wm(struct drm_plane *plane,
5354	serge	3747	struct drm_crtc *crtc,
5060	serge	3748	uint32_t sprite_width, uint32_t sprite_height,
		3749	int pixel_size, bool enabled, bool scaled)
4104	Serge	3750	{
4560	Serge	3751	struct drm_device *dev = plane->dev;
5354	serge	3752	struct intel_plane *intel_plane = to_intel_plane(plane);
4104	Serge	3753
4560	Serge	3754	/*
		3755	* IVB workaround: must disable low power watermarks for at least
		3756	* one frame before enabling scaling. LP watermarks can be re-enabled
		3757	* when scaling is disabled.
		3758	*
		3759	* WaCxSRDisabledForSpriteScaling:ivb
		3760	*/
		3761	if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
		3762	intel_wait_for_vblank(dev, intel_plane->pipe);
		3763
		3764	ilk_update_wm(crtc);
4104	Serge	3765	}
		3766
5354	serge	3767	static void skl_pipe_wm_active_state(uint32_t val,
		3768	struct skl_pipe_wm *active,
		3769	bool is_transwm,
		3770	bool is_cursor,
		3771	int i,
		3772	int level)
		3773	{
		3774	bool is_enabled = (val & PLANE_WM_EN) != 0;
		3775
		3776	if (!is_transwm) {
		3777	if (!is_cursor) {
		3778	active->wm[level].plane_en[i] = is_enabled;
		3779	active->wm[level].plane_res_b[i] =
		3780	val & PLANE_WM_BLOCKS_MASK;
		3781	active->wm[level].plane_res_l[i] =
		3782	(val >> PLANE_WM_LINES_SHIFT) &
		3783	PLANE_WM_LINES_MASK;
		3784	} else {
6084	serge	3785	active->wm[level].plane_en[PLANE_CURSOR] = is_enabled;
		3786	active->wm[level].plane_res_b[PLANE_CURSOR] =
5354	serge	3787	val & PLANE_WM_BLOCKS_MASK;
6084	serge	3788	active->wm[level].plane_res_l[PLANE_CURSOR] =
5354	serge	3789	(val >> PLANE_WM_LINES_SHIFT) &
		3790	PLANE_WM_LINES_MASK;
		3791	}
		3792	} else {
		3793	if (!is_cursor) {
		3794	active->trans_wm.plane_en[i] = is_enabled;
		3795	active->trans_wm.plane_res_b[i] =
		3796	val & PLANE_WM_BLOCKS_MASK;
		3797	active->trans_wm.plane_res_l[i] =
		3798	(val >> PLANE_WM_LINES_SHIFT) &
		3799	PLANE_WM_LINES_MASK;
		3800	} else {
6084	serge	3801	active->trans_wm.plane_en[PLANE_CURSOR] = is_enabled;
		3802	active->trans_wm.plane_res_b[PLANE_CURSOR] =
5354	serge	3803	val & PLANE_WM_BLOCKS_MASK;
6084	serge	3804	active->trans_wm.plane_res_l[PLANE_CURSOR] =
5354	serge	3805	(val >> PLANE_WM_LINES_SHIFT) &
		3806	PLANE_WM_LINES_MASK;
		3807	}
		3808	}
		3809	}
		3810
		3811	static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
		3812	{
		3813	struct drm_device *dev = crtc->dev;
		3814	struct drm_i915_private *dev_priv = dev->dev_private;
		3815	struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
		3816	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3817	struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
		3818	enum pipe pipe = intel_crtc->pipe;
		3819	int level, i, max_level;
		3820	uint32_t temp;
		3821
		3822	max_level = ilk_wm_max_level(dev);
		3823
		3824	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
		3825
		3826	for (level = 0; level <= max_level; level++) {
		3827	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3828	hw->plane[pipe][i][level] =
		3829	I915_READ(PLANE_WM(pipe, i, level));
6084	serge	3830	hw->plane[pipe][PLANE_CURSOR][level] = I915_READ(CUR_WM(pipe, level));
5354	serge	3831	}
		3832
		3833	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3834	hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
6084	serge	3835	hw->plane_trans[pipe][PLANE_CURSOR] = I915_READ(CUR_WM_TRANS(pipe));
5354	serge	3836
6084	serge	3837	if (!intel_crtc->active)
5354	serge	3838	return;
		3839
		3840	hw->dirty[pipe] = true;
		3841
		3842	active->linetime = hw->wm_linetime[pipe];
		3843
		3844	for (level = 0; level <= max_level; level++) {
		3845	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3846	temp = hw->plane[pipe][i][level];
		3847	skl_pipe_wm_active_state(temp, active, false,
		3848	false, i, level);
		3849	}
6084	serge	3850	temp = hw->plane[pipe][PLANE_CURSOR][level];
5354	serge	3851	skl_pipe_wm_active_state(temp, active, false, true, i, level);
		3852	}
		3853
		3854	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3855	temp = hw->plane_trans[pipe][i];
		3856	skl_pipe_wm_active_state(temp, active, true, false, i, 0);
		3857	}
		3858
6084	serge	3859	temp = hw->plane_trans[pipe][PLANE_CURSOR];
5354	serge	3860	skl_pipe_wm_active_state(temp, active, true, true, i, 0);
		3861	}
		3862
		3863	void skl_wm_get_hw_state(struct drm_device *dev)
		3864	{
		3865	struct drm_i915_private *dev_priv = dev->dev_private;
		3866	struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
		3867	struct drm_crtc *crtc;
		3868
		3869	skl_ddb_get_hw_state(dev_priv, ddb);
		3870	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
		3871	skl_pipe_wm_get_hw_state(crtc);
		3872	}
		3873
4560	Serge	3874	static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3031	serge	3875	{
4560	Serge	3876	struct drm_device *dev = crtc->dev;
		3877	struct drm_i915_private *dev_priv = dev->dev_private;
		3878	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		3879	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3880	struct intel_pipe_wm *active = &intel_crtc->wm.active;
		3881	enum pipe pipe = intel_crtc->pipe;
		3882	static const unsigned int wm0_pipe_reg[] = {
		3883	[PIPE_A] = WM0_PIPEA_ILK,
		3884	[PIPE_B] = WM0_PIPEB_ILK,
		3885	[PIPE_C] = WM0_PIPEC_IVB,
		3886	};
3031	serge	3887
4560	Serge	3888	hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
		3889	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		3890	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3031	serge	3891
6660	serge	3892	memset(active, 0, sizeof(*active));
		3893
6084	serge	3894	active->pipe_enabled = intel_crtc->active;
5060	serge	3895
		3896	if (active->pipe_enabled) {
4560	Serge	3897	u32 tmp = hw->wm_pipe[pipe];
3031	serge	3898
4560	Serge	3899	/*
		3900	* For active pipes LP0 watermark is marked as
		3901	* enabled, and LP1+ watermaks as disabled since
		3902	* we can't really reverse compute them in case
		3903	* multiple pipes are active.
		3904	*/
		3905	active->wm[0].enable = true;
		3906	active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
		3907	active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
		3908	active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
		3909	active->linetime = hw->wm_linetime[pipe];
		3910	} else {
		3911	int level, max_level = ilk_wm_max_level(dev);
3031	serge	3912
4560	Serge	3913	/*
		3914	* For inactive pipes, all watermark levels
		3915	* should be marked as enabled but zeroed,
		3916	* which is what we'd compute them to.
		3917	*/
		3918	for (level = 0; level <= max_level; level++)
		3919	active->wm[level].enable = true;
3031	serge	3920	}
		3921	}
		3922
6084	serge	3923	#define _FW_WM(value, plane) \
		3924	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
		3925	#define _FW_WM_VLV(value, plane) \
		3926	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
		3927
		3928	static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
		3929	struct vlv_wm_values *wm)
		3930	{
		3931	enum pipe pipe;
		3932	uint32_t tmp;
		3933
		3934	for_each_pipe(dev_priv, pipe) {
		3935	tmp = I915_READ(VLV_DDL(pipe));
		3936
		3937	wm->ddl[pipe].primary =
		3938	(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3939	wm->ddl[pipe].cursor =
		3940	(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3941	wm->ddl[pipe].sprite[0] =
		3942	(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3943	wm->ddl[pipe].sprite[1] =
		3944	(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3945	}
		3946
		3947	tmp = I915_READ(DSPFW1);
		3948	wm->sr.plane = _FW_WM(tmp, SR);
		3949	wm->pipe[PIPE_B].cursor = _FW_WM(tmp, CURSORB);
		3950	wm->pipe[PIPE_B].primary = _FW_WM_VLV(tmp, PLANEB);
		3951	wm->pipe[PIPE_A].primary = _FW_WM_VLV(tmp, PLANEA);
		3952
		3953	tmp = I915_READ(DSPFW2);
		3954	wm->pipe[PIPE_A].sprite[1] = _FW_WM_VLV(tmp, SPRITEB);
		3955	wm->pipe[PIPE_A].cursor = _FW_WM(tmp, CURSORA);
		3956	wm->pipe[PIPE_A].sprite[0] = _FW_WM_VLV(tmp, SPRITEA);
		3957
		3958	tmp = I915_READ(DSPFW3);
		3959	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
		3960
		3961	if (IS_CHERRYVIEW(dev_priv)) {
		3962	tmp = I915_READ(DSPFW7_CHV);
		3963	wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
		3964	wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
		3965
		3966	tmp = I915_READ(DSPFW8_CHV);
		3967	wm->pipe[PIPE_C].sprite[1] = _FW_WM_VLV(tmp, SPRITEF);
		3968	wm->pipe[PIPE_C].sprite[0] = _FW_WM_VLV(tmp, SPRITEE);
		3969
		3970	tmp = I915_READ(DSPFW9_CHV);
		3971	wm->pipe[PIPE_C].primary = _FW_WM_VLV(tmp, PLANEC);
		3972	wm->pipe[PIPE_C].cursor = _FW_WM(tmp, CURSORC);
		3973
		3974	tmp = I915_READ(DSPHOWM);
		3975	wm->sr.plane \|= _FW_WM(tmp, SR_HI) << 9;
		3976	wm->pipe[PIPE_C].sprite[1] \|= _FW_WM(tmp, SPRITEF_HI) << 8;
		3977	wm->pipe[PIPE_C].sprite[0] \|= _FW_WM(tmp, SPRITEE_HI) << 8;
		3978	wm->pipe[PIPE_C].primary \|= _FW_WM(tmp, PLANEC_HI) << 8;
		3979	wm->pipe[PIPE_B].sprite[1] \|= _FW_WM(tmp, SPRITED_HI) << 8;
		3980	wm->pipe[PIPE_B].sprite[0] \|= _FW_WM(tmp, SPRITEC_HI) << 8;
		3981	wm->pipe[PIPE_B].primary \|= _FW_WM(tmp, PLANEB_HI) << 8;
		3982	wm->pipe[PIPE_A].sprite[1] \|= _FW_WM(tmp, SPRITEB_HI) << 8;
		3983	wm->pipe[PIPE_A].sprite[0] \|= _FW_WM(tmp, SPRITEA_HI) << 8;
		3984	wm->pipe[PIPE_A].primary \|= _FW_WM(tmp, PLANEA_HI) << 8;
		3985	} else {
		3986	tmp = I915_READ(DSPFW7);
		3987	wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
		3988	wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
		3989
		3990	tmp = I915_READ(DSPHOWM);
		3991	wm->sr.plane \|= _FW_WM(tmp, SR_HI) << 9;
		3992	wm->pipe[PIPE_B].sprite[1] \|= _FW_WM(tmp, SPRITED_HI) << 8;
		3993	wm->pipe[PIPE_B].sprite[0] \|= _FW_WM(tmp, SPRITEC_HI) << 8;
		3994	wm->pipe[PIPE_B].primary \|= _FW_WM(tmp, PLANEB_HI) << 8;
		3995	wm->pipe[PIPE_A].sprite[1] \|= _FW_WM(tmp, SPRITEB_HI) << 8;
		3996	wm->pipe[PIPE_A].sprite[0] \|= _FW_WM(tmp, SPRITEA_HI) << 8;
		3997	wm->pipe[PIPE_A].primary \|= _FW_WM(tmp, PLANEA_HI) << 8;
		3998	}
		3999	}
		4000
		4001	#undef _FW_WM
		4002	#undef _FW_WM_VLV
		4003
		4004	void vlv_wm_get_hw_state(struct drm_device *dev)
		4005	{
		4006	struct drm_i915_private *dev_priv = to_i915(dev);
		4007	struct vlv_wm_values *wm = &dev_priv->wm.vlv;
		4008	struct intel_plane *plane;
		4009	enum pipe pipe;
		4010	u32 val;
		4011
		4012	vlv_read_wm_values(dev_priv, wm);
		4013
		4014	for_each_intel_plane(dev, plane) {
		4015	switch (plane->base.type) {
		4016	int sprite;
		4017	case DRM_PLANE_TYPE_CURSOR:
		4018	plane->wm.fifo_size = 63;
		4019	break;
		4020	case DRM_PLANE_TYPE_PRIMARY:
		4021	plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0);
		4022	break;
		4023	case DRM_PLANE_TYPE_OVERLAY:
		4024	sprite = plane->plane;
		4025	plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1);
		4026	break;
		4027	}
		4028	}
		4029
		4030	wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
		4031	wm->level = VLV_WM_LEVEL_PM2;
		4032
		4033	if (IS_CHERRYVIEW(dev_priv)) {
		4034	mutex_lock(&dev_priv->rps.hw_lock);
		4035
		4036	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
		4037	if (val & DSP_MAXFIFO_PM5_ENABLE)
		4038	wm->level = VLV_WM_LEVEL_PM5;
		4039
		4040	/*
		4041	* If DDR DVFS is disabled in the BIOS, Punit
		4042	* will never ack the request. So if that happens
		4043	* assume we don't have to enable/disable DDR DVFS
		4044	* dynamically. To test that just set the REQ_ACK
		4045	* bit to poke the Punit, but don't change the
		4046	* HIGH/LOW bits so that we don't actually change
		4047	* the current state.
		4048	*/
		4049	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		4050	val \|= FORCE_DDR_FREQ_REQ_ACK;
		4051	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
		4052
		4053	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		4054	FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
		4055	DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
		4056	"assuming DDR DVFS is disabled\n");
		4057	dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
		4058	} else {
		4059	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		4060	if ((val & FORCE_DDR_HIGH_FREQ) == 0)
		4061	wm->level = VLV_WM_LEVEL_DDR_DVFS;
		4062	}
		4063
		4064	mutex_unlock(&dev_priv->rps.hw_lock);
		4065	}
		4066
		4067	for_each_pipe(dev_priv, pipe)
		4068	DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
		4069	pipe_name(pipe), wm->pipe[pipe].primary, wm->pipe[pipe].cursor,
		4070	wm->pipe[pipe].sprite[0], wm->pipe[pipe].sprite[1]);
		4071
		4072	DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
		4073	wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
		4074	}
		4075
4560	Serge	4076	void ilk_wm_get_hw_state(struct drm_device *dev)
3031	serge	4077	{
		4078	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4079	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		4080	struct drm_crtc *crtc;
3031	serge	4081
5060	serge	4082	for_each_crtc(dev, crtc)
4560	Serge	4083	ilk_pipe_wm_get_hw_state(crtc);
4104	Serge	4084
4560	Serge	4085	hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
		4086	hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
		4087	hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3031	serge	4088
4560	Serge	4089	hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
5060	serge	4090	if (INTEL_INFO(dev)->gen >= 7) {
5354	serge	4091	hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		4092	hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
5060	serge	4093	}
3031	serge	4094
4560	Serge	4095	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4096	hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
		4097	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
		4098	else if (IS_IVYBRIDGE(dev))
		4099	hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
		4100	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3031	serge	4101
4560	Serge	4102	hw->enable_fbc_wm =
		4103	!(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3031	serge	4104	}
		4105
		4106	/**
		4107	* intel_update_watermarks - update FIFO watermark values based on current modes
		4108	*
		4109	* Calculate watermark values for the various WM regs based on current mode
		4110	* and plane configuration.
		4111	*
		4112	* There are several cases to deal with here:
		4113	* - normal (i.e. non-self-refresh)
		4114	* - self-refresh (SR) mode
		4115	* - lines are large relative to FIFO size (buffer can hold up to 2)
		4116	* - lines are small relative to FIFO size (buffer can hold more than 2
		4117	* lines), so need to account for TLB latency
		4118	*
		4119	* The normal calculation is:
		4120	* watermark = dotclock * bytes per pixel * latency
		4121	* where latency is platform & configuration dependent (we assume pessimal
		4122	* values here).
		4123	*
		4124	* The SR calculation is:
		4125	* watermark = (trunc(latency/line time)+1) * surface width *
		4126	* bytes per pixel
		4127	* where
		4128	* line time = htotal / dotclock
		4129	* surface width = hdisplay for normal plane and 64 for cursor
		4130	* and latency is assumed to be high, as above.
		4131	*
		4132	* The final value programmed to the register should always be rounded up,
		4133	* and include an extra 2 entries to account for clock crossings.
		4134	*
		4135	* We don't use the sprite, so we can ignore that. And on Crestline we have
		4136	* to set the non-SR watermarks to 8.
		4137	*/
4560	Serge	4138	void intel_update_watermarks(struct drm_crtc *crtc)
3031	serge	4139	{
4560	Serge	4140	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3031	serge	4141
		4142	if (dev_priv->display.update_wm)
4560	Serge	4143	dev_priv->display.update_wm(crtc);
3031	serge	4144	}
		4145
4104	Serge	4146	void intel_update_sprite_watermarks(struct drm_plane *plane,
		4147	struct drm_crtc *crtc,
5060	serge	4148	uint32_t sprite_width,
		4149	uint32_t sprite_height,
		4150	int pixel_size,
4104	Serge	4151	bool enabled, bool scaled)
3031	serge	4152	{
4104	Serge	4153	struct drm_i915_private *dev_priv = plane->dev->dev_private;
3031	serge	4154
		4155	if (dev_priv->display.update_sprite_wm)
5060	serge	4156	dev_priv->display.update_sprite_wm(plane, crtc,
		4157	sprite_width, sprite_height,
4104	Serge	4158	pixel_size, enabled, scaled);
3031	serge	4159	}
		4160
		4161	/**
		4162	* Lock protecting IPS related data structures
		4163	*/
		4164	DEFINE_SPINLOCK(mchdev_lock);
		4165
		4166	/* Global for IPS driver to get at the current i915 device. Protected by
		4167	* mchdev_lock. */
		4168	static struct drm_i915_private *i915_mch_dev;
		4169
		4170	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		4171	{
		4172	struct drm_i915_private *dev_priv = dev->dev_private;
		4173	u16 rgvswctl;
		4174
		4175	assert_spin_locked(&mchdev_lock);
		4176
		4177	rgvswctl = I915_READ16(MEMSWCTL);
		4178	if (rgvswctl & MEMCTL_CMD_STS) {
		4179	DRM_DEBUG("gpu busy, RCS change rejected\n");
		4180	return false; /* still busy with another command */
		4181	}
		4182
		4183	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		4184	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		4185	I915_WRITE16(MEMSWCTL, rgvswctl);
		4186	POSTING_READ16(MEMSWCTL);
		4187
		4188	rgvswctl \|= MEMCTL_CMD_STS;
		4189	I915_WRITE16(MEMSWCTL, rgvswctl);
		4190
		4191	return true;
		4192	}
		4193
		4194	static void ironlake_enable_drps(struct drm_device *dev)
		4195	{
		4196	struct drm_i915_private *dev_priv = dev->dev_private;
		4197	u32 rgvmodectl = I915_READ(MEMMODECTL);
		4198	u8 fmax, fmin, fstart, vstart;
		4199
		4200	spin_lock_irq(&mchdev_lock);
		4201
		4202	/* Enable temp reporting */
		4203	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		4204	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		4205
		4206	/* 100ms RC evaluation intervals */
		4207	I915_WRITE(RCUPEI, 100000);
		4208	I915_WRITE(RCDNEI, 100000);
		4209
		4210	/* Set max/min thresholds to 90ms and 80ms respectively */
		4211	I915_WRITE(RCBMAXAVG, 90000);
		4212	I915_WRITE(RCBMINAVG, 80000);
		4213
		4214	I915_WRITE(MEMIHYST, 1);
		4215
		4216	/* Set up min, max, and cur for interrupt handling */
		4217	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		4218	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		4219	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		4220	MEMMODE_FSTART_SHIFT;
		4221
6084	serge	4222	vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
3031	serge	4223	PXVFREQ_PX_SHIFT;
		4224
		4225	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		4226	dev_priv->ips.fstart = fstart;
		4227
		4228	dev_priv->ips.max_delay = fstart;
		4229	dev_priv->ips.min_delay = fmin;
		4230	dev_priv->ips.cur_delay = fstart;
		4231
		4232	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		4233	fmax, fmin, fstart);
		4234
		4235	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		4236
		4237	/*
		4238	* Interrupts will be enabled in ironlake_irq_postinstall
		4239	*/
		4240
		4241	I915_WRITE(VIDSTART, vstart);
		4242	POSTING_READ(VIDSTART);
		4243
		4244	rgvmodectl \|= MEMMODE_SWMODE_EN;
		4245	I915_WRITE(MEMMODECTL, rgvmodectl);
		4246
		4247	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		4248	DRM_ERROR("stuck trying to change perf mode\n");
		4249	mdelay(1);
		4250
		4251	ironlake_set_drps(dev, fstart);
		4252
6084	serge	4253	dev_priv->ips.last_count1 = I915_READ(DMIEC) +
		4254	I915_READ(DDREC) + I915_READ(CSIEC);
5060	serge	4255	dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6084	serge	4256	dev_priv->ips.last_count2 = I915_READ(GFXEC);
5060	serge	4257	dev_priv->ips.last_time2 = ktime_get_raw_ns();
3031	serge	4258
		4259	spin_unlock_irq(&mchdev_lock);
		4260	}
		4261
		4262	static void ironlake_disable_drps(struct drm_device *dev)
		4263	{
		4264	struct drm_i915_private *dev_priv = dev->dev_private;
		4265	u16 rgvswctl;
		4266
		4267	spin_lock_irq(&mchdev_lock);
		4268
		4269	rgvswctl = I915_READ16(MEMSWCTL);
		4270
		4271	/* Ack interrupts, disable EFC interrupt */
		4272	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		4273	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		4274	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		4275	I915_WRITE(DEIIR, DE_PCU_EVENT);
		4276	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		4277
		4278	/* Go back to the starting frequency */
		4279	ironlake_set_drps(dev, dev_priv->ips.fstart);
		4280	mdelay(1);
		4281	rgvswctl \|= MEMCTL_CMD_STS;
		4282	I915_WRITE(MEMSWCTL, rgvswctl);
		4283	mdelay(1);
		4284
		4285	spin_unlock_irq(&mchdev_lock);
		4286	}
		4287
		4288	/* There's a funny hw issue where the hw returns all 0 when reading from
		4289	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		4290	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		4291	* all limits and the gpu stuck at whatever frequency it is at atm).
		4292	*/
6084	serge	4293	static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3031	serge	4294	{
		4295	u32 limits;
		4296
		4297	/* Only set the down limit when we've reached the lowest level to avoid
		4298	* getting more interrupts, otherwise leave this clear. This prevents a
		4299	* race in the hw when coming out of rc6: There's a tiny window where
		4300	* the hw runs at the minimal clock before selecting the desired
		4301	* frequency, if the down threshold expires in that window we will not
		4302	* receive a down interrupt. */
6084	serge	4303	if (IS_GEN9(dev_priv->dev)) {
		4304	limits = (dev_priv->rps.max_freq_softlimit) << 23;
		4305	if (val <= dev_priv->rps.min_freq_softlimit)
		4306	limits \|= (dev_priv->rps.min_freq_softlimit) << 14;
		4307	} else {
		4308	limits = dev_priv->rps.max_freq_softlimit << 24;
		4309	if (val <= dev_priv->rps.min_freq_softlimit)
		4310	limits \|= dev_priv->rps.min_freq_softlimit << 16;
		4311	}
3031	serge	4312
		4313	return limits;
		4314	}
		4315
4560	Serge	4316	static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
		4317	{
		4318	int new_power;
6084	serge	4319	u32 threshold_up = 0, threshold_down = 0; /* in % */
		4320	u32 ei_up = 0, ei_down = 0;
4560	Serge	4321
		4322	new_power = dev_priv->rps.power;
		4323	switch (dev_priv->rps.power) {
		4324	case LOW_POWER:
5060	serge	4325	if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
4560	Serge	4326	new_power = BETWEEN;
		4327	break;
		4328
		4329	case BETWEEN:
5060	serge	4330	if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
4560	Serge	4331	new_power = LOW_POWER;
5060	serge	4332	else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
4560	Serge	4333	new_power = HIGH_POWER;
		4334	break;
		4335
		4336	case HIGH_POWER:
5060	serge	4337	if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
4560	Serge	4338	new_power = BETWEEN;
		4339	break;
		4340	}
		4341	/* Max/min bins are special */
6084	serge	4342	if (val <= dev_priv->rps.min_freq_softlimit)
4560	Serge	4343	new_power = LOW_POWER;
6084	serge	4344	if (val >= dev_priv->rps.max_freq_softlimit)
4560	Serge	4345	new_power = HIGH_POWER;
		4346	if (new_power == dev_priv->rps.power)
		4347	return;
		4348
		4349	/* Note the units here are not exactly 1us, but 1280ns. */
		4350	switch (new_power) {
		4351	case LOW_POWER:
		4352	/* Upclock if more than 95% busy over 16ms */
6084	serge	4353	ei_up = 16000;
		4354	threshold_up = 95;
4560	Serge	4355
		4356	/* Downclock if less than 85% busy over 32ms */
6084	serge	4357	ei_down = 32000;
		4358	threshold_down = 85;
4560	Serge	4359	break;
		4360
		4361	case BETWEEN:
		4362	/* Upclock if more than 90% busy over 13ms */
6084	serge	4363	ei_up = 13000;
		4364	threshold_up = 90;
4560	Serge	4365
		4366	/* Downclock if less than 75% busy over 32ms */
6084	serge	4367	ei_down = 32000;
		4368	threshold_down = 75;
4560	Serge	4369	break;
		4370
		4371	case HIGH_POWER:
		4372	/* Upclock if more than 85% busy over 10ms */
6084	serge	4373	ei_up = 10000;
		4374	threshold_up = 85;
4560	Serge	4375
		4376	/* Downclock if less than 60% busy over 32ms */
6084	serge	4377	ei_down = 32000;
		4378	threshold_down = 60;
4560	Serge	4379	break;
		4380	}
		4381
6084	serge	4382	I915_WRITE(GEN6_RP_UP_EI,
		4383	GT_INTERVAL_FROM_US(dev_priv, ei_up));
		4384	I915_WRITE(GEN6_RP_UP_THRESHOLD,
		4385	GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100)));
		4386
		4387	I915_WRITE(GEN6_RP_DOWN_EI,
		4388	GT_INTERVAL_FROM_US(dev_priv, ei_down));
		4389	I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
		4390	GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100)));
		4391
		4392	I915_WRITE(GEN6_RP_CONTROL,
		4393	GEN6_RP_MEDIA_TURBO \|
		4394	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4395	GEN6_RP_MEDIA_IS_GFX \|
		4396	GEN6_RP_ENABLE \|
		4397	GEN6_RP_UP_BUSY_AVG \|
		4398	GEN6_RP_DOWN_IDLE_AVG);
		4399
4560	Serge	4400	dev_priv->rps.power = new_power;
6084	serge	4401	dev_priv->rps.up_threshold = threshold_up;
		4402	dev_priv->rps.down_threshold = threshold_down;
4560	Serge	4403	dev_priv->rps.last_adj = 0;
		4404	}
		4405
5060	serge	4406	static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
		4407	{
		4408	u32 mask = 0;
		4409
		4410	if (val > dev_priv->rps.min_freq_softlimit)
6084	serge	4411	mask \|= GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_DOWN_THRESHOLD \| GEN6_PM_RP_DOWN_TIMEOUT;
5060	serge	4412	if (val < dev_priv->rps.max_freq_softlimit)
6084	serge	4413	mask \|= GEN6_PM_RP_UP_EI_EXPIRED \| GEN6_PM_RP_UP_THRESHOLD;
5060	serge	4414
		4415	mask &= dev_priv->pm_rps_events;
		4416
6084	serge	4417	return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
5060	serge	4418	}
		4419
		4420	/* gen6_set_rps is called to update the frequency request, but should also be
		4421	* called when the range (min_delay and max_delay) is modified so that we can
		4422	* update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
6084	serge	4423	static void gen6_set_rps(struct drm_device *dev, u8 val)
3031	serge	4424	{
		4425	struct drm_i915_private *dev_priv = dev->dev_private;
		4426
6084	serge	4427	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
		4428	if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0))
		4429	return;
		4430
3243	Serge	4431	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6084	serge	4432	WARN_ON(val > dev_priv->rps.max_freq);
		4433	WARN_ON(val < dev_priv->rps.min_freq);
3031	serge	4434
5060	serge	4435	/* min/max delay may still have been modified so be sure to
		4436	* write the limits value.
		4437	*/
		4438	if (val != dev_priv->rps.cur_freq) {
5354	serge	4439	gen6_set_rps_thresholds(dev_priv, val);
4560	Serge	4440
6084	serge	4441	if (IS_GEN9(dev))
5354	serge	4442	I915_WRITE(GEN6_RPNSWREQ,
6084	serge	4443	GEN9_FREQUENCY(val));
		4444	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4445	I915_WRITE(GEN6_RPNSWREQ,
5354	serge	4446	HSW_FREQUENCY(val));
		4447	else
		4448	I915_WRITE(GEN6_RPNSWREQ,
		4449	GEN6_FREQUENCY(val) \|
		4450	GEN6_OFFSET(0) \|
		4451	GEN6_AGGRESSIVE_TURBO);
5060	serge	4452	}
3031	serge	4453
		4454	/* Make sure we continue to get interrupts
		4455	* until we hit the minimum or maximum frequencies.
		4456	*/
6084	serge	4457	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
5060	serge	4458	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3031	serge	4459
		4460	POSTING_READ(GEN6_RPNSWREQ);
		4461
5060	serge	4462	dev_priv->rps.cur_freq = val;
6084	serge	4463	trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
3031	serge	4464	}
		4465
6084	serge	4466	static void valleyview_set_rps(struct drm_device *dev, u8 val)
5060	serge	4467	{
6084	serge	4468	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4469
6084	serge	4470	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4471	WARN_ON(val > dev_priv->rps.max_freq);
		4472	WARN_ON(val < dev_priv->rps.min_freq);
5060	serge	4473
6084	serge	4474	if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
		4475	"Odd GPU freq value\n"))
		4476	val &= ~1;
5060	serge	4477
6084	serge	4478	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
5060	serge	4479
6084	serge	4480	if (val != dev_priv->rps.cur_freq) {
		4481	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
		4482	if (!IS_CHERRYVIEW(dev_priv))
		4483	gen6_set_rps_thresholds(dev_priv, val);
		4484	}
5060	serge	4485
6084	serge	4486	dev_priv->rps.cur_freq = val;
		4487	trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
		4488	}
5060	serge	4489
6084	serge	4490	/* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
		4491	*
		4492	* * If Gfx is Idle, then
		4493	* 1. Forcewake Media well.
		4494	* 2. Request idle freq.
		4495	* 3. Release Forcewake of Media well.
		4496	*/
		4497	static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
		4498	{
		4499	u32 val = dev_priv->rps.idle_freq;
5060	serge	4500
6084	serge	4501	if (dev_priv->rps.cur_freq <= val)
		4502	return;
5060	serge	4503
6084	serge	4504	/* Wake up the media well, as that takes a lot less
		4505	* power than the Render well. */
		4506	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
		4507	valleyview_set_rps(dev_priv->dev, val);
		4508	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
5060	serge	4509	}
		4510
6084	serge	4511	void gen6_rps_busy(struct drm_i915_private *dev_priv)
3031	serge	4512	{
4560	Serge	4513	mutex_lock(&dev_priv->rps.hw_lock);
		4514	if (dev_priv->rps.enabled) {
6084	serge	4515	if (dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_UP_EI_EXPIRED))
		4516	gen6_rps_reset_ei(dev_priv);
		4517	I915_WRITE(GEN6_PMINTRMSK,
		4518	gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
4560	Serge	4519	}
		4520	mutex_unlock(&dev_priv->rps.hw_lock);
		4521	}
4104	Serge	4522
6084	serge	4523	void gen6_rps_idle(struct drm_i915_private *dev_priv)
4560	Serge	4524	{
		4525	struct drm_device *dev = dev_priv->dev;
4104	Serge	4526
4560	Serge	4527	mutex_lock(&dev_priv->rps.hw_lock);
		4528	if (dev_priv->rps.enabled) {
		4529	if (IS_VALLEYVIEW(dev))
6084	serge	4530	vlv_set_rps_idle(dev_priv);
4560	Serge	4531	else
6084	serge	4532	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4560	Serge	4533	dev_priv->rps.last_adj = 0;
6660	serge	4534	I915_WRITE(GEN6_PMINTRMSK,
		4535	gen6_sanitize_rps_pm_mask(dev_priv, ~0));
4560	Serge	4536	}
		4537	mutex_unlock(&dev_priv->rps.hw_lock);
6084	serge	4538
		4539	spin_lock(&dev_priv->rps.client_lock);
		4540	while (!list_empty(&dev_priv->rps.clients))
		4541	list_del_init(dev_priv->rps.clients.next);
		4542	spin_unlock(&dev_priv->rps.client_lock);
4104	Serge	4543	}
		4544
6084	serge	4545	void gen6_rps_boost(struct drm_i915_private *dev_priv,
		4546	struct intel_rps_client *rps,
		4547	unsigned long submitted)
4104	Serge	4548	{
6084	serge	4549	/* This is intentionally racy! We peek at the state here, then
		4550	* validate inside the RPS worker.
		4551	*/
		4552	if (!(dev_priv->mm.busy &&
		4553	dev_priv->rps.enabled &&
		4554	dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit))
		4555	return;
3031	serge	4556
6084	serge	4557	/* Force a RPS boost (and don't count it against the client) if
		4558	* the GPU is severely congested.
		4559	*/
		4560	if (rps && time_after(jiffies, submitted + DRM_I915_THROTTLE_JIFFIES))
		4561	rps = NULL;
4104	Serge	4562
6084	serge	4563	spin_lock(&dev_priv->rps.client_lock);
		4564	if (rps == NULL \|\| list_empty(&rps->link)) {
		4565	spin_lock_irq(&dev_priv->irq_lock);
		4566	if (dev_priv->rps.interrupts_enabled) {
		4567	dev_priv->rps.client_boost = true;
		4568	queue_work(dev_priv->wq, &dev_priv->rps.work);
		4569	}
		4570	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	4571
6084	serge	4572	if (rps != NULL) {
		4573	list_add(&rps->link, &dev_priv->rps.clients);
		4574	rps->boosts++;
		4575	} else
		4576	dev_priv->rps.boosts++;
		4577	}
		4578	spin_unlock(&dev_priv->rps.client_lock);
		4579	}
4104	Serge	4580
6084	serge	4581	void intel_set_rps(struct drm_device *dev, u8 val)
		4582	{
		4583	if (IS_VALLEYVIEW(dev))
		4584	valleyview_set_rps(dev, val);
		4585	else
		4586	gen6_set_rps(dev, val);
4104	Serge	4587	}
		4588
5354	serge	4589	static void gen9_disable_rps(struct drm_device *dev)
5060	serge	4590	{
		4591	struct drm_i915_private *dev_priv = dev->dev_private;
		4592
5354	serge	4593	I915_WRITE(GEN6_RC_CONTROL, 0);
6084	serge	4594	I915_WRITE(GEN9_PG_ENABLE, 0);
5060	serge	4595	}
		4596
4104	Serge	4597	static void gen6_disable_rps(struct drm_device *dev)
		4598	{
		4599	struct drm_i915_private *dev_priv = dev->dev_private;
		4600
		4601	I915_WRITE(GEN6_RC_CONTROL, 0);
		4602	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		4603	}
		4604
5060	serge	4605	static void cherryview_disable_rps(struct drm_device *dev)
		4606	{
		4607	struct drm_i915_private *dev_priv = dev->dev_private;
		4608
		4609	I915_WRITE(GEN6_RC_CONTROL, 0);
		4610	}
		4611
4104	Serge	4612	static void valleyview_disable_rps(struct drm_device *dev)
		4613	{
		4614	struct drm_i915_private *dev_priv = dev->dev_private;
		4615
5354	serge	4616	/* we're doing forcewake before Disabling RC6,
		4617	* This what the BIOS expects when going into suspend */
6084	serge	4618	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4619
4104	Serge	4620	I915_WRITE(GEN6_RC_CONTROL, 0);
		4621
6084	serge	4622	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	4623	}
		4624
4560	Serge	4625	static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
		4626	{
5060	serge	4627	if (IS_VALLEYVIEW(dev)) {
		4628	if (mode & (GEN7_RC_CTL_TO_MODE \| GEN6_RC_CTL_EI_MODE(1)))
		4629	mode = GEN6_RC_CTL_RC6_ENABLE;
		4630	else
		4631	mode = 0;
		4632	}
5354	serge	4633	if (HAS_RC6p(dev))
		4634	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
6084	serge	4635	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		4636	(mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		4637	(mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
5354	serge	4638
		4639	else
		4640	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
		4641	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
4560	Serge	4642	}
		4643
5060	serge	4644	static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
3031	serge	4645	{
6084	serge	4646	/* No RC6 before Ironlake and code is gone for ilk. */
		4647	if (INTEL_INFO(dev)->gen < 6)
4104	Serge	4648	return 0;
		4649
3031	serge	4650	/* Respect the kernel parameter if it is set */
5060	serge	4651	if (enable_rc6 >= 0) {
		4652	int mask;
3031	serge	4653
5354	serge	4654	if (HAS_RC6p(dev))
5060	serge	4655	mask = INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE \|
		4656	INTEL_RC6pp_ENABLE;
		4657	else
		4658	mask = INTEL_RC6_ENABLE;
		4659
		4660	if ((enable_rc6 & mask) != enable_rc6)
		4661	DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
6084	serge	4662	enable_rc6 & mask, enable_rc6, mask);
5060	serge	4663
		4664	return enable_rc6 & mask;
		4665	}
		4666
		4667	if (IS_IVYBRIDGE(dev))
		4668	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
3031	serge	4669
6084	serge	4670	return INTEL_RC6_ENABLE;
5060	serge	4671	}
3031	serge	4672
5060	serge	4673	int intel_enable_rc6(const struct drm_device *dev)
		4674	{
		4675	return i915.enable_rc6;
3031	serge	4676	}
		4677
5354	serge	4678	static void gen6_init_rps_frequencies(struct drm_device *dev)
5060	serge	4679	{
		4680	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4681	uint32_t rp_state_cap;
		4682	u32 ddcc_status = 0;
		4683	int ret;
5060	serge	4684
		4685	/* All of these values are in units of 50MHz */
		4686	dev_priv->rps.cur_freq = 0;
5354	serge	4687	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
6084	serge	4688	if (IS_BROXTON(dev)) {
		4689	rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
		4690	dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
		4691	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4692	dev_priv->rps.min_freq = (rp_state_cap >> 0) & 0xff;
		4693	} else {
		4694	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		4695	dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
		4696	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4697	dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
		4698	}
		4699
5060	serge	4700	/* hw_max = RP0 until we check for overclocking */
		4701	dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
4104	Serge	4702
5354	serge	4703	dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
6084	serge	4704	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev) \|\| IS_SKYLAKE(dev)) {
5354	serge	4705	ret = sandybridge_pcode_read(dev_priv,
		4706	HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
		4707	&ddcc_status);
		4708	if (0 == ret)
		4709	dev_priv->rps.efficient_freq =
6084	serge	4710	clamp_t(u8,
		4711	((ddcc_status >> 8) & 0xff),
		4712	dev_priv->rps.min_freq,
		4713	dev_priv->rps.max_freq);
5354	serge	4714	}
		4715
6084	serge	4716	if (IS_SKYLAKE(dev)) {
		4717	/* Store the frequency values in 16.66 MHZ units, which is
		4718	the natural hardware unit for SKL */
		4719	dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
		4720	dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER;
		4721	dev_priv->rps.min_freq *= GEN9_FREQ_SCALER;
		4722	dev_priv->rps.max_freq *= GEN9_FREQ_SCALER;
		4723	dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER;
		4724	}
		4725
		4726	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		4727
5060	serge	4728	/* Preserve min/max settings in case of re-init */
		4729	if (dev_priv->rps.max_freq_softlimit == 0)
		4730	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4104	Serge	4731
5354	serge	4732	if (dev_priv->rps.min_freq_softlimit == 0) {
		4733	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4734	dev_priv->rps.min_freq_softlimit =
6084	serge	4735	max_t(int, dev_priv->rps.efficient_freq,
		4736	intel_freq_opcode(dev_priv, 450));
5354	serge	4737	else
		4738	dev_priv->rps.min_freq_softlimit =
		4739	dev_priv->rps.min_freq;
		4740	}
4104	Serge	4741	}
		4742
6084	serge	4743	/* See the Gen9_GT_PM_Programming_Guide doc for the below */
5354	serge	4744	static void gen9_enable_rps(struct drm_device *dev)
		4745	{
		4746	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	4747
		4748	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
		4749
		4750	gen6_init_rps_frequencies(dev);
		4751
		4752	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
		4753	if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) {
		4754	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4755	return;
		4756	}
		4757
		4758	/* Program defaults and thresholds for RPS*/
		4759	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4760	GEN9_FREQUENCY(dev_priv->rps.rp1_freq));
		4761
		4762	/* 1 second timeout*/
		4763	I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
		4764	GT_INTERVAL_FROM_US(dev_priv, 1000000));
		4765
		4766	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
		4767
		4768	/* Leaning on the below call to gen6_set_rps to program/setup the
		4769	* Up/Down EI & threshold registers, as well as the RP_CONTROL,
		4770	* RP_INTERRUPT_LIMITS & RPNSWREQ registers */
		4771	dev_priv->rps.power = HIGH_POWER; /* force a reset */
		4772	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
		4773
		4774	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4775	}
		4776
		4777	static void gen9_enable_rc6(struct drm_device *dev)
		4778	{
		4779	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4780	struct intel_engine_cs *ring;
		4781	uint32_t rc6_mask = 0;
		4782	int unused;
		4783
		4784	/* 1a: Software RC state - RC0 */
		4785	I915_WRITE(GEN6_RC_STATE, 0);
		4786
		4787	/* 1b: Get forcewake during program sequence. Although the driver
		4788	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	4789	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4790
		4791	/* 2a: Disable RC states. */
		4792	I915_WRITE(GEN6_RC_CONTROL, 0);
		4793
		4794	/* 2b: Program RC6 thresholds.*/
6084	serge	4795
		4796	/* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
		4797	if (IS_SKYLAKE(dev))
		4798	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
		4799	else
		4800	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
5354	serge	4801	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4802	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4803	for_each_ring(ring, dev_priv, unused)
		4804	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
6084	serge	4805
		4806	if (HAS_GUC_UCODE(dev))
		4807	I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
		4808
5354	serge	4809	I915_WRITE(GEN6_RC_SLEEP, 0);
		4810
6084	serge	4811	/* 2c: Program Coarse Power Gating Policies. */
		4812	I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
		4813	I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
		4814
5354	serge	4815	/* 3a: Enable RC6 */
		4816	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4817	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
		4818	DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
		4819	"on" : "off");
6084	serge	4820	/* WaRsUseTimeoutMode */
		4821	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_D0) \|\|
		4822	(IS_BROXTON(dev) && INTEL_REVID(dev) <= BXT_REVID_A0)) {
		4823	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
		4824	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4825	GEN7_RC_CTL_TO_MODE \|
		4826	rc6_mask);
		4827	} else {
		4828	I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
		4829	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4830	GEN6_RC_CTL_EI_MODE(1) \|
		4831	rc6_mask);
		4832	}
5354	serge	4833
6084	serge	4834	/*
		4835	* 3b: Enable Coarse Power Gating only when RC6 is enabled.
		4836	* WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6.
		4837	*/
		4838	if ((IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) \|\|
		4839	((IS_SKL_GT3(dev) \|\| IS_SKL_GT4(dev)) && (INTEL_REVID(dev) <= SKL_REVID_F0)))
		4840	I915_WRITE(GEN9_PG_ENABLE, 0);
		4841	else
		4842	I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
		4843	(GEN9_RENDER_PG_ENABLE \| GEN9_MEDIA_PG_ENABLE) : 0);
5354	serge	4844
6084	serge	4845	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4846
5354	serge	4847	}
		4848
4560	Serge	4849	static void gen8_enable_rps(struct drm_device *dev)
		4850	{
		4851	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4852	struct intel_engine_cs *ring;
5354	serge	4853	uint32_t rc6_mask = 0;
4560	Serge	4854	int unused;
		4855
		4856	/* 1a: Software RC state - RC0 */
		4857	I915_WRITE(GEN6_RC_STATE, 0);
		4858
		4859	/* 1c & 1d: Get forcewake during program sequence. Although the driver
		4860	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	4861	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4560	Serge	4862
		4863	/* 2a: Disable RC states. */
		4864	I915_WRITE(GEN6_RC_CONTROL, 0);
		4865
5354	serge	4866	/* Initialize rps frequencies */
		4867	gen6_init_rps_frequencies(dev);
4560	Serge	4868
		4869	/* 2b: Program RC6 thresholds.*/
		4870	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		4871	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4872	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4873	for_each_ring(ring, dev_priv, unused)
		4874	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4875	I915_WRITE(GEN6_RC_SLEEP, 0);
5060	serge	4876	if (IS_BROADWELL(dev))
		4877	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
		4878	else
6084	serge	4879	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
4560	Serge	4880
		4881	/* 3: Enable RC6 */
		4882	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4883	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
5060	serge	4884	intel_print_rc6_info(dev, rc6_mask);
		4885	if (IS_BROADWELL(dev))
		4886	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4887	GEN7_RC_CTL_TO_MODE \|
		4888	rc6_mask);
		4889	else
6084	serge	4890	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4891	GEN6_RC_CTL_EI_MODE(1) \|
		4892	rc6_mask);
4560	Serge	4893
		4894	/* 4 Program defaults and thresholds for RPS*/
5060	serge	4895	I915_WRITE(GEN6_RPNSWREQ,
		4896	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
		4897	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4898	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4560	Serge	4899	/* NB: Docs say 1s, and 1000000 - which aren't equivalent */
		4900	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
		4901
		4902	/* Docs recommend 900MHz, and 300 MHz respectively */
		4903	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
5060	serge	4904	dev_priv->rps.max_freq_softlimit << 24 \|
		4905	dev_priv->rps.min_freq_softlimit << 16);
4560	Serge	4906
		4907	I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
		4908	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
		4909	I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
		4910	I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
		4911
		4912	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		4913
		4914	/* 5: Enable RPS */
		4915	I915_WRITE(GEN6_RP_CONTROL,
		4916	GEN6_RP_MEDIA_TURBO \|
		4917	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4918	GEN6_RP_MEDIA_IS_GFX \|
		4919	GEN6_RP_ENABLE \|
		4920	GEN6_RP_UP_BUSY_AVG \|
		4921	GEN6_RP_DOWN_IDLE_AVG);
		4922
		4923	/* 6: Ring frequency + overclocking (our driver does this later */
		4924
5354	serge	4925	dev_priv->rps.power = HIGH_POWER; /* force a reset */
6084	serge	4926	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4560	Serge	4927
6084	serge	4928	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4560	Serge	4929	}
		4930
3031	serge	4931	static void gen6_enable_rps(struct drm_device *dev)
		4932	{
		4933	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4934	struct intel_engine_cs *ring;
		4935	u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
3031	serge	4936	u32 gtfifodbg;
		4937	int rc6_mode;
3243	Serge	4938	int i, ret;
3031	serge	4939
3243	Serge	4940	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4941
		4942	/* Here begins a magic sequence of register writes to enable
		4943	* auto-downclocking.
		4944	*
		4945	* Perhaps there might be some value in exposing these to
		4946	* userspace...
		4947	*/
		4948	I915_WRITE(GEN6_RC_STATE, 0);
		4949
		4950	/* Clear the DBG now so we don't confuse earlier errors */
		4951	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		4952	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		4953	I915_WRITE(GTFIFODBG, gtfifodbg);
		4954	}
		4955
6084	serge	4956	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
3031	serge	4957
5354	serge	4958	/* Initialize rps frequencies */
		4959	gen6_init_rps_frequencies(dev);
3031	serge	4960
		4961	/* disable the counters and set deterministic thresholds */
		4962	I915_WRITE(GEN6_RC_CONTROL, 0);
		4963
		4964	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		4965	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		4966	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		4967	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		4968	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		4969
		4970	for_each_ring(ring, dev_priv, i)
		4971	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4972
		4973	I915_WRITE(GEN6_RC_SLEEP, 0);
		4974	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4560	Serge	4975	if (IS_IVYBRIDGE(dev))
4104	Serge	4976	I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
		4977	else
6084	serge	4978	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	4979	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	4980	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		4981
		4982	/* Check if we are enabling RC6 */
		4983	rc6_mode = intel_enable_rc6(dev_priv->dev);
		4984	if (rc6_mode & INTEL_RC6_ENABLE)
		4985	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		4986
		4987	/* We don't use those on Haswell */
		4988	if (!IS_HASWELL(dev)) {
		4989	if (rc6_mode & INTEL_RC6p_ENABLE)
		4990	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		4991
		4992	if (rc6_mode & INTEL_RC6pp_ENABLE)
		4993	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		4994	}
		4995
4560	Serge	4996	intel_print_rc6_info(dev, rc6_mask);
3031	serge	4997
		4998	I915_WRITE(GEN6_RC_CONTROL,
		4999	rc6_mask \|
		5000	GEN6_RC_CTL_EI_MODE(1) \|
		5001	GEN6_RC_CTL_HW_ENABLE);
		5002
4560	Serge	5003	/* Power down if completely idle for over 50ms */
		5004	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
3031	serge	5005	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5006
3243	Serge	5007	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
5060	serge	5008	if (ret)
		5009	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		5010
6084	serge	5011	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
		5012	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		5013	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
5060	serge	5014	(dev_priv->rps.max_freq_softlimit & 0xff) * 50,
6084	serge	5015	(pcu_mbox & 0xff) * 50);
5060	serge	5016	dev_priv->rps.max_freq = pcu_mbox & 0xff;
3031	serge	5017	}
		5018
4560	Serge	5019	dev_priv->rps.power = HIGH_POWER; /* force a reset */
6084	serge	5020	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
3031	serge	5021
3243	Serge	5022	rc6vids = 0;
		5023	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		5024	if (IS_GEN6(dev) && ret) {
		5025	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		5026	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		5027	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		5028	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		5029	rc6vids &= 0xffff00;
		5030	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		5031	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		5032	if (ret)
		5033	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		5034	}
		5035
6084	serge	5036	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
3031	serge	5037	}
		5038
5060	serge	5039	static void __gen6_update_ring_freq(struct drm_device *dev)
3031	serge	5040	{
		5041	struct drm_i915_private *dev_priv = dev->dev_private;
		5042	int min_freq = 15;
3746	Serge	5043	unsigned int gpu_freq;
		5044	unsigned int max_ia_freq, min_ring_freq;
6084	serge	5045	unsigned int max_gpu_freq, min_gpu_freq;
3031	serge	5046	int scaling_factor = 180;
4560	Serge	5047	struct cpufreq_policy *policy;
3031	serge	5048
3243	Serge	5049	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	5050
		5051	max_ia_freq = cpufreq_quick_get_max(0);
		5052	/*
5060	serge	5053	* Default to measured freq if none found, PCU will ensure we
		5054	* don't go over
3031	serge	5055	*/
		5056	max_ia_freq = tsc_khz;
		5057
		5058	/* Convert from kHz to MHz */
		5059	max_ia_freq /= 1000;
		5060
4560	Serge	5061	min_ring_freq = I915_READ(DCLK) & 0xf;
		5062	/* convert DDR frequency from units of 266.6MHz to bandwidth */
		5063	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3746	Serge	5064
6084	serge	5065	if (IS_SKYLAKE(dev)) {
		5066	/* Convert GT frequency to 50 HZ units */
		5067	min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
		5068	max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
		5069	} else {
		5070	min_gpu_freq = dev_priv->rps.min_freq;
		5071	max_gpu_freq = dev_priv->rps.max_freq;
		5072	}
		5073
3031	serge	5074	/*
		5075	* For each potential GPU frequency, load a ring frequency we'd like
		5076	* to use for memory access. We do this by specifying the IA frequency
		5077	* the PCU should use as a reference to determine the ring frequency.
		5078	*/
6084	serge	5079	for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
		5080	int diff = max_gpu_freq - gpu_freq;
3746	Serge	5081	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	5082
6084	serge	5083	if (IS_SKYLAKE(dev)) {
		5084	/*
		5085	* ring_freq = 2 * GT. ring_freq is in 100MHz units
		5086	* No floor required for ring frequency on SKL.
		5087	*/
		5088	ring_freq = gpu_freq;
		5089	} else if (INTEL_INFO(dev)->gen >= 8) {
4560	Serge	5090	/* max(2 * GT, DDR). NB: GT is 50MHz units */
		5091	ring_freq = max(min_ring_freq, gpu_freq);
		5092	} else if (IS_HASWELL(dev)) {
		5093	ring_freq = mult_frac(gpu_freq, 5, 4);
3746	Serge	5094	ring_freq = max(min_ring_freq, ring_freq);
		5095	/* leave ia_freq as the default, chosen by cpufreq */
		5096	} else {
		5097	/* On older processors, there is no separate ring
		5098	* clock domain, so in order to boost the bandwidth
		5099	* of the ring, we need to upclock the CPU (ia_freq).
		5100	*
		5101	* For GPU frequencies less than 750MHz,
		5102	* just use the lowest ring freq.
6084	serge	5103	*/
		5104	if (gpu_freq < min_freq)
		5105	ia_freq = 800;
		5106	else
		5107	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		5108	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	5109	}
3031	serge	5110
3243	Serge	5111	sandybridge_pcode_write(dev_priv,
		5112	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	5113	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		5114	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		5115	gpu_freq);
3031	serge	5116	}
		5117	}
		5118
5060	serge	5119	void gen6_update_ring_freq(struct drm_device *dev)
4104	Serge	5120	{
5060	serge	5121	struct drm_i915_private *dev_priv = dev->dev_private;
		5122
6084	serge	5123	if (!HAS_CORE_RING_FREQ(dev))
5060	serge	5124	return;
		5125
		5126	mutex_lock(&dev_priv->rps.hw_lock);
		5127	__gen6_update_ring_freq(dev);
		5128	mutex_unlock(&dev_priv->rps.hw_lock);
		5129	}
		5130
		5131	static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
		5132	{
6084	serge	5133	struct drm_device *dev = dev_priv->dev;
4104	Serge	5134	u32 val, rp0;
		5135
6084	serge	5136	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
5060	serge	5137
6084	serge	5138	switch (INTEL_INFO(dev)->eu_total) {
		5139	case 8:
		5140	/* (2 * 4) config */
		5141	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
		5142	break;
		5143	case 12:
		5144	/* (2 * 6) config */
		5145	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
		5146	break;
		5147	case 16:
		5148	/* (2 * 8) config */
		5149	default:
		5150	/* Setting (2 * 8) Min RP0 for any other combination */
		5151	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
		5152	break;
		5153	}
		5154
		5155	rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
		5156
5060	serge	5157	return rp0;
		5158	}
		5159
		5160	static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5161	{
		5162	u32 val, rpe;
		5163
		5164	val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
		5165	rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
		5166
		5167	return rpe;
		5168	}
		5169
		5170	static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5171	{
		5172	u32 val, rp1;
		5173
6084	serge	5174	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
		5175	rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
5060	serge	5176
		5177	return rp1;
		5178	}
		5179
		5180	static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5181	{
		5182	u32 val, rp1;
		5183
4104	Serge	5184	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5185
5060	serge	5186	rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
		5187
		5188	return rp1;
		5189	}
		5190
		5191	static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
		5192	{
		5193	u32 val, rp0;
		5194
		5195	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5196
4104	Serge	5197	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
		5198	/* Clamp to max */
		5199	rp0 = min_t(u32, rp0, 0xea);
		5200
		5201	return rp0;
		5202	}
		5203
		5204	static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5205	{
		5206	u32 val, rpe;
		5207
		5208	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
		5209	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
		5210	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
		5211	rpe \|= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
		5212
		5213	return rpe;
		5214	}
		5215
5060	serge	5216	static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
4104	Serge	5217	{
		5218	return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
		5219	}
		5220
5060	serge	5221	/* Check that the pctx buffer wasn't move under us. */
		5222	static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
		5223	{
		5224	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5225
		5226	WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
		5227	dev_priv->vlv_pctx->stolen->start);
		5228	}
		5229
		5230
		5231	/* Check that the pcbr address is not empty. */
		5232	static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
		5233	{
		5234	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5235
		5236	WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
		5237	}
		5238
		5239	static void cherryview_setup_pctx(struct drm_device *dev)
		5240	{
		5241	struct drm_i915_private *dev_priv = dev->dev_private;
		5242	unsigned long pctx_paddr, paddr;
		5243	struct i915_gtt *gtt = &dev_priv->gtt;
		5244	u32 pcbr;
		5245	int pctx_size = 32*1024;
		5246
		5247	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5248
		5249	pcbr = I915_READ(VLV_PCBR);
		5250	if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
5354	serge	5251	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
5060	serge	5252	paddr = (dev_priv->mm.stolen_base +
		5253	(gtt->stolen_size - pctx_size));
		5254
		5255	pctx_paddr = (paddr & (~4095));
		5256	I915_WRITE(VLV_PCBR, pctx_paddr);
		5257	}
5354	serge	5258
		5259	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5060	serge	5260	}
		5261
4104	Serge	5262	static void valleyview_setup_pctx(struct drm_device *dev)
		5263	{
		5264	struct drm_i915_private *dev_priv = dev->dev_private;
		5265	struct drm_i915_gem_object *pctx;
		5266	unsigned long pctx_paddr;
		5267	u32 pcbr;
		5268	int pctx_size = 24*1024;
		5269
5060	serge	5270	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5271
4104	Serge	5272	pcbr = I915_READ(VLV_PCBR);
		5273	if (pcbr) {
		5274	/* BIOS set it up already, grab the pre-alloc'd space */
		5275	int pcbr_offset;
		5276
		5277	pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
		5278	pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
		5279	pcbr_offset,
		5280	I915_GTT_OFFSET_NONE,
		5281	pctx_size);
		5282	goto out;
		5283	}
		5284
5354	serge	5285	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
		5286
4104	Serge	5287	/*
		5288	* From the Gunit register HAS:
		5289	* The Gfx driver is expected to program this register and ensure
		5290	* proper allocation within Gfx stolen memory. For example, this
		5291	* register should be programmed such than the PCBR range does not
		5292	* overlap with other ranges, such as the frame buffer, protected
		5293	* memory, or any other relevant ranges.
		5294	*/
		5295	pctx = i915_gem_object_create_stolen(dev, pctx_size);
		5296	if (!pctx) {
		5297	DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
		5298	return;
		5299	}
		5300
		5301	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
		5302	I915_WRITE(VLV_PCBR, pctx_paddr);
		5303
		5304	out:
5354	serge	5305	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
4104	Serge	5306	dev_priv->vlv_pctx = pctx;
		5307	}
		5308
5060	serge	5309	static void valleyview_cleanup_pctx(struct drm_device *dev)
		5310	{
		5311	struct drm_i915_private *dev_priv = dev->dev_private;
		5312
		5313	if (WARN_ON(!dev_priv->vlv_pctx))
		5314	return;
		5315
		5316	drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
		5317	dev_priv->vlv_pctx = NULL;
		5318	}
		5319
		5320	static void valleyview_init_gt_powersave(struct drm_device *dev)
		5321	{
		5322	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5323	u32 val;
5060	serge	5324
		5325	valleyview_setup_pctx(dev);
		5326
		5327	mutex_lock(&dev_priv->rps.hw_lock);
		5328
5354	serge	5329	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5330	switch ((val >> 6) & 3) {
		5331	case 0:
		5332	case 1:
		5333	dev_priv->mem_freq = 800;
		5334	break;
		5335	case 2:
		5336	dev_priv->mem_freq = 1066;
		5337	break;
		5338	case 3:
		5339	dev_priv->mem_freq = 1333;
		5340	break;
		5341	}
		5342	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5343
5060	serge	5344	dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
		5345	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5346	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6084	serge	5347	intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5060	serge	5348	dev_priv->rps.max_freq);
		5349
		5350	dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
		5351	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6084	serge	5352	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5353	dev_priv->rps.efficient_freq);
		5354
		5355	dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
		5356	DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
6084	serge	5357	intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5060	serge	5358	dev_priv->rps.rp1_freq);
		5359
		5360	dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
		5361	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6084	serge	5362	intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5060	serge	5363	dev_priv->rps.min_freq);
		5364
6084	serge	5365	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		5366
5060	serge	5367	/* Preserve min/max settings in case of re-init */
		5368	if (dev_priv->rps.max_freq_softlimit == 0)
		5369	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5370
		5371	if (dev_priv->rps.min_freq_softlimit == 0)
		5372	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5373
		5374	mutex_unlock(&dev_priv->rps.hw_lock);
		5375	}
		5376
		5377	static void cherryview_init_gt_powersave(struct drm_device *dev)
		5378	{
		5379	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5380	u32 val;
5060	serge	5381
		5382	cherryview_setup_pctx(dev);
		5383
		5384	mutex_lock(&dev_priv->rps.hw_lock);
		5385
6084	serge	5386	mutex_lock(&dev_priv->sb_lock);
5354	serge	5387	val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
6084	serge	5388	mutex_unlock(&dev_priv->sb_lock);
5354	serge	5389
		5390	switch ((val >> 2) & 0x7) {
		5391	case 3:
		5392	dev_priv->mem_freq = 2000;
		5393	break;
6084	serge	5394	default:
5354	serge	5395	dev_priv->mem_freq = 1600;
		5396	break;
		5397	}
		5398	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5399
5060	serge	5400	dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
		5401	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5402	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6084	serge	5403	intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5060	serge	5404	dev_priv->rps.max_freq);
		5405
		5406	dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
		5407	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6084	serge	5408	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5409	dev_priv->rps.efficient_freq);
		5410
		5411	dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
		5412	DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
6084	serge	5413	intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5060	serge	5414	dev_priv->rps.rp1_freq);
		5415
6084	serge	5416	/* PUnit validated range is only [RPe, RP0] */
		5417	dev_priv->rps.min_freq = dev_priv->rps.efficient_freq;
5060	serge	5418	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6084	serge	5419	intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5060	serge	5420	dev_priv->rps.min_freq);
		5421
5354	serge	5422	WARN_ONCE((dev_priv->rps.max_freq \|
		5423	dev_priv->rps.efficient_freq \|
		5424	dev_priv->rps.rp1_freq \|
		5425	dev_priv->rps.min_freq) & 1,
		5426	"Odd GPU freq values\n");
		5427
6084	serge	5428	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		5429
5060	serge	5430	/* Preserve min/max settings in case of re-init */
		5431	if (dev_priv->rps.max_freq_softlimit == 0)
		5432	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5433
		5434	if (dev_priv->rps.min_freq_softlimit == 0)
		5435	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5436
		5437	mutex_unlock(&dev_priv->rps.hw_lock);
		5438	}
		5439
		5440	static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
		5441	{
		5442	valleyview_cleanup_pctx(dev);
		5443	}
		5444
		5445	static void cherryview_enable_rps(struct drm_device *dev)
		5446	{
		5447	struct drm_i915_private *dev_priv = dev->dev_private;
		5448	struct intel_engine_cs *ring;
		5449	u32 gtfifodbg, val, rc6_mode = 0, pcbr;
		5450	int i;
		5451
		5452	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5453
		5454	gtfifodbg = I915_READ(GTFIFODBG);
		5455	if (gtfifodbg) {
		5456	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5457	gtfifodbg);
		5458	I915_WRITE(GTFIFODBG, gtfifodbg);
		5459	}
		5460
		5461	cherryview_check_pctx(dev_priv);
		5462
		5463	/* 1a & 1b: Get forcewake during program sequence. Although the driver
		5464	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	5465	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5060	serge	5466
6084	serge	5467	/* Disable RC states. */
		5468	I915_WRITE(GEN6_RC_CONTROL, 0);
		5469
5060	serge	5470	/* 2a: Program RC6 thresholds.*/
		5471	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		5472	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		5473	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		5474
		5475	for_each_ring(ring, dev_priv, i)
		5476	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5477	I915_WRITE(GEN6_RC_SLEEP, 0);
		5478
6084	serge	5479	/* TO threshold set to 500 us ( 0x186 * 1.28 us) */
		5480	I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
5060	serge	5481
		5482	/* allows RC6 residency counter to work */
		5483	I915_WRITE(VLV_COUNTER_CONTROL,
		5484	_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH \|
		5485	VLV_MEDIA_RC6_COUNT_EN \|
		5486	VLV_RENDER_RC6_COUNT_EN));
		5487
		5488	/* For now we assume BIOS is allocating and populating the PCBR */
		5489	pcbr = I915_READ(VLV_PCBR);
		5490
		5491	/* 3: Enable RC6 */
		5492	if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
		5493	(pcbr >> VLV_PCBR_ADDR_SHIFT))
6084	serge	5494	rc6_mode = GEN7_RC_CTL_TO_MODE;
5060	serge	5495
		5496	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5497
		5498	/* 4 Program defaults and thresholds for RPS*/
6084	serge	5499	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5060	serge	5500	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5501	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5502	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5503	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5504
		5505	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5506
		5507	/* 5: Enable RPS */
		5508	I915_WRITE(GEN6_RP_CONTROL,
		5509	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
6084	serge	5510	GEN6_RP_MEDIA_IS_GFX \|
5060	serge	5511	GEN6_RP_ENABLE \|
		5512	GEN6_RP_UP_BUSY_AVG \|
		5513	GEN6_RP_DOWN_IDLE_AVG);
		5514
6084	serge	5515	/* Setting Fixed Bias */
		5516	val = VLV_OVERRIDE_EN \|
		5517	VLV_SOC_TDP_EN \|
		5518	CHV_BIAS_CPU_50_SOC_50;
		5519	vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
		5520
5060	serge	5521	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5522
5354	serge	5523	/* RPS code assumes GPLL is used */
		5524	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5525
6084	serge	5526	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
5060	serge	5527	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5528
		5529	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
		5530	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
6084	serge	5531	intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5060	serge	5532	dev_priv->rps.cur_freq);
		5533
		5534	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
6084	serge	5535	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5536	dev_priv->rps.efficient_freq);
		5537
		5538	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
		5539
6084	serge	5540	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5060	serge	5541	}
		5542
4104	Serge	5543	static void valleyview_enable_rps(struct drm_device *dev)
		5544	{
		5545	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	5546	struct intel_engine_cs *ring;
4560	Serge	5547	u32 gtfifodbg, val, rc6_mode = 0;
4104	Serge	5548	int i;
		5549
		5550	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5551
5060	serge	5552	valleyview_check_pctx(dev_priv);
		5553
4104	Serge	5554	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4560	Serge	5555	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5556	gtfifodbg);
4104	Serge	5557	I915_WRITE(GTFIFODBG, gtfifodbg);
		5558	}
		5559
4560	Serge	5560	/* If VLV, Forcewake all wells, else re-direct to regular path */
6084	serge	5561	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4104	Serge	5562
6084	serge	5563	/* Disable RC states. */
		5564	I915_WRITE(GEN6_RC_CONTROL, 0);
		5565
		5566	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
4104	Serge	5567	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5568	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5569	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5570	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5571
		5572	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5573
		5574	I915_WRITE(GEN6_RP_CONTROL,
		5575	GEN6_RP_MEDIA_TURBO \|
		5576	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		5577	GEN6_RP_MEDIA_IS_GFX \|
		5578	GEN6_RP_ENABLE \|
		5579	GEN6_RP_UP_BUSY_AVG \|
		5580	GEN6_RP_DOWN_IDLE_CONT);
		5581
		5582	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
		5583	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		5584	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		5585
		5586	for_each_ring(ring, dev_priv, i)
		5587	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5588
4560	Serge	5589	I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4104	Serge	5590
		5591	/* allows RC6 residency counter to work */
4560	Serge	5592	I915_WRITE(VLV_COUNTER_CONTROL,
5060	serge	5593	_MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN \|
		5594	VLV_RENDER_RC0_COUNT_EN \|
4560	Serge	5595	VLV_MEDIA_RC6_COUNT_EN \|
		5596	VLV_RENDER_RC6_COUNT_EN));
5060	serge	5597
4560	Serge	5598	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		5599	rc6_mode = GEN7_RC_CTL_TO_MODE \| VLV_RC_CTL_CTX_RST_PARALLEL;
4104	Serge	5600
4560	Serge	5601	intel_print_rc6_info(dev, rc6_mode);
		5602
		5603	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5604
6084	serge	5605	/* Setting Fixed Bias */
		5606	val = VLV_OVERRIDE_EN \|
		5607	VLV_SOC_TDP_EN \|
		5608	VLV_BIAS_CPU_125_SOC_875;
		5609	vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
		5610
4104	Serge	5611	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5612
5354	serge	5613	/* RPS code assumes GPLL is used */
		5614	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5615
6084	serge	5616	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
4104	Serge	5617	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5618
5060	serge	5619	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4104	Serge	5620	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
6084	serge	5621	intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5060	serge	5622	dev_priv->rps.cur_freq);
4104	Serge	5623
		5624	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
6084	serge	5625	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5626	dev_priv->rps.efficient_freq);
4104	Serge	5627
5060	serge	5628	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4104	Serge	5629
6084	serge	5630	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	5631	}
		5632
3031	serge	5633	static unsigned long intel_pxfreq(u32 vidfreq)
		5634	{
		5635	unsigned long freq;
		5636	int div = (vidfreq & 0x3f0000) >> 16;
		5637	int post = (vidfreq & 0x3000) >> 12;
		5638	int pre = (vidfreq & 0x7);
		5639
		5640	if (!pre)
		5641	return 0;
		5642
		5643	freq = ((div * 133333) / ((1<
		5644
		5645	return freq;
		5646	}
		5647
		5648	static const struct cparams {
		5649	u16 i;
		5650	u16 t;
		5651	u16 m;
		5652	u16 c;
		5653	} cparams[] = {
		5654	{ 1, 1333, 301, 28664 },
		5655	{ 1, 1066, 294, 24460 },
		5656	{ 1, 800, 294, 25192 },
		5657	{ 0, 1333, 276, 27605 },
		5658	{ 0, 1066, 276, 27605 },
		5659	{ 0, 800, 231, 23784 },
		5660	};
		5661
		5662	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		5663	{
		5664	u64 total_count, diff, ret;
		5665	u32 count1, count2, count3, m = 0, c = 0;
5060	serge	5666	unsigned long now = jiffies_to_msecs(jiffies), diff1;
3031	serge	5667	int i;
		5668
		5669	assert_spin_locked(&mchdev_lock);
		5670
		5671	diff1 = now - dev_priv->ips.last_time1;
		5672
		5673	/* Prevent division-by-zero if we are asking too fast.
		5674	* Also, we don't get interesting results if we are polling
		5675	* faster than once in 10ms, so just return the saved value
		5676	* in such cases.
		5677	*/
		5678	if (diff1 <= 10)
		5679	return dev_priv->ips.chipset_power;
		5680
		5681	count1 = I915_READ(DMIEC);
		5682	count2 = I915_READ(DDREC);
		5683	count3 = I915_READ(CSIEC);
		5684
		5685	total_count = count1 + count2 + count3;
		5686
		5687	/* FIXME: handle per-counter overflow */
		5688	if (total_count < dev_priv->ips.last_count1) {
		5689	diff = ~0UL - dev_priv->ips.last_count1;
		5690	diff += total_count;
		5691	} else {
		5692	diff = total_count - dev_priv->ips.last_count1;
		5693	}
		5694
		5695	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		5696	if (cparams[i].i == dev_priv->ips.c_m &&
		5697	cparams[i].t == dev_priv->ips.r_t) {
		5698	m = cparams[i].m;
		5699	c = cparams[i].c;
		5700	break;
		5701	}
		5702	}
		5703
		5704	diff = div_u64(diff, diff1);
		5705	ret = ((m * diff) + c);
		5706	ret = div_u64(ret, 10);
		5707
		5708	dev_priv->ips.last_count1 = total_count;
		5709	dev_priv->ips.last_time1 = now;
		5710
		5711	dev_priv->ips.chipset_power = ret;
		5712
		5713	return ret;
		5714	}
		5715
		5716	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		5717	{
5060	serge	5718	struct drm_device *dev = dev_priv->dev;
3031	serge	5719	unsigned long val;
		5720
5060	serge	5721	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5722	return 0;
		5723
		5724	spin_lock_irq(&mchdev_lock);
		5725
		5726	val = __i915_chipset_val(dev_priv);
		5727
		5728	spin_unlock_irq(&mchdev_lock);
		5729
		5730	return val;
		5731	}
		5732
		5733	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		5734	{
		5735	unsigned long m, x, b;
		5736	u32 tsfs;
		5737
		5738	tsfs = I915_READ(TSFS);
		5739
		5740	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		5741	x = I915_READ8(TR1);
		5742
		5743	b = tsfs & TSFS_INTR_MASK;
		5744
		5745	return ((m * x) / 127) - b;
		5746	}
		5747
6084	serge	5748	static int _pxvid_to_vd(u8 pxvid)
3031	serge	5749	{
6084	serge	5750	if (pxvid == 0)
		5751	return 0;
		5752
		5753	if (pxvid >= 8 && pxvid < 31)
		5754	pxvid = 31;
		5755
		5756	return (pxvid + 2) * 125;
		5757	}
		5758
		5759	static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		5760	{
5060	serge	5761	struct drm_device *dev = dev_priv->dev;
6084	serge	5762	const int vd = _pxvid_to_vd(pxvid);
		5763	const int vm = vd - 1125;
		5764
5060	serge	5765	if (INTEL_INFO(dev)->is_mobile)
6084	serge	5766	return vm > 0 ? vm : 0;
		5767
		5768	return vd;
3031	serge	5769	}
		5770
		5771	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5772	{
5060	serge	5773	u64 now, diff, diffms;
3031	serge	5774	u32 count;
		5775
		5776	assert_spin_locked(&mchdev_lock);
		5777
5060	serge	5778	now = ktime_get_raw_ns();
		5779	diffms = now - dev_priv->ips.last_time2;
		5780	do_div(diffms, NSEC_PER_MSEC);
3031	serge	5781
		5782	/* Don't divide by 0 */
		5783	if (!diffms)
		5784	return;
		5785
		5786	count = I915_READ(GFXEC);
		5787
		5788	if (count < dev_priv->ips.last_count2) {
		5789	diff = ~0UL - dev_priv->ips.last_count2;
		5790	diff += count;
		5791	} else {
		5792	diff = count - dev_priv->ips.last_count2;
		5793	}
		5794
		5795	dev_priv->ips.last_count2 = count;
		5796	dev_priv->ips.last_time2 = now;
		5797
		5798	/* More magic constants... */
		5799	diff = diff * 1181;
		5800	diff = div_u64(diff, diffms * 10);
		5801	dev_priv->ips.gfx_power = diff;
		5802	}
		5803
		5804	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5805	{
5060	serge	5806	struct drm_device *dev = dev_priv->dev;
		5807
		5808	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5809	return;
		5810
		5811	spin_lock_irq(&mchdev_lock);
		5812
		5813	__i915_update_gfx_val(dev_priv);
		5814
		5815	spin_unlock_irq(&mchdev_lock);
		5816	}
		5817
		5818	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		5819	{
		5820	unsigned long t, corr, state1, corr2, state2;
		5821	u32 pxvid, ext_v;
		5822
		5823	assert_spin_locked(&mchdev_lock);
		5824
6084	serge	5825	pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq));
3031	serge	5826	pxvid = (pxvid >> 24) & 0x7f;
		5827	ext_v = pvid_to_extvid(dev_priv, pxvid);
		5828
		5829	state1 = ext_v;
		5830
		5831	t = i915_mch_val(dev_priv);
		5832
		5833	/* Revel in the empirically derived constants */
		5834
		5835	/* Correction factor in 1/100000 units */
		5836	if (t > 80)
		5837	corr = ((t * 2349) + 135940);
		5838	else if (t >= 50)
		5839	corr = ((t * 964) + 29317);
		5840	else /* < 50 */
		5841	corr = ((t * 301) + 1004);
		5842
		5843	corr = corr * ((150142 * state1) / 10000 - 78642);
		5844	corr /= 100000;
		5845	corr2 = (corr * dev_priv->ips.corr);
		5846
		5847	state2 = (corr2 * state1) / 10000;
		5848	state2 /= 100; /* convert to mW */
		5849
		5850	__i915_update_gfx_val(dev_priv);
		5851
		5852	return dev_priv->ips.gfx_power + state2;
		5853	}
		5854
		5855	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		5856	{
5060	serge	5857	struct drm_device *dev = dev_priv->dev;
3031	serge	5858	unsigned long val;
		5859
5060	serge	5860	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5861	return 0;
		5862
		5863	spin_lock_irq(&mchdev_lock);
		5864
		5865	val = __i915_gfx_val(dev_priv);
		5866
		5867	spin_unlock_irq(&mchdev_lock);
		5868
		5869	return val;
		5870	}
		5871
		5872	/**
		5873	* i915_read_mch_val - return value for IPS use
		5874	*
		5875	* Calculate and return a value for the IPS driver to use when deciding whether
		5876	* we have thermal and power headroom to increase CPU or GPU power budget.
		5877	*/
		5878	unsigned long i915_read_mch_val(void)
		5879	{
		5880	struct drm_i915_private *dev_priv;
		5881	unsigned long chipset_val, graphics_val, ret = 0;
		5882
		5883	spin_lock_irq(&mchdev_lock);
		5884	if (!i915_mch_dev)
		5885	goto out_unlock;
		5886	dev_priv = i915_mch_dev;
		5887
		5888	chipset_val = __i915_chipset_val(dev_priv);
		5889	graphics_val = __i915_gfx_val(dev_priv);
		5890
		5891	ret = chipset_val + graphics_val;
		5892
		5893	out_unlock:
		5894	spin_unlock_irq(&mchdev_lock);
		5895
		5896	return ret;
		5897	}
		5898	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		5899
		5900	/**
		5901	* i915_gpu_raise - raise GPU frequency limit
		5902	*
		5903	* Raise the limit; IPS indicates we have thermal headroom.
		5904	*/
		5905	bool i915_gpu_raise(void)
		5906	{
		5907	struct drm_i915_private *dev_priv;
		5908	bool ret = true;
		5909
		5910	spin_lock_irq(&mchdev_lock);
		5911	if (!i915_mch_dev) {
		5912	ret = false;
		5913	goto out_unlock;
		5914	}
		5915	dev_priv = i915_mch_dev;
		5916
		5917	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		5918	dev_priv->ips.max_delay--;
		5919
		5920	out_unlock:
		5921	spin_unlock_irq(&mchdev_lock);
		5922
		5923	return ret;
		5924	}
		5925	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		5926
		5927	/**
		5928	* i915_gpu_lower - lower GPU frequency limit
		5929	*
		5930	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		5931	* frequency maximum.
		5932	*/
		5933	bool i915_gpu_lower(void)
		5934	{
		5935	struct drm_i915_private *dev_priv;
		5936	bool ret = true;
		5937
		5938	spin_lock_irq(&mchdev_lock);
		5939	if (!i915_mch_dev) {
		5940	ret = false;
		5941	goto out_unlock;
		5942	}
		5943	dev_priv = i915_mch_dev;
		5944
		5945	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		5946	dev_priv->ips.max_delay++;
		5947
		5948	out_unlock:
		5949	spin_unlock_irq(&mchdev_lock);
		5950
		5951	return ret;
		5952	}
		5953	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		5954
		5955	/**
		5956	* i915_gpu_busy - indicate GPU business to IPS
		5957	*
		5958	* Tell the IPS driver whether or not the GPU is busy.
		5959	*/
		5960	bool i915_gpu_busy(void)
		5961	{
		5962	struct drm_i915_private *dev_priv;
5060	serge	5963	struct intel_engine_cs *ring;
3031	serge	5964	bool ret = false;
		5965	int i;
		5966
		5967	spin_lock_irq(&mchdev_lock);
		5968	if (!i915_mch_dev)
		5969	goto out_unlock;
		5970	dev_priv = i915_mch_dev;
		5971
		5972	for_each_ring(ring, dev_priv, i)
		5973	ret \|= !list_empty(&ring->request_list);
		5974
		5975	out_unlock:
		5976	spin_unlock_irq(&mchdev_lock);
		5977
		5978	return ret;
		5979	}
		5980	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		5981
		5982	/**
		5983	* i915_gpu_turbo_disable - disable graphics turbo
		5984	*
		5985	* Disable graphics turbo by resetting the max frequency and setting the
		5986	* current frequency to the default.
		5987	*/
		5988	bool i915_gpu_turbo_disable(void)
		5989	{
		5990	struct drm_i915_private *dev_priv;
		5991	bool ret = true;
		5992
		5993	spin_lock_irq(&mchdev_lock);
		5994	if (!i915_mch_dev) {
		5995	ret = false;
		5996	goto out_unlock;
		5997	}
		5998	dev_priv = i915_mch_dev;
		5999
		6000	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		6001
		6002	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		6003	ret = false;
		6004
		6005	out_unlock:
		6006	spin_unlock_irq(&mchdev_lock);
		6007
		6008	return ret;
		6009	}
		6010	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		6011
		6012	/**
		6013	* Tells the intel_ips driver that the i915 driver is now loaded, if
		6014	* IPS got loaded first.
		6015	*
		6016	* This awkward dance is so that neither module has to depend on the
		6017	* other in order for IPS to do the appropriate communication of
		6018	* GPU turbo limits to i915.
		6019	*/
		6020	static void
		6021	ips_ping_for_i915_load(void)
		6022	{
		6023	void (*link)(void);
		6024
		6025	// link = symbol_get(ips_link_to_i915_driver);
		6026	// if (link) {
		6027	// link();
		6028	// symbol_put(ips_link_to_i915_driver);
		6029	// }
		6030	}
		6031
		6032	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		6033	{
		6034	/* We only register the i915 ips part with intel-ips once everything is
		6035	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		6036	spin_lock_irq(&mchdev_lock);
		6037	i915_mch_dev = dev_priv;
		6038	spin_unlock_irq(&mchdev_lock);
		6039
		6040	ips_ping_for_i915_load();
		6041	}
		6042
		6043	void intel_gpu_ips_teardown(void)
		6044	{
		6045	spin_lock_irq(&mchdev_lock);
		6046	i915_mch_dev = NULL;
		6047	spin_unlock_irq(&mchdev_lock);
		6048	}
5060	serge	6049
3031	serge	6050	static void intel_init_emon(struct drm_device *dev)
		6051	{
		6052	struct drm_i915_private *dev_priv = dev->dev_private;
		6053	u32 lcfuse;
		6054	u8 pxw[16];
		6055	int i;
		6056
		6057	/* Disable to program */
		6058	I915_WRITE(ECR, 0);
		6059	POSTING_READ(ECR);
		6060
		6061	/* Program energy weights for various events */
		6062	I915_WRITE(SDEW, 0x15040d00);
		6063	I915_WRITE(CSIEW0, 0x007f0000);
		6064	I915_WRITE(CSIEW1, 0x1e220004);
		6065	I915_WRITE(CSIEW2, 0x04000004);
		6066
		6067	for (i = 0; i < 5; i++)
6084	serge	6068	I915_WRITE(PEW(i), 0);
3031	serge	6069	for (i = 0; i < 3; i++)
6084	serge	6070	I915_WRITE(DEW(i), 0);
3031	serge	6071
		6072	/* Program P-state weights to account for frequency power adjustment */
		6073	for (i = 0; i < 16; i++) {
6084	serge	6074	u32 pxvidfreq = I915_READ(PXVFREQ(i));
3031	serge	6075	unsigned long freq = intel_pxfreq(pxvidfreq);
		6076	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		6077	PXVFREQ_PX_SHIFT;
		6078	unsigned long val;
		6079
		6080	val = vid * vid;
		6081	val *= (freq / 1000);
		6082	val *= 255;
		6083	val /= (127127900);
		6084	if (val > 0xff)
		6085	DRM_ERROR("bad pxval: %ld\n", val);
		6086	pxw[i] = val;
		6087	}
		6088	/* Render standby states get 0 weight */
		6089	pxw[14] = 0;
		6090	pxw[15] = 0;
		6091
		6092	for (i = 0; i < 4; i++) {
		6093	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		6094	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
6084	serge	6095	I915_WRITE(PXW(i), val);
3031	serge	6096	}
		6097
		6098	/* Adjust magic regs to magic values (more experimental results) */
		6099	I915_WRITE(OGW0, 0);
		6100	I915_WRITE(OGW1, 0);
		6101	I915_WRITE(EG0, 0x00007f00);
		6102	I915_WRITE(EG1, 0x0000000e);
		6103	I915_WRITE(EG2, 0x000e0000);
		6104	I915_WRITE(EG3, 0x68000300);
		6105	I915_WRITE(EG4, 0x42000000);
		6106	I915_WRITE(EG5, 0x00140031);
		6107	I915_WRITE(EG6, 0);
		6108	I915_WRITE(EG7, 0);
		6109
		6110	for (i = 0; i < 8; i++)
6084	serge	6111	I915_WRITE(PXWL(i), 0);
3031	serge	6112
		6113	/* Enable PMON + select events */
		6114	I915_WRITE(ECR, 0x80000019);
		6115
		6116	lcfuse = I915_READ(LCFUSE02);
		6117
		6118	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		6119	}
		6120
5060	serge	6121	void intel_init_gt_powersave(struct drm_device *dev)
		6122	{
		6123	i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
		6124
		6125	if (IS_CHERRYVIEW(dev))
		6126	cherryview_init_gt_powersave(dev);
		6127	else if (IS_VALLEYVIEW(dev))
		6128	valleyview_init_gt_powersave(dev);
		6129	}
		6130
		6131	void intel_cleanup_gt_powersave(struct drm_device *dev)
		6132	{
		6133	if (IS_CHERRYVIEW(dev))
		6134	return;
		6135	else if (IS_VALLEYVIEW(dev))
		6136	valleyview_cleanup_gt_powersave(dev);
		6137	}
		6138
5354	serge	6139	static void gen6_suspend_rps(struct drm_device *dev)
		6140	{
		6141	struct drm_i915_private *dev_priv = dev->dev_private;
		6142
		6143	// flush_delayed_work(&dev_priv->rps.delayed_resume_work);
		6144
6084	serge	6145	gen6_disable_rps_interrupts(dev);
5354	serge	6146	}
		6147
5060	serge	6148	/**
		6149	* intel_suspend_gt_powersave - suspend PM work and helper threads
		6150	* @dev: drm device
		6151	*
		6152	* We don't want to disable RC6 or other features here, we just want
		6153	* to make sure any work we've queued has finished and won't bother
		6154	* us while we're suspended.
		6155	*/
		6156	void intel_suspend_gt_powersave(struct drm_device *dev)
		6157	{
		6158	struct drm_i915_private *dev_priv = dev->dev_private;
		6159
5354	serge	6160	if (INTEL_INFO(dev)->gen < 6)
		6161	return;
5060	serge	6162
5354	serge	6163	gen6_suspend_rps(dev);
5060	serge	6164
		6165	/* Force GPU to min freq during suspend */
		6166	gen6_rps_idle(dev_priv);
		6167	}
		6168
3031	serge	6169	void intel_disable_gt_powersave(struct drm_device *dev)
		6170	{
3243	Serge	6171	struct drm_i915_private *dev_priv = dev->dev_private;
		6172
3031	serge	6173	if (IS_IRONLAKE_M(dev)) {
		6174	ironlake_disable_drps(dev);
4293	Serge	6175	} else if (INTEL_INFO(dev)->gen >= 6) {
5060	serge	6176	intel_suspend_gt_powersave(dev);
		6177
3482	Serge	6178	mutex_lock(&dev_priv->rps.hw_lock);
5354	serge	6179	if (INTEL_INFO(dev)->gen >= 9)
		6180	gen9_disable_rps(dev);
		6181	else if (IS_CHERRYVIEW(dev))
5060	serge	6182	cherryview_disable_rps(dev);
		6183	else if (IS_VALLEYVIEW(dev))
4104	Serge	6184	valleyview_disable_rps(dev);
		6185	else
6084	serge	6186	gen6_disable_rps(dev);
5354	serge	6187
4560	Serge	6188	dev_priv->rps.enabled = false;
3480	Serge	6189	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	6190	}
		6191	}
		6192
3482	Serge	6193	static void intel_gen6_powersave_work(struct work_struct *work)
		6194	{
		6195	struct drm_i915_private *dev_priv =
		6196	container_of(work, struct drm_i915_private,
		6197	rps.delayed_resume_work.work);
		6198	struct drm_device *dev = dev_priv->dev;
		6199
		6200	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	6201
6084	serge	6202	gen6_reset_rps_interrupts(dev);
5354	serge	6203
5060	serge	6204	if (IS_CHERRYVIEW(dev)) {
		6205	cherryview_enable_rps(dev);
		6206	} else if (IS_VALLEYVIEW(dev)) {
4104	Serge	6207	valleyview_enable_rps(dev);
5354	serge	6208	} else if (INTEL_INFO(dev)->gen >= 9) {
6084	serge	6209	gen9_enable_rc6(dev);
5354	serge	6210	gen9_enable_rps(dev);
6084	serge	6211	if (IS_SKYLAKE(dev))
		6212	__gen6_update_ring_freq(dev);
4560	Serge	6213	} else if (IS_BROADWELL(dev)) {
		6214	gen8_enable_rps(dev);
5060	serge	6215	__gen6_update_ring_freq(dev);
4104	Serge	6216	} else {
6084	serge	6217	gen6_enable_rps(dev);
5060	serge	6218	__gen6_update_ring_freq(dev);
4104	Serge	6219	}
6084	serge	6220
		6221	WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
		6222	WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq);
		6223
		6224	WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq);
		6225	WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq);
		6226
4560	Serge	6227	dev_priv->rps.enabled = true;
5354	serge	6228
6084	serge	6229	gen6_enable_rps_interrupts(dev);
5354	serge	6230
3482	Serge	6231	mutex_unlock(&dev_priv->rps.hw_lock);
5060	serge	6232
		6233	intel_runtime_pm_put(dev_priv);
3482	Serge	6234	}
		6235
3031	serge	6236	void intel_enable_gt_powersave(struct drm_device *dev)
		6237	{
3243	Serge	6238	struct drm_i915_private *dev_priv = dev->dev_private;
		6239
6084	serge	6240	/* Powersaving is controlled by the host when inside a VM */
		6241	if (intel_vgpu_active(dev))
		6242	return;
		6243
3031	serge	6244	if (IS_IRONLAKE_M(dev)) {
5060	serge	6245	mutex_lock(&dev->struct_mutex);
3031	serge	6246	ironlake_enable_drps(dev);
		6247	intel_init_emon(dev);
5060	serge	6248	mutex_unlock(&dev->struct_mutex);
		6249	} else if (INTEL_INFO(dev)->gen >= 6) {
3243	Serge	6250	/*
		6251	* PCU communication is slow and this doesn't need to be
		6252	* done at any specific time, so do this out of our fast path
		6253	* to make resume and init faster.
5060	serge	6254	*
		6255	* We depend on the HW RC6 power context save/restore
		6256	* mechanism when entering D3 through runtime PM suspend. So
		6257	* disable RPM until RPS/RC6 is properly setup. We can only
		6258	* get here via the driver load/system resume/runtime resume
		6259	* paths, so the _noresume version is enough (and in case of
		6260	* runtime resume it's necessary).
3243	Serge	6261	*/
5060	serge	6262	if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		6263	round_jiffies_up_relative(HZ)))
		6264	intel_runtime_pm_get_noresume(dev_priv);
3031	serge	6265	}
		6266	}
		6267
5060	serge	6268	void intel_reset_gt_powersave(struct drm_device *dev)
		6269	{
		6270	struct drm_i915_private *dev_priv = dev->dev_private;
		6271
5354	serge	6272	if (INTEL_INFO(dev)->gen < 6)
		6273	return;
		6274
		6275	gen6_suspend_rps(dev);
5060	serge	6276	dev_priv->rps.enabled = false;
		6277	}
		6278
3243	Serge	6279	static void ibx_init_clock_gating(struct drm_device *dev)
		6280	{
		6281	struct drm_i915_private *dev_priv = dev->dev_private;
		6282
		6283	/*
		6284	* On Ibex Peak and Cougar Point, we need to disable clock
		6285	* gating for the panel power sequencer or it will fail to
		6286	* start up when no ports are active.
		6287	*/
		6288	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		6289	}
		6290
4104	Serge	6291	static void g4x_disable_trickle_feed(struct drm_device *dev)
		6292	{
		6293	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	6294	enum pipe pipe;
4104	Serge	6295
5354	serge	6296	for_each_pipe(dev_priv, pipe) {
4104	Serge	6297	I915_WRITE(DSPCNTR(pipe),
		6298	I915_READ(DSPCNTR(pipe)) \|
		6299	DISPPLANE_TRICKLE_FEED_DISABLE);
6084	serge	6300
		6301	I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
		6302	POSTING_READ(DSPSURF(pipe));
4104	Serge	6303	}
		6304	}
		6305
4560	Serge	6306	static void ilk_init_lp_watermarks(struct drm_device *dev)
		6307	{
		6308	struct drm_i915_private *dev_priv = dev->dev_private;
		6309
		6310	I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
		6311	I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
		6312	I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
		6313
		6314	/*
		6315	* Don't touch WM1S_LP_EN here.
		6316	* Doing so could cause underruns.
		6317	*/
		6318	}
		6319
3031	serge	6320	static void ironlake_init_clock_gating(struct drm_device *dev)
		6321	{
		6322	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6323	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6324
4104	Serge	6325	/*
		6326	* Required for FBC
		6327	* WaFbcDisableDpfcClockGating:ilk
		6328	*/
3243	Serge	6329	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		6330	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		6331	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	6332
		6333	I915_WRITE(PCH_3DCGDIS0,
		6334	MARIUNIT_CLOCK_GATE_DISABLE \|
		6335	SVSMUNIT_CLOCK_GATE_DISABLE);
		6336	I915_WRITE(PCH_3DCGDIS1,
		6337	VFMUNIT_CLOCK_GATE_DISABLE);
		6338
		6339	/*
		6340	* According to the spec the following bits should be set in
		6341	* order to enable memory self-refresh
		6342	* The bit 22/21 of 0x42004
		6343	* The bit 5 of 0x42020
		6344	* The bit 15 of 0x45000
		6345	*/
		6346	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6347	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6348	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	6349	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	6350	I915_WRITE(DISP_ARB_CTL,
		6351	(I915_READ(DISP_ARB_CTL) \|
		6352	DISP_FBC_WM_DIS));
		6353
4560	Serge	6354	ilk_init_lp_watermarks(dev);
		6355
3031	serge	6356	/*
		6357	* Based on the document from hardware guys the following bits
		6358	* should be set unconditionally in order to enable FBC.
		6359	* The bit 22 of 0x42000
		6360	* The bit 22 of 0x42004
		6361	* The bit 7,8,9 of 0x42020.
		6362	*/
		6363	if (IS_IRONLAKE_M(dev)) {
4104	Serge	6364	/* WaFbcAsynchFlipDisableFbcQueue:ilk */
3031	serge	6365	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6366	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6367	ILK_FBCQ_DIS);
		6368	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6369	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6370	ILK_DPARB_GATE);
		6371	}
		6372
3243	Serge	6373	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		6374
3031	serge	6375	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6376	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6377	ILK_ELPIN_409_SELECT);
		6378	I915_WRITE(_3D_CHICKEN2,
		6379	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		6380	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	6381
4104	Serge	6382	/* WaDisableRenderCachePipelinedFlush:ilk */
3243	Serge	6383	I915_WRITE(CACHE_MODE_0,
		6384	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		6385
5060	serge	6386	/* WaDisable_RenderCache_OperationalFlush:ilk */
		6387	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6388
4104	Serge	6389	g4x_disable_trickle_feed(dev);
		6390
3243	Serge	6391	ibx_init_clock_gating(dev);
3031	serge	6392	}
		6393
3243	Serge	6394	static void cpt_init_clock_gating(struct drm_device *dev)
		6395	{
		6396	struct drm_i915_private *dev_priv = dev->dev_private;
		6397	int pipe;
3746	Serge	6398	uint32_t val;
3243	Serge	6399
		6400	/*
		6401	* On Ibex Peak and Cougar Point, we need to disable clock
		6402	* gating for the panel power sequencer or it will fail to
		6403	* start up when no ports are active.
		6404	*/
4280	Serge	6405	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE \|
		6406	PCH_DPLUNIT_CLOCK_GATE_DISABLE \|
		6407	PCH_CPUNIT_CLOCK_GATE_DISABLE);
3243	Serge	6408	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		6409	DPLS_EDP_PPS_FIX_DIS);
		6410	/* The below fixes the weird display corruption, a few pixels shifted
		6411	* downward, on (only) LVDS of some HP laptops with IVY.
		6412	*/
5354	serge	6413	for_each_pipe(dev_priv, pipe) {
3746	Serge	6414	val = I915_READ(TRANS_CHICKEN2(pipe));
		6415	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		6416	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4104	Serge	6417	if (dev_priv->vbt.fdi_rx_polarity_inverted)
3746	Serge	6418	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		6419	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		6420	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		6421	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		6422	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		6423	}
3243	Serge	6424	/* WADP0ClockGatingDisable */
5354	serge	6425	for_each_pipe(dev_priv, pipe) {
3243	Serge	6426	I915_WRITE(TRANS_CHICKEN1(pipe),
		6427	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		6428	}
		6429	}
		6430
3480	Serge	6431	static void gen6_check_mch_setup(struct drm_device *dev)
		6432	{
		6433	struct drm_i915_private *dev_priv = dev->dev_private;
		6434	uint32_t tmp;
		6435
		6436	tmp = I915_READ(MCH_SSKPD);
5060	serge	6437	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
		6438	DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
		6439	tmp);
3480	Serge	6440	}
		6441
3031	serge	6442	static void gen6_init_clock_gating(struct drm_device *dev)
		6443	{
		6444	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6445	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6446
3243	Serge	6447	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	6448
		6449	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6450	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6451	ILK_ELPIN_409_SELECT);
		6452
4104	Serge	6453	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3243	Serge	6454	I915_WRITE(_3D_CHICKEN,
		6455	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		6456
5060	serge	6457	/* WaDisable_RenderCache_OperationalFlush:snb */
		6458	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6459
		6460	/*
		6461	* BSpec recoomends 8x4 when MSAA is used,
		6462	* however in practice 16x4 seems fastest.
		6463	*
		6464	* Note that PS/WM thread counts depend on the WIZ hashing
		6465	* disable bit, which we don't touch here, but it's good
		6466	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6467	*/
		6468	I915_WRITE(GEN6_GT_MODE,
5354	serge	6469	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6470
4560	Serge	6471	ilk_init_lp_watermarks(dev);
3031	serge	6472
		6473	I915_WRITE(CACHE_MODE_0,
		6474	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		6475
		6476	I915_WRITE(GEN6_UCGCTL1,
		6477	I915_READ(GEN6_UCGCTL1) \|
		6478	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		6479	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6480
		6481	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		6482	* gating disable must be set. Failure to set it results in
		6483	* flickering pixels due to Z write ordering failures after
		6484	* some amount of runtime in the Mesa "fire" demo, and Unigine
		6485	* Sanctuary and Tropics, and apparently anything else with
		6486	* alpha test or pixel discard.
		6487	*
		6488	* According to the spec, bit 11 (RCCUNIT) must also be set,
		6489	* but we didn't debug actual testcases to find it out.
		6490	*
5060	serge	6491	* WaDisableRCCUnitClockGating:snb
		6492	* WaDisableRCPBUnitClockGating:snb
3031	serge	6493	*/
		6494	I915_WRITE(GEN6_UCGCTL2,
		6495	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		6496	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		6497
5060	serge	6498	/* WaStripsFansDisableFastClipPerformanceFix:snb */
		6499	I915_WRITE(_3D_CHICKEN3,
		6500	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
3031	serge	6501
		6502	/*
5060	serge	6503	* Bspec says:
		6504	* "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
		6505	* 3DSTATE_SF number of SF output attributes is more than 16."
		6506	*/
		6507	I915_WRITE(_3D_CHICKEN3,
		6508	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
		6509
		6510	/*
3031	serge	6511	* According to the spec the following bits should be
		6512	* set in order to enable memory self-refresh and fbc:
		6513	* The bit21 and bit22 of 0x42000
		6514	* The bit21 and bit22 of 0x42004
		6515	* The bit5 and bit7 of 0x42020
		6516	* The bit14 of 0x70180
		6517	* The bit14 of 0x71180
4104	Serge	6518	*
		6519	* WaFbcAsynchFlipDisableFbcQueue:snb
3031	serge	6520	*/
		6521	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6522	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6523	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		6524	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6525	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6526	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	6527	I915_WRITE(ILK_DSPCLK_GATE_D,
		6528	I915_READ(ILK_DSPCLK_GATE_D) \|
		6529	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		6530	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	6531
4104	Serge	6532	g4x_disable_trickle_feed(dev);
3031	serge	6533
3243	Serge	6534	cpt_init_clock_gating(dev);
3480	Serge	6535
		6536	gen6_check_mch_setup(dev);
3031	serge	6537	}
		6538
		6539	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		6540	{
		6541	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		6542
5060	serge	6543	/*
		6544	* WaVSThreadDispatchOverride:ivb,vlv
		6545	*
		6546	* This actually overrides the dispatch
		6547	* mode for all thread types.
		6548	*/
3031	serge	6549	reg &= ~GEN7_FF_SCHED_MASK;
		6550	reg \|= GEN7_FF_TS_SCHED_HW;
		6551	reg \|= GEN7_FF_VS_SCHED_HW;
		6552	reg \|= GEN7_FF_DS_SCHED_HW;
		6553
		6554	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		6555	}
		6556
3243	Serge	6557	static void lpt_init_clock_gating(struct drm_device *dev)
		6558	{
		6559	struct drm_i915_private *dev_priv = dev->dev_private;
		6560
		6561	/*
		6562	* TODO: this bit should only be enabled when really needed, then
		6563	* disabled when not needed anymore in order to save power.
		6564	*/
6084	serge	6565	if (HAS_PCH_LPT_LP(dev))
3243	Serge	6566	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		6567	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		6568	PCH_LP_PARTITION_LEVEL_DISABLE);
4104	Serge	6569
		6570	/* WADPOClockGatingDisable:hsw */
6084	serge	6571	I915_WRITE(TRANS_CHICKEN1(PIPE_A),
		6572	I915_READ(TRANS_CHICKEN1(PIPE_A)) \|
4104	Serge	6573	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3243	Serge	6574	}
		6575
4104	Serge	6576	static void lpt_suspend_hw(struct drm_device *dev)
		6577	{
		6578	struct drm_i915_private *dev_priv = dev->dev_private;
		6579
6084	serge	6580	if (HAS_PCH_LPT_LP(dev)) {
4104	Serge	6581	uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
		6582
		6583	val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		6584	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
		6585	}
		6586	}
		6587
5354	serge	6588	static void broadwell_init_clock_gating(struct drm_device *dev)
3031	serge	6589	{
		6590	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	6591	enum pipe pipe;
6084	serge	6592	uint32_t misccpctl;
3031	serge	6593
6084	serge	6594	ilk_init_lp_watermarks(dev);
3031	serge	6595
4560	Serge	6596	/* WaSwitchSolVfFArbitrationPriority:bdw */
		6597	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6598
		6599	/* WaPsrDPAMaskVBlankInSRD:bdw */
		6600	I915_WRITE(CHICKEN_PAR1_1,
		6601	I915_READ(CHICKEN_PAR1_1) \| DPA_MASK_VBLANK_SRD);
		6602
		6603	/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
5354	serge	6604	for_each_pipe(dev_priv, pipe) {
5060	serge	6605	I915_WRITE(CHICKEN_PIPESL_1(pipe),
		6606	I915_READ(CHICKEN_PIPESL_1(pipe)) \|
		6607	BDW_DPRS_MASK_VBLANK_SRD);
4560	Serge	6608	}
		6609
		6610	/* WaVSRefCountFullforceMissDisable:bdw */
		6611	/* WaDSRefCountFullforceMissDisable:bdw */
		6612	I915_WRITE(GEN7_FF_THREAD_MODE,
		6613	I915_READ(GEN7_FF_THREAD_MODE) &
		6614	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
5060	serge	6615
		6616	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6617	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6618
		6619	/* WaDisableSDEUnitClockGating:bdw */
		6620	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6621	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		6622
6084	serge	6623	/*
		6624	* WaProgramL3SqcReg1Default:bdw
		6625	* WaTempDisableDOPClkGating:bdw
		6626	*/
		6627	misccpctl = I915_READ(GEN7_MISCCPCTL);
		6628	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		6629	I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT);
6660	serge	6630	/*
		6631	* Wait at least 100 clocks before re-enabling clock gating. See
		6632	* the definition of L3SQCREG1 in BSpec.
		6633	*/
		6634	POSTING_READ(GEN8_L3SQCREG1);
		6635	udelay(1);
6084	serge	6636	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		6637
		6638	/*
		6639	* WaGttCachingOffByDefault:bdw
		6640	* GTT cache may not work with big pages, so if those
		6641	* are ever enabled GTT cache may need to be disabled.
		6642	*/
		6643	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
		6644
5354	serge	6645	lpt_init_clock_gating(dev);
4560	Serge	6646	}
		6647
		6648	static void haswell_init_clock_gating(struct drm_device *dev)
		6649	{
		6650	struct drm_i915_private *dev_priv = dev->dev_private;
		6651
		6652	ilk_init_lp_watermarks(dev);
		6653
4104	Serge	6654	/* L3 caching of data atomics doesn't work -- disable it. */
		6655	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
		6656	I915_WRITE(HSW_ROW_CHICKEN3,
		6657	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
		6658
		6659	/* This is required by WaCatErrorRejectionIssue:hsw */
3031	serge	6660	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6661	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6662	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6663
4104	Serge	6664	/* WaVSRefCountFullforceMissDisable:hsw */
5060	serge	6665	I915_WRITE(GEN7_FF_THREAD_MODE,
		6666	I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
3031	serge	6667
5060	serge	6668	/* WaDisable_RenderCache_OperationalFlush:hsw */
		6669	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6670
		6671	/* enable HiZ Raw Stall Optimization */
		6672	I915_WRITE(CACHE_MODE_0_GEN7,
		6673	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6674
4104	Serge	6675	/* WaDisable4x2SubspanOptimization:hsw */
3031	serge	6676	I915_WRITE(CACHE_MODE_1,
		6677	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6678
5060	serge	6679	/*
		6680	* BSpec recommends 8x4 when MSAA is used,
		6681	* however in practice 16x4 seems fastest.
		6682	*
		6683	* Note that PS/WM thread counts depend on the WIZ hashing
		6684	* disable bit, which we don't touch here, but it's good
		6685	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6686	*/
		6687	I915_WRITE(GEN7_GT_MODE,
5354	serge	6688	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6689
6084	serge	6690	/* WaSampleCChickenBitEnable:hsw */
		6691	I915_WRITE(HALF_SLICE_CHICKEN3,
		6692	_MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
		6693
4104	Serge	6694	/* WaSwitchSolVfFArbitrationPriority:hsw */
3746	Serge	6695	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6696
4104	Serge	6697	/* WaRsPkgCStateDisplayPMReq:hsw */
		6698	I915_WRITE(CHICKEN_PAR1_1,
		6699	I915_READ(CHICKEN_PAR1_1) \| FORCE_ARB_IDLE_PLANES);
3031	serge	6700
3243	Serge	6701	lpt_init_clock_gating(dev);
3031	serge	6702	}
		6703
		6704	static void ivybridge_init_clock_gating(struct drm_device *dev)
		6705	{
		6706	struct drm_i915_private *dev_priv = dev->dev_private;
		6707	uint32_t snpcr;
		6708
4560	Serge	6709	ilk_init_lp_watermarks(dev);
3031	serge	6710
3243	Serge	6711	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	6712
4104	Serge	6713	/* WaDisableEarlyCull:ivb */
3243	Serge	6714	I915_WRITE(_3D_CHICKEN3,
		6715	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6716
4104	Serge	6717	/* WaDisableBackToBackFlipFix:ivb */
3031	serge	6718	I915_WRITE(IVB_CHICKEN3,
		6719	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6720	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6721
4104	Serge	6722	/* WaDisablePSDDualDispatchEnable:ivb */
3243	Serge	6723	if (IS_IVB_GT1(dev))
		6724	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		6725	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		6726
5060	serge	6727	/* WaDisable_RenderCache_OperationalFlush:ivb */
		6728	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6729
4104	Serge	6730	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
3031	serge	6731	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		6732	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		6733
4104	Serge	6734	/* WaApplyL3ControlAndL3ChickenMode:ivb */
3031	serge	6735	I915_WRITE(GEN7_L3CNTLREG1,
		6736	GEN7_WA_FOR_GEN7_L3_CONTROL);
		6737	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
6084	serge	6738	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	6739	if (IS_IVB_GT1(dev))
		6740	I915_WRITE(GEN7_ROW_CHICKEN2,
		6741	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6742	else {
		6743	/* must write both registers */
		6744	I915_WRITE(GEN7_ROW_CHICKEN2,
		6745	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3243	Serge	6746	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		6747	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6748	}
3031	serge	6749
4104	Serge	6750	/* WaForceL3Serialization:ivb */
3243	Serge	6751	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6752	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6753
5060	serge	6754	/*
3031	serge	6755	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6756	* This implements the WaDisableRCZUnitClockGating:ivb workaround.
3031	serge	6757	*/
		6758	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6759	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6760
4104	Serge	6761	/* This is required by WaCatErrorRejectionIssue:ivb */
3031	serge	6762	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6763	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6764	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6765
4104	Serge	6766	g4x_disable_trickle_feed(dev);
3031	serge	6767
		6768	gen7_setup_fixed_func_scheduler(dev_priv);
		6769
5060	serge	6770	if (0) { /* causes HiZ corruption on ivb:gt1 */
		6771	/* enable HiZ Raw Stall Optimization */
		6772	I915_WRITE(CACHE_MODE_0_GEN7,
		6773	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6774	}
		6775
4104	Serge	6776	/* WaDisable4x2SubspanOptimization:ivb */
3031	serge	6777	I915_WRITE(CACHE_MODE_1,
		6778	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6779
5060	serge	6780	/*
		6781	* BSpec recommends 8x4 when MSAA is used,
		6782	* however in practice 16x4 seems fastest.
		6783	*
		6784	* Note that PS/WM thread counts depend on the WIZ hashing
		6785	* disable bit, which we don't touch here, but it's good
		6786	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6787	*/
		6788	I915_WRITE(GEN7_GT_MODE,
5354	serge	6789	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6790
3031	serge	6791	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		6792	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		6793	snpcr \|= GEN6_MBC_SNPCR_MED;
		6794	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	6795
3746	Serge	6796	if (!HAS_PCH_NOP(dev))
6084	serge	6797	cpt_init_clock_gating(dev);
3480	Serge	6798
		6799	gen6_check_mch_setup(dev);
3031	serge	6800	}
		6801
6084	serge	6802	static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
		6803	{
		6804	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
		6805
		6806	/*
		6807	* Disable trickle feed and enable pnd deadline calculation
		6808	*/
		6809	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
		6810	I915_WRITE(CBR1_VLV, 0);
		6811	}
		6812
3031	serge	6813	static void valleyview_init_clock_gating(struct drm_device *dev)
		6814	{
		6815	struct drm_i915_private *dev_priv = dev->dev_private;
		6816
6084	serge	6817	vlv_init_display_clock_gating(dev_priv);
3031	serge	6818
4104	Serge	6819	/* WaDisableEarlyCull:vlv */
3243	Serge	6820	I915_WRITE(_3D_CHICKEN3,
		6821	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6822
4104	Serge	6823	/* WaDisableBackToBackFlipFix:vlv */
3031	serge	6824	I915_WRITE(IVB_CHICKEN3,
		6825	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6826	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6827
5060	serge	6828	/* WaPsdDispatchEnable:vlv */
4104	Serge	6829	/* WaDisablePSDDualDispatchEnable:vlv */
3243	Serge	6830	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	6831	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		6832	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	6833
5060	serge	6834	/* WaDisable_RenderCache_OperationalFlush:vlv */
		6835	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6836
4104	Serge	6837	/* WaForceL3Serialization:vlv */
3243	Serge	6838	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6839	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6840
4104	Serge	6841	/* WaDisableDopClockGating:vlv */
3243	Serge	6842	I915_WRITE(GEN7_ROW_CHICKEN2,
		6843	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		6844
4104	Serge	6845	/* This is required by WaCatErrorRejectionIssue:vlv */
3031	serge	6846	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6847	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6848	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6849
5060	serge	6850	gen7_setup_fixed_func_scheduler(dev_priv);
		6851
		6852	/*
3031	serge	6853	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6854	* This implements the WaDisableRCZUnitClockGating:vlv workaround.
3031	serge	6855	*/
		6856	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6857	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6858
5060	serge	6859	/* WaDisableL3Bank2xClockGate:vlv
		6860	* Disabling L3 clock gating- MMIO 940c[25] = 1
		6861	* Set bit 25, to disable L3_BANK_2x_CLK_GATING */
		6862	I915_WRITE(GEN7_UCGCTL4,
		6863	I915_READ(GEN7_UCGCTL4) \| GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
3031	serge	6864
5060	serge	6865	/*
		6866	* BSpec says this must be set, even though
		6867	* WaDisable4x2SubspanOptimization isn't listed for VLV.
		6868	*/
3031	serge	6869	I915_WRITE(CACHE_MODE_1,
		6870	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6871
		6872	/*
6084	serge	6873	* BSpec recommends 8x4 when MSAA is used,
		6874	* however in practice 16x4 seems fastest.
		6875	*
		6876	* Note that PS/WM thread counts depend on the WIZ hashing
		6877	* disable bit, which we don't touch here, but it's good
		6878	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6879	*/
		6880	I915_WRITE(GEN7_GT_MODE,
		6881	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
		6882
		6883	/*
5060	serge	6884	* WaIncreaseL3CreditsForVLVB0:vlv
		6885	* This is the hardware default actually.
		6886	*/
		6887	I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
		6888
		6889	/*
4104	Serge	6890	* WaDisableVLVClockGating_VBIIssue:vlv
3243	Serge	6891	* Disable clock gating on th GCFG unit to prevent a delay
		6892	* in the reporting of vblank events.
		6893	*/
5060	serge	6894	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
		6895	}
3746	Serge	6896
5060	serge	6897	static void cherryview_init_clock_gating(struct drm_device *dev)
		6898	{
		6899	struct drm_i915_private *dev_priv = dev->dev_private;
		6900
6084	serge	6901	vlv_init_display_clock_gating(dev_priv);
5060	serge	6902
		6903	/* WaVSRefCountFullforceMissDisable:chv */
		6904	/* WaDSRefCountFullforceMissDisable:chv */
		6905	I915_WRITE(GEN7_FF_THREAD_MODE,
		6906	I915_READ(GEN7_FF_THREAD_MODE) &
		6907	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
		6908
		6909	/* WaDisableSemaphoreAndSyncFlipWait:chv */
		6910	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6911	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6912
		6913	/* WaDisableCSUnitClockGating:chv */
		6914	I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) \|
		6915	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6916
		6917	/* WaDisableSDEUnitClockGating:chv */
		6918	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6919	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6084	serge	6920
		6921	/*
		6922	* GTT cache may not work with big pages, so if those
		6923	* are ever enabled GTT cache may need to be disabled.
		6924	*/
		6925	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
3031	serge	6926	}
		6927
		6928	static void g4x_init_clock_gating(struct drm_device *dev)
		6929	{
		6930	struct drm_i915_private *dev_priv = dev->dev_private;
		6931	uint32_t dspclk_gate;
		6932
		6933	I915_WRITE(RENCLK_GATE_D1, 0);
		6934	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		6935	GS_UNIT_CLOCK_GATE_DISABLE \|
		6936	CL_UNIT_CLOCK_GATE_DISABLE);
		6937	I915_WRITE(RAMCLK_GATE_D, 0);
		6938	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		6939	OVRUNIT_CLOCK_GATE_DISABLE \|
		6940	OVCUNIT_CLOCK_GATE_DISABLE;
		6941	if (IS_GM45(dev))
		6942	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		6943	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	6944
		6945	/* WaDisableRenderCachePipelinedFlush */
		6946	I915_WRITE(CACHE_MODE_0,
		6947	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4104	Serge	6948
5060	serge	6949	/* WaDisable_RenderCache_OperationalFlush:g4x */
		6950	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6951
4104	Serge	6952	g4x_disable_trickle_feed(dev);
3031	serge	6953	}
		6954
		6955	static void crestline_init_clock_gating(struct drm_device *dev)
		6956	{
		6957	struct drm_i915_private *dev_priv = dev->dev_private;
		6958
		6959	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		6960	I915_WRITE(RENCLK_GATE_D2, 0);
		6961	I915_WRITE(DSPCLK_GATE_D, 0);
		6962	I915_WRITE(RAMCLK_GATE_D, 0);
		6963	I915_WRITE16(DEUC, 0);
4104	Serge	6964	I915_WRITE(MI_ARB_STATE,
		6965	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	6966
		6967	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		6968	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6969	}
		6970
		6971	static void broadwater_init_clock_gating(struct drm_device *dev)
		6972	{
		6973	struct drm_i915_private *dev_priv = dev->dev_private;
		6974
		6975	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		6976	I965_RCC_CLOCK_GATE_DISABLE \|
		6977	I965_RCPB_CLOCK_GATE_DISABLE \|
		6978	I965_ISC_CLOCK_GATE_DISABLE \|
		6979	I965_FBC_CLOCK_GATE_DISABLE);
		6980	I915_WRITE(RENCLK_GATE_D2, 0);
4104	Serge	6981	I915_WRITE(MI_ARB_STATE,
		6982	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	6983
		6984	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		6985	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6986	}
		6987
		6988	static void gen3_init_clock_gating(struct drm_device *dev)
		6989	{
		6990	struct drm_i915_private *dev_priv = dev->dev_private;
		6991	u32 dstate = I915_READ(D_STATE);
		6992
		6993	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		6994	DSTATE_DOT_CLOCK_GATING;
		6995	I915_WRITE(D_STATE, dstate);
		6996
		6997	if (IS_PINEVIEW(dev))
		6998	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		6999
		7000	/* IIR "flip pending" means done if this bit is set */
		7001	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
5060	serge	7002
		7003	/* interrupts should cause a wake up from C3 */
		7004	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
		7005
		7006	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		7007	I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
5354	serge	7008
		7009	I915_WRITE(MI_ARB_STATE,
		7010	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	7011	}
		7012
		7013	static void i85x_init_clock_gating(struct drm_device *dev)
		7014	{
		7015	struct drm_i915_private *dev_priv = dev->dev_private;
		7016
		7017	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
5060	serge	7018
		7019	/* interrupts should cause a wake up from C3 */
		7020	I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) \|
		7021	_MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
5354	serge	7022
		7023	I915_WRITE(MEM_MODE,
		7024	_MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	7025	}
		7026
		7027	static void i830_init_clock_gating(struct drm_device *dev)
		7028	{
		7029	struct drm_i915_private *dev_priv = dev->dev_private;
		7030
		7031	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
5354	serge	7032
		7033	I915_WRITE(MEM_MODE,
		7034	_MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) \|
		7035	_MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
3031	serge	7036	}
		7037
		7038	void intel_init_clock_gating(struct drm_device *dev)
		7039	{
		7040	struct drm_i915_private *dev_priv = dev->dev_private;
		7041
6084	serge	7042	if (dev_priv->display.init_clock_gating)
		7043	dev_priv->display.init_clock_gating(dev);
3031	serge	7044	}
		7045
4104	Serge	7046	void intel_suspend_hw(struct drm_device *dev)
		7047	{
		7048	if (HAS_PCH_LPT(dev))
		7049	lpt_suspend_hw(dev);
		7050	}
		7051
5354	serge	7052	/* Set up chip specific power management-related functions */
		7053	void intel_init_pm(struct drm_device *dev)
		7054	{
		7055	struct drm_i915_private *dev_priv = dev->dev_private;
		7056
6084	serge	7057	intel_fbc_init(dev_priv);
5354	serge	7058
3031	serge	7059	/* For cxsr */
		7060	if (IS_PINEVIEW(dev))
		7061	i915_pineview_get_mem_freq(dev);
		7062	else if (IS_GEN5(dev))
		7063	i915_ironlake_get_mem_freq(dev);
		7064
		7065	/* For FIFO watermark updates */
5354	serge	7066	if (INTEL_INFO(dev)->gen >= 9) {
		7067	skl_setup_wm_latency(dev);
		7068
6084	serge	7069	if (IS_BROXTON(dev))
		7070	dev_priv->display.init_clock_gating =
		7071	bxt_init_clock_gating;
5354	serge	7072	dev_priv->display.update_wm = skl_update_wm;
		7073	dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
		7074	} else if (HAS_PCH_SPLIT(dev)) {
5060	serge	7075	ilk_setup_wm_latency(dev);
4104	Serge	7076
4560	Serge	7077	if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
		7078	dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) \|\|
		7079	(!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
		7080	dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
		7081	dev_priv->display.update_wm = ilk_update_wm;
		7082	dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
6084	serge	7083	} else {
		7084	DRM_DEBUG_KMS("Failed to read display plane latency. "
		7085	"Disable CxSR\n");
		7086	}
4560	Serge	7087
		7088	if (IS_GEN5(dev))
		7089	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		7090	else if (IS_GEN6(dev))
3031	serge	7091	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
4560	Serge	7092	else if (IS_IVYBRIDGE(dev))
3031	serge	7093	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
4560	Serge	7094	else if (IS_HASWELL(dev))
3031	serge	7095	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
4560	Serge	7096	else if (INTEL_INFO(dev)->gen == 8)
5354	serge	7097	dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
5060	serge	7098	} else if (IS_CHERRYVIEW(dev)) {
6084	serge	7099	vlv_setup_wm_latency(dev);
		7100
		7101	dev_priv->display.update_wm = vlv_update_wm;
5060	serge	7102	dev_priv->display.init_clock_gating =
		7103	cherryview_init_clock_gating;
3031	serge	7104	} else if (IS_VALLEYVIEW(dev)) {
6084	serge	7105	vlv_setup_wm_latency(dev);
		7106
		7107	dev_priv->display.update_wm = vlv_update_wm;
3031	serge	7108	dev_priv->display.init_clock_gating =
		7109	valleyview_init_clock_gating;
		7110	} else if (IS_PINEVIEW(dev)) {
		7111	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		7112	dev_priv->is_ddr3,
		7113	dev_priv->fsb_freq,
		7114	dev_priv->mem_freq)) {
		7115	DRM_INFO("failed to find known CxSR latency "
		7116	"(found ddr%s fsb freq %d, mem freq %d), "
		7117	"disabling CxSR\n",
		7118	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		7119	dev_priv->fsb_freq, dev_priv->mem_freq);
		7120	/* Disable CxSR and never update its watermark again */
5060	serge	7121	intel_set_memory_cxsr(dev_priv, false);
3031	serge	7122	dev_priv->display.update_wm = NULL;
		7123	} else
		7124	dev_priv->display.update_wm = pineview_update_wm;
		7125	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		7126	} else if (IS_G4X(dev)) {
		7127	dev_priv->display.update_wm = g4x_update_wm;
		7128	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		7129	} else if (IS_GEN4(dev)) {
		7130	dev_priv->display.update_wm = i965_update_wm;
		7131	if (IS_CRESTLINE(dev))
		7132	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		7133	else if (IS_BROADWATER(dev))
		7134	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		7135	} else if (IS_GEN3(dev)) {
		7136	dev_priv->display.update_wm = i9xx_update_wm;
		7137	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		7138	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
4560	Serge	7139	} else if (IS_GEN2(dev)) {
		7140	if (INTEL_INFO(dev)->num_pipes == 1) {
		7141	dev_priv->display.update_wm = i845_update_wm;
		7142	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		7143	} else {
		7144	dev_priv->display.update_wm = i9xx_update_wm;
6084	serge	7145	dev_priv->display.get_fifo_size = i830_get_fifo_size;
4560	Serge	7146	}
		7147
		7148	if (IS_I85X(dev) \|\| IS_I865G(dev))
6084	serge	7149	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
4560	Serge	7150	else
		7151	dev_priv->display.init_clock_gating = i830_init_clock_gating;
3031	serge	7152	} else {
4560	Serge	7153	DRM_ERROR("unexpected fall-through in intel_init_pm\n");
3031	serge	7154	}
		7155	}
		7156
5354	serge	7157	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u32 mbox, u32 val)
3243	Serge	7158	{
		7159	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	7160
3243	Serge	7161	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7162	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		7163	return -EAGAIN;
		7164	}
3031	serge	7165
3243	Serge	7166	I915_WRITE(GEN6_PCODE_DATA, *val);
5354	serge	7167	I915_WRITE(GEN6_PCODE_DATA1, 0);
3243	Serge	7168	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7169
		7170	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7171	500)) {
		7172	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		7173	return -ETIMEDOUT;
6084	serge	7174	}
3243	Serge	7175
		7176	*val = I915_READ(GEN6_PCODE_DATA);
		7177	I915_WRITE(GEN6_PCODE_DATA, 0);
		7178
		7179	return 0;
		7180	}
		7181
5354	serge	7182	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
3243	Serge	7183	{
		7184	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		7185
		7186	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7187	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		7188	return -EAGAIN;
6084	serge	7189	}
3243	Serge	7190
		7191	I915_WRITE(GEN6_PCODE_DATA, val);
		7192	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7193
		7194	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7195	500)) {
		7196	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		7197	return -ETIMEDOUT;
3031	serge	7198	}
3243	Serge	7199
		7200	I915_WRITE(GEN6_PCODE_DATA, 0);
		7201
		7202	return 0;
3031	serge	7203	}
3746	Serge	7204
5354	serge	7205	static int vlv_gpu_freq_div(unsigned int czclk_freq)
3746	Serge	7206	{
5354	serge	7207	switch (czclk_freq) {
		7208	case 200:
		7209	return 10;
		7210	case 267:
		7211	return 12;
		7212	case 320:
		7213	case 333:
		7214	return 16;
		7215	case 400:
		7216	return 20;
4104	Serge	7217	default:
		7218	return -1;
		7219	}
5354	serge	7220	}
3746	Serge	7221
5354	serge	7222	static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7223	{
6084	serge	7224	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5354	serge	7225
		7226	div = vlv_gpu_freq_div(czclk_freq);
		7227	if (div < 0)
		7228	return div;
		7229
		7230	return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
4104	Serge	7231	}
3746	Serge	7232
5060	serge	7233	static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
4104	Serge	7234	{
6084	serge	7235	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
3746	Serge	7236
5354	serge	7237	mul = vlv_gpu_freq_div(czclk_freq);
		7238	if (mul < 0)
		7239	return mul;
3746	Serge	7240
5354	serge	7241	return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
3746	Serge	7242	}
		7243
5060	serge	7244	static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7245	{
6084	serge	7246	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5060	serge	7247
5354	serge	7248	div = vlv_gpu_freq_div(czclk_freq) / 2;
		7249	if (div < 0)
		7250	return div;
5060	serge	7251
5354	serge	7252	return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
5060	serge	7253	}
		7254
		7255	static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7256	{
6084	serge	7257	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5060	serge	7258
5354	serge	7259	mul = vlv_gpu_freq_div(czclk_freq) / 2;
		7260	if (mul < 0)
		7261	return mul;
5060	serge	7262
5354	serge	7263	/* CHV needs even values */
		7264	return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
5060	serge	7265	}
		7266
6084	serge	7267	int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
5060	serge	7268	{
6084	serge	7269	if (IS_GEN9(dev_priv->dev))
		7270	return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
		7271	GEN9_FREQ_SCALER);
		7272	else if (IS_CHERRYVIEW(dev_priv->dev))
		7273	return chv_gpu_freq(dev_priv, val);
		7274	else if (IS_VALLEYVIEW(dev_priv->dev))
		7275	return byt_gpu_freq(dev_priv, val);
		7276	else
		7277	return val * GT_FREQUENCY_MULTIPLIER;
		7278	}
5060	serge	7279
6084	serge	7280	int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7281	{
		7282	if (IS_GEN9(dev_priv->dev))
		7283	return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
		7284	GT_FREQUENCY_MULTIPLIER);
		7285	else if (IS_CHERRYVIEW(dev_priv->dev))
		7286	return chv_freq_opcode(dev_priv, val);
5060	serge	7287	else if (IS_VALLEYVIEW(dev_priv->dev))
6084	serge	7288	return byt_freq_opcode(dev_priv, val);
		7289	else
		7290	return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
		7291	}
5060	serge	7292
6084	serge	7293	struct request_boost {
		7294	struct work_struct work;
		7295	struct drm_i915_gem_request *req;
		7296	};
		7297
		7298	static void __intel_rps_boost_work(struct work_struct *work)
		7299	{
		7300	struct request_boost *boost = container_of(work, struct request_boost, work);
		7301	struct drm_i915_gem_request *req = boost->req;
		7302
		7303	if (!i915_gem_request_completed(req, true))
		7304	gen6_rps_boost(to_i915(req->ring->dev), NULL,
		7305	req->emitted_jiffies);
		7306
		7307	i915_gem_request_unreference__unlocked(req);
		7308	kfree(boost);
5060	serge	7309	}
		7310
6084	serge	7311	void intel_queue_rps_boost_for_request(struct drm_device *dev,
		7312	struct drm_i915_gem_request *req)
5060	serge	7313	{
6084	serge	7314	struct request_boost *boost;
5060	serge	7315
6084	serge	7316	if (req == NULL \|\| INTEL_INFO(dev)->gen < 6)
		7317	return;
5060	serge	7318
6084	serge	7319	if (i915_gem_request_completed(req, true))
		7320	return;
		7321
		7322	boost = kmalloc(sizeof(*boost), GFP_ATOMIC);
		7323	if (boost == NULL)
		7324	return;
		7325
		7326	i915_gem_request_reference(req);
		7327	boost->req = req;
		7328
		7329	INIT_WORK(&boost->work, __intel_rps_boost_work);
		7330	queue_work(to_i915(dev)->wq, &boost->work);
5060	serge	7331	}
		7332
4560	Serge	7333	void intel_pm_setup(struct drm_device *dev)
3746	Serge	7334	{
4104	Serge	7335	struct drm_i915_private *dev_priv = dev->dev_private;
		7336
4560	Serge	7337	mutex_init(&dev_priv->rps.hw_lock);
6084	serge	7338	spin_lock_init(&dev_priv->rps.client_lock);
4560	Serge	7339
4104	Serge	7340	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		7341	intel_gen6_powersave_work);
6084	serge	7342	INIT_LIST_HEAD(&dev_priv->rps.clients);
		7343	INIT_LIST_HEAD(&dev_priv->rps.semaphores.link);
		7344	INIT_LIST_HEAD(&dev_priv->rps.mmioflips.link);
5060	serge	7345
		7346	dev_priv->pm.suspended = false;
3746	Serge	7347	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 6660