WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	//#include "../../../platform/x86/intel_ips.h"
		32	#include
		33
4560	Serge	34
3031	serge	35	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		36
		37	void getrawmonotonic(struct timespec *ts);
		38
4560	Serge	39	/**
		40	* RC6 is a special power stage which allows the GPU to enter an very
		41	* low-voltage mode when idle, using down to 0V while at this stage. This
		42	* stage is entered automatically when the GPU is idle when RC6 support is
		43	* enabled, and as soon as new workload arises GPU wakes up automatically as well.
		44	*
		45	* There are different RC6 modes available in Intel GPU, which differentiate
		46	* among each other with the latency required to enter and leave RC6 and
		47	* voltage consumed by the GPU in different states.
		48	*
		49	* The combination of the following flags define which states GPU is allowed
		50	* to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
		51	* RC6pp is deepest RC6. Their support by hardware varies according to the
		52	* GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
		53	* which brings the most power savings; deeper states save more power, but
		54	* require higher latency to switch to and wake up.
		55	*/
		56	#define INTEL_RC6_ENABLE (1<<0)
		57	#define INTEL_RC6p_ENABLE (1<<1)
		58	#define INTEL_RC6pp_ENABLE (1<<2)
		59
6084	serge	60	static void bxt_init_clock_gating(struct drm_device *dev)
5354	serge	61	{
		62	struct drm_i915_private *dev_priv = dev->dev_private;
		63
6084	serge	64	/* WaDisableSDEUnitClockGating:bxt */
5354	serge	65	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		66	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		67
		68	/*
6084	serge	69	* FIXME:
		70	* GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
5354	serge	71	*/
6084	serge	72	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		73	GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
5354	serge	74	}
		75
3031	serge	76	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		77	{
5060	serge	78	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	79	u32 tmp;
		80
		81	tmp = I915_READ(CLKCFG);
		82
		83	switch (tmp & CLKCFG_FSB_MASK) {
		84	case CLKCFG_FSB_533:
		85	dev_priv->fsb_freq = 533; /* 1334 /
		86	break;
		87	case CLKCFG_FSB_800:
		88	dev_priv->fsb_freq = 800; /* 2004 /
		89	break;
		90	case CLKCFG_FSB_667:
		91	dev_priv->fsb_freq = 667; /* 1674 /
		92	break;
		93	case CLKCFG_FSB_400:
		94	dev_priv->fsb_freq = 400; /* 1004 /
		95	break;
		96	}
		97
		98	switch (tmp & CLKCFG_MEM_MASK) {
		99	case CLKCFG_MEM_533:
		100	dev_priv->mem_freq = 533;
		101	break;
		102	case CLKCFG_MEM_667:
		103	dev_priv->mem_freq = 667;
		104	break;
		105	case CLKCFG_MEM_800:
		106	dev_priv->mem_freq = 800;
		107	break;
		108	}
		109
		110	/* detect pineview DDR3 setting */
		111	tmp = I915_READ(CSHRDDR3CTL);
		112	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		113	}
		114
		115	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		116	{
5060	serge	117	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	118	u16 ddrpll, csipll;
		119
		120	ddrpll = I915_READ16(DDRMPLL1);
		121	csipll = I915_READ16(CSIPLL0);
		122
		123	switch (ddrpll & 0xff) {
		124	case 0xc:
		125	dev_priv->mem_freq = 800;
		126	break;
		127	case 0x10:
		128	dev_priv->mem_freq = 1066;
		129	break;
		130	case 0x14:
		131	dev_priv->mem_freq = 1333;
		132	break;
		133	case 0x18:
		134	dev_priv->mem_freq = 1600;
		135	break;
		136	default:
		137	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		138	ddrpll & 0xff);
		139	dev_priv->mem_freq = 0;
		140	break;
		141	}
		142
		143	dev_priv->ips.r_t = dev_priv->mem_freq;
		144
		145	switch (csipll & 0x3ff) {
		146	case 0x00c:
		147	dev_priv->fsb_freq = 3200;
		148	break;
		149	case 0x00e:
		150	dev_priv->fsb_freq = 3733;
		151	break;
		152	case 0x010:
		153	dev_priv->fsb_freq = 4266;
		154	break;
		155	case 0x012:
		156	dev_priv->fsb_freq = 4800;
		157	break;
		158	case 0x014:
		159	dev_priv->fsb_freq = 5333;
		160	break;
		161	case 0x016:
		162	dev_priv->fsb_freq = 5866;
		163	break;
		164	case 0x018:
		165	dev_priv->fsb_freq = 6400;
		166	break;
		167	default:
		168	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		169	csipll & 0x3ff);
		170	dev_priv->fsb_freq = 0;
		171	break;
		172	}
		173
		174	if (dev_priv->fsb_freq == 3200) {
		175	dev_priv->ips.c_m = 0;
		176	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		177	dev_priv->ips.c_m = 1;
		178	} else {
		179	dev_priv->ips.c_m = 2;
		180	}
		181	}
		182
		183	static const struct cxsr_latency cxsr_latency_table[] = {
		184	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		185	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		186	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		187	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		188	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		189
		190	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		191	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		192	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		193	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		194	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		195
		196	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		197	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		198	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		199	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		200	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		201
		202	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		203	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		204	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		205	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		206	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		207
		208	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		209	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		210	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		211	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		212	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		213
		214	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		215	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		216	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		217	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		218	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		219	};
		220
		221	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		222	int is_ddr3,
		223	int fsb,
		224	int mem)
		225	{
		226	const struct cxsr_latency *latency;
		227	int i;
		228
		229	if (fsb == 0 \|\| mem == 0)
		230	return NULL;
		231
		232	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		233	latency = &cxsr_latency_table[i];
		234	if (is_desktop == latency->is_desktop &&
		235	is_ddr3 == latency->is_ddr3 &&
		236	fsb == latency->fsb_freq && mem == latency->mem_freq)
		237	return latency;
		238	}
		239
		240	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		241
		242	return NULL;
		243	}
		244
6084	serge	245	static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
		246	{
		247	u32 val;
		248
		249	mutex_lock(&dev_priv->rps.hw_lock);
		250
		251	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		252	if (enable)
		253	val &= ~FORCE_DDR_HIGH_FREQ;
		254	else
		255	val \|= FORCE_DDR_HIGH_FREQ;
		256	val &= ~FORCE_DDR_LOW_FREQ;
		257	val \|= FORCE_DDR_FREQ_REQ_ACK;
		258	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
		259
		260	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		261	FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
		262	DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
		263
		264	mutex_unlock(&dev_priv->rps.hw_lock);
		265	}
		266
		267	static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
		268	{
		269	u32 val;
		270
		271	mutex_lock(&dev_priv->rps.hw_lock);
		272
		273	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
		274	if (enable)
		275	val \|= DSP_MAXFIFO_PM5_ENABLE;
		276	else
		277	val &= ~DSP_MAXFIFO_PM5_ENABLE;
		278	vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
		279
		280	mutex_unlock(&dev_priv->rps.hw_lock);
		281	}
		282
		283	#define FW_WM(value, plane) \
		284	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
		285
5060	serge	286	void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
3031	serge	287	{
5060	serge	288	struct drm_device *dev = dev_priv->dev;
		289	u32 val;
3031	serge	290
5060	serge	291	if (IS_VALLEYVIEW(dev)) {
		292	I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
6084	serge	293	POSTING_READ(FW_BLC_SELF_VLV);
		294	dev_priv->wm.vlv.cxsr = enable;
5060	serge	295	} else if (IS_G4X(dev) \|\| IS_CRESTLINE(dev)) {
		296	I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
6084	serge	297	POSTING_READ(FW_BLC_SELF);
5060	serge	298	} else if (IS_PINEVIEW(dev)) {
		299	val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
		300	val \|= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
		301	I915_WRITE(DSPFW3, val);
6084	serge	302	POSTING_READ(DSPFW3);
5060	serge	303	} else if (IS_I945G(dev) \|\| IS_I945GM(dev)) {
		304	val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
		305	_MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
		306	I915_WRITE(FW_BLC_SELF, val);
6084	serge	307	POSTING_READ(FW_BLC_SELF);
5060	serge	308	} else if (IS_I915GM(dev)) {
		309	val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
		310	_MASKED_BIT_DISABLE(INSTPM_SELF_EN);
		311	I915_WRITE(INSTPM, val);
6084	serge	312	POSTING_READ(INSTPM);
5060	serge	313	} else {
		314	return;
		315	}
		316
		317	DRM_DEBUG_KMS("memory self-refresh is %s\n",
		318	enable ? "enabled" : "disabled");
3031	serge	319	}
		320
6084	serge	321
3031	serge	322	/*
		323	* Latency for FIFO fetches is dependent on several factors:
		324	* - memory configuration (speed, channels)
		325	* - chipset
		326	* - current MCH state
		327	* It can be fairly high in some situations, so here we assume a fairly
		328	* pessimal value. It's a tradeoff between extra memory fetches (if we
		329	* set this value too high, the FIFO will fetch frequently to stay full)
		330	* and power consumption (set it too low to save power and we might see
		331	* FIFO underruns and display "flicker").
		332	*
		333	* A value of 5us seems to be a good balance; safe for very low end
		334	* platforms but not overly aggressive on lower latency configs.
		335	*/
5354	serge	336	static const int pessimal_latency_ns = 5000;
3031	serge	337
6084	serge	338	#define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
		339	((((dsparb) >> (lo_shift)) & 0xff) \| ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
		340
		341	static int vlv_get_fifo_size(struct drm_device *dev,
		342	enum pipe pipe, int plane)
		343	{
		344	struct drm_i915_private *dev_priv = dev->dev_private;
		345	int sprite0_start, sprite1_start, size;
		346
		347	switch (pipe) {
		348	uint32_t dsparb, dsparb2, dsparb3;
		349	case PIPE_A:
		350	dsparb = I915_READ(DSPARB);
		351	dsparb2 = I915_READ(DSPARB2);
		352	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
		353	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
		354	break;
		355	case PIPE_B:
		356	dsparb = I915_READ(DSPARB);
		357	dsparb2 = I915_READ(DSPARB2);
		358	sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
		359	sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
		360	break;
		361	case PIPE_C:
		362	dsparb2 = I915_READ(DSPARB2);
		363	dsparb3 = I915_READ(DSPARB3);
		364	sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
		365	sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
		366	break;
		367	default:
		368	return 0;
		369	}
		370
		371	switch (plane) {
		372	case 0:
		373	size = sprite0_start;
		374	break;
		375	case 1:
		376	size = sprite1_start - sprite0_start;
		377	break;
		378	case 2:
		379	size = 512 - 1 - sprite1_start;
		380	break;
		381	default:
		382	return 0;
		383	}
		384
		385	DRM_DEBUG_KMS("Pipe %c %s %c FIFO size: %d\n",
		386	pipe_name(pipe), plane == 0 ? "primary" : "sprite",
		387	plane == 0 ? plane_name(pipe) : sprite_name(pipe, plane - 1),
		388	size);
		389
		390	return size;
		391	}
		392
3031	serge	393	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		394	{
		395	struct drm_i915_private *dev_priv = dev->dev_private;
		396	uint32_t dsparb = I915_READ(DSPARB);
		397	int size;
		398
		399	size = dsparb & 0x7f;
		400	if (plane)
		401	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		402
		403	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		404	plane ? "B" : "A", size);
		405
		406	return size;
		407	}
		408
4560	Serge	409	static int i830_get_fifo_size(struct drm_device *dev, int plane)
3031	serge	410	{
		411	struct drm_i915_private *dev_priv = dev->dev_private;
		412	uint32_t dsparb = I915_READ(DSPARB);
		413	int size;
		414
		415	size = dsparb & 0x1ff;
		416	if (plane)
		417	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		418	size >>= 1; /* Convert to cachelines */
		419
		420	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		421	plane ? "B" : "A", size);
		422
		423	return size;
		424	}
		425
		426	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		427	{
		428	struct drm_i915_private *dev_priv = dev->dev_private;
		429	uint32_t dsparb = I915_READ(DSPARB);
		430	int size;
		431
		432	size = dsparb & 0x7f;
		433	size >>= 2; /* Convert to cachelines */
		434
		435	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		436	plane ? "B" : "A",
		437	size);
		438
		439	return size;
		440	}
		441
		442	/* Pineview has different values for various configs */
		443	static const struct intel_watermark_params pineview_display_wm = {
5060	serge	444	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		445	.max_wm = PINEVIEW_MAX_WM,
		446	.default_wm = PINEVIEW_DFT_WM,
		447	.guard_size = PINEVIEW_GUARD_WM,
		448	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	449	};
		450	static const struct intel_watermark_params pineview_display_hplloff_wm = {
5060	serge	451	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		452	.max_wm = PINEVIEW_MAX_WM,
		453	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
		454	.guard_size = PINEVIEW_GUARD_WM,
		455	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	456	};
		457	static const struct intel_watermark_params pineview_cursor_wm = {
5060	serge	458	.fifo_size = PINEVIEW_CURSOR_FIFO,
		459	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		460	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		461	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		462	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	463	};
		464	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
5060	serge	465	.fifo_size = PINEVIEW_CURSOR_FIFO,
		466	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		467	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		468	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		469	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	470	};
		471	static const struct intel_watermark_params g4x_wm_info = {
5060	serge	472	.fifo_size = G4X_FIFO_SIZE,
		473	.max_wm = G4X_MAX_WM,
		474	.default_wm = G4X_MAX_WM,
		475	.guard_size = 2,
		476	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	477	};
		478	static const struct intel_watermark_params g4x_cursor_wm_info = {
5060	serge	479	.fifo_size = I965_CURSOR_FIFO,
		480	.max_wm = I965_CURSOR_MAX_WM,
		481	.default_wm = I965_CURSOR_DFT_WM,
		482	.guard_size = 2,
		483	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	484	};
		485	static const struct intel_watermark_params valleyview_wm_info = {
5060	serge	486	.fifo_size = VALLEYVIEW_FIFO_SIZE,
		487	.max_wm = VALLEYVIEW_MAX_WM,
		488	.default_wm = VALLEYVIEW_MAX_WM,
		489	.guard_size = 2,
		490	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	491	};
		492	static const struct intel_watermark_params valleyview_cursor_wm_info = {
5060	serge	493	.fifo_size = I965_CURSOR_FIFO,
		494	.max_wm = VALLEYVIEW_CURSOR_MAX_WM,
		495	.default_wm = I965_CURSOR_DFT_WM,
		496	.guard_size = 2,
		497	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	498	};
		499	static const struct intel_watermark_params i965_cursor_wm_info = {
5060	serge	500	.fifo_size = I965_CURSOR_FIFO,
		501	.max_wm = I965_CURSOR_MAX_WM,
		502	.default_wm = I965_CURSOR_DFT_WM,
		503	.guard_size = 2,
		504	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	505	};
		506	static const struct intel_watermark_params i945_wm_info = {
5060	serge	507	.fifo_size = I945_FIFO_SIZE,
		508	.max_wm = I915_MAX_WM,
		509	.default_wm = 1,
		510	.guard_size = 2,
		511	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	512	};
		513	static const struct intel_watermark_params i915_wm_info = {
5060	serge	514	.fifo_size = I915_FIFO_SIZE,
		515	.max_wm = I915_MAX_WM,
		516	.default_wm = 1,
		517	.guard_size = 2,
		518	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	519	};
5354	serge	520	static const struct intel_watermark_params i830_a_wm_info = {
5060	serge	521	.fifo_size = I855GM_FIFO_SIZE,
		522	.max_wm = I915_MAX_WM,
		523	.default_wm = 1,
		524	.guard_size = 2,
		525	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	526	};
5354	serge	527	static const struct intel_watermark_params i830_bc_wm_info = {
		528	.fifo_size = I855GM_FIFO_SIZE,
		529	.max_wm = I915_MAX_WM/2,
		530	.default_wm = 1,
		531	.guard_size = 2,
		532	.cacheline_size = I830_FIFO_LINE_SIZE,
		533	};
4560	Serge	534	static const struct intel_watermark_params i845_wm_info = {
5060	serge	535	.fifo_size = I830_FIFO_SIZE,
		536	.max_wm = I915_MAX_WM,
		537	.default_wm = 1,
		538	.guard_size = 2,
		539	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	540	};
		541
		542	/**
		543	* intel_calculate_wm - calculate watermark level
		544	* @clock_in_khz: pixel clock
		545	* @wm: chip FIFO params
		546	* @pixel_size: display pixel size
		547	* @latency_ns: memory latency for the platform
		548	*
		549	* Calculate the watermark level (the level at which the display plane will
		550	* start fetching from memory again). Each chip has a different display
		551	* FIFO size and allocation, so the caller needs to figure that out and pass
		552	* in the correct intel_watermark_params structure.
		553	*
		554	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		555	* on the pixel size. When it reaches the watermark level, it'll start
		556	* fetching FIFO line sized based chunks from memory until the FIFO fills
		557	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		558	* will occur, and a display engine hang could result.
		559	*/
		560	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		561	const struct intel_watermark_params *wm,
		562	int fifo_size,
		563	int pixel_size,
		564	unsigned long latency_ns)
		565	{
		566	long entries_required, wm_size;
		567
		568	/*
		569	* Note: we need to make sure we don't overflow for various clock &
		570	* latency values.
		571	* clocks go from a few thousand to several hundred thousand.
		572	* latency is usually a few thousand
		573	*/
		574	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		575	1000;
		576	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		577
		578	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		579
		580	wm_size = fifo_size - (entries_required + wm->guard_size);
		581
		582	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		583
		584	/* Don't promote wm_size to unsigned... */
		585	if (wm_size > (long)wm->max_wm)
		586	wm_size = wm->max_wm;
		587	if (wm_size <= 0)
		588	wm_size = wm->default_wm;
5354	serge	589
		590	/*
		591	* Bspec seems to indicate that the value shouldn't be lower than
		592	* 'burst size + 1'. Certainly 830 is quite unhappy with low values.
		593	* Lets go for 8 which is the burst size since certain platforms
		594	* already use a hardcoded 8 (which is what the spec says should be
		595	* done).
		596	*/
		597	if (wm_size <= 8)
		598	wm_size = 8;
		599
3031	serge	600	return wm_size;
		601	}
		602
		603	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		604	{
		605	struct drm_crtc crtc, enabled = NULL;
		606
5060	serge	607	for_each_crtc(dev, crtc) {
3243	Serge	608	if (intel_crtc_active(crtc)) {
3031	serge	609	if (enabled)
		610	return NULL;
		611	enabled = crtc;
		612	}
		613	}
		614
		615	return enabled;
		616	}
		617
4560	Serge	618	static void pineview_update_wm(struct drm_crtc *unused_crtc)
3031	serge	619	{
4560	Serge	620	struct drm_device *dev = unused_crtc->dev;
3031	serge	621	struct drm_i915_private *dev_priv = dev->dev_private;
		622	struct drm_crtc *crtc;
		623	const struct cxsr_latency *latency;
		624	u32 reg;
		625	unsigned long wm;
		626
		627	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		628	dev_priv->fsb_freq, dev_priv->mem_freq);
		629	if (!latency) {
		630	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
5060	serge	631	intel_set_memory_cxsr(dev_priv, false);
3031	serge	632	return;
		633	}
		634
		635	crtc = single_enabled_crtc(dev);
		636	if (crtc) {
6084	serge	637	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
		638	int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
		639	int clock = adjusted_mode->crtc_clock;
3031	serge	640
		641	/* Display SR */
		642	wm = intel_calculate_wm(clock, &pineview_display_wm,
		643	pineview_display_wm.fifo_size,
		644	pixel_size, latency->display_sr);
		645	reg = I915_READ(DSPFW1);
		646	reg &= ~DSPFW_SR_MASK;
6084	serge	647	reg \|= FW_WM(wm, SR);
3031	serge	648	I915_WRITE(DSPFW1, reg);
		649	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		650
		651	/* cursor SR */
		652	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		653	pineview_display_wm.fifo_size,
		654	pixel_size, latency->cursor_sr);
		655	reg = I915_READ(DSPFW3);
		656	reg &= ~DSPFW_CURSOR_SR_MASK;
6084	serge	657	reg \|= FW_WM(wm, CURSOR_SR);
3031	serge	658	I915_WRITE(DSPFW3, reg);
		659
		660	/* Display HPLL off SR */
		661	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		662	pineview_display_hplloff_wm.fifo_size,
		663	pixel_size, latency->display_hpll_disable);
		664	reg = I915_READ(DSPFW3);
		665	reg &= ~DSPFW_HPLL_SR_MASK;
6084	serge	666	reg \|= FW_WM(wm, HPLL_SR);
3031	serge	667	I915_WRITE(DSPFW3, reg);
		668
		669	/* cursor HPLL off SR */
		670	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		671	pineview_display_hplloff_wm.fifo_size,
		672	pixel_size, latency->cursor_hpll_disable);
		673	reg = I915_READ(DSPFW3);
		674	reg &= ~DSPFW_HPLL_CURSOR_MASK;
6084	serge	675	reg \|= FW_WM(wm, HPLL_CURSOR);
3031	serge	676	I915_WRITE(DSPFW3, reg);
		677	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		678
5060	serge	679	intel_set_memory_cxsr(dev_priv, true);
3031	serge	680	} else {
5060	serge	681	intel_set_memory_cxsr(dev_priv, false);
3031	serge	682	}
		683	}
		684
		685	static bool g4x_compute_wm0(struct drm_device *dev,
		686	int plane,
		687	const struct intel_watermark_params *display,
		688	int display_latency_ns,
		689	const struct intel_watermark_params *cursor,
		690	int cursor_latency_ns,
		691	int *plane_wm,
		692	int *cursor_wm)
		693	{
		694	struct drm_crtc *crtc;
4560	Serge	695	const struct drm_display_mode *adjusted_mode;
3031	serge	696	int htotal, hdisplay, clock, pixel_size;
		697	int line_time_us, line_count;
		698	int entries, tlb_miss;
		699
		700	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	701	if (!intel_crtc_active(crtc)) {
3031	serge	702	*cursor_wm = cursor->guard_size;
		703	*plane_wm = display->guard_size;
6084	serge	704	return false;
3031	serge	705	}
		706
6084	serge	707	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	708	clock = adjusted_mode->crtc_clock;
		709	htotal = adjusted_mode->crtc_htotal;
6084	serge	710	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		711	pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	712
		713	/* Use the small buffer method to calculate plane watermark */
		714	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		715	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		716	if (tlb_miss > 0)
		717	entries += tlb_miss;
		718	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		719	*plane_wm = entries + display->guard_size;
		720	if (*plane_wm > (int)display->max_wm)
		721	*plane_wm = display->max_wm;
		722
		723	/* Use the large buffer method to calculate cursor watermark */
5060	serge	724	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	725	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
6084	serge	726	entries = line_count * crtc->cursor->state->crtc_w * pixel_size;
3031	serge	727	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		728	if (tlb_miss > 0)
		729	entries += tlb_miss;
		730	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		731	*cursor_wm = entries + cursor->guard_size;
		732	if (*cursor_wm > (int)cursor->max_wm)
		733	*cursor_wm = (int)cursor->max_wm;
		734
		735	return true;
		736	}
		737
		738	/*
		739	* Check the wm result.
		740	*
		741	* If any calculated watermark values is larger than the maximum value that
		742	* can be programmed into the associated watermark register, that watermark
		743	* must be disabled.
		744	*/
		745	static bool g4x_check_srwm(struct drm_device *dev,
		746	int display_wm, int cursor_wm,
		747	const struct intel_watermark_params *display,
		748	const struct intel_watermark_params *cursor)
		749	{
		750	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		751	display_wm, cursor_wm);
		752
		753	if (display_wm > display->max_wm) {
		754	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		755	display_wm, display->max_wm);
		756	return false;
		757	}
		758
		759	if (cursor_wm > cursor->max_wm) {
		760	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		761	cursor_wm, cursor->max_wm);
		762	return false;
		763	}
		764
		765	if (!(display_wm \|\| cursor_wm)) {
		766	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		767	return false;
		768	}
		769
		770	return true;
		771	}
		772
		773	static bool g4x_compute_srwm(struct drm_device *dev,
		774	int plane,
		775	int latency_ns,
		776	const struct intel_watermark_params *display,
		777	const struct intel_watermark_params *cursor,
		778	int display_wm, int cursor_wm)
		779	{
		780	struct drm_crtc *crtc;
4560	Serge	781	const struct drm_display_mode *adjusted_mode;
3031	serge	782	int hdisplay, htotal, pixel_size, clock;
		783	unsigned long line_time_us;
		784	int line_count, line_size;
		785	int small, large;
		786	int entries;
		787
		788	if (!latency_ns) {
		789	display_wm = cursor_wm = 0;
		790	return false;
		791	}
		792
		793	crtc = intel_get_crtc_for_plane(dev, plane);
6084	serge	794	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	795	clock = adjusted_mode->crtc_clock;
		796	htotal = adjusted_mode->crtc_htotal;
6084	serge	797	hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		798	pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	799
5060	serge	800	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	801	line_count = (latency_ns / line_time_us + 1000) / 1000;
		802	line_size = hdisplay * pixel_size;
		803
		804	/* Use the minimum of the small and large buffer method for primary */
		805	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		806	large = line_count * line_size;
		807
		808	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		809	*display_wm = entries + display->guard_size;
		810
		811	/* calculate the self-refresh watermark for display cursor */
6084	serge	812	entries = line_count * pixel_size * crtc->cursor->state->crtc_w;
3031	serge	813	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		814	*cursor_wm = entries + cursor->guard_size;
		815
		816	return g4x_check_srwm(dev,
		817	display_wm, cursor_wm,
		818	display, cursor);
		819	}
		820
6084	serge	821	#define FW_WM_VLV(value, plane) \
		822	(((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
		823
		824	static void vlv_write_wm_values(struct intel_crtc *crtc,
		825	const struct vlv_wm_values *wm)
3031	serge	826	{
6084	serge	827	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
		828	enum pipe pipe = crtc->pipe;
3031	serge	829
6084	serge	830	I915_WRITE(VLV_DDL(pipe),
		831	(wm->ddl[pipe].cursor << DDL_CURSOR_SHIFT) \|
		832	(wm->ddl[pipe].sprite[1] << DDL_SPRITE_SHIFT(1)) \|
		833	(wm->ddl[pipe].sprite[0] << DDL_SPRITE_SHIFT(0)) \|
		834	(wm->ddl[pipe].primary << DDL_PLANE_SHIFT));
3031	serge	835
6084	serge	836	I915_WRITE(DSPFW1,
		837	FW_WM(wm->sr.plane, SR) \|
		838	FW_WM(wm->pipe[PIPE_B].cursor, CURSORB) \|
		839	FW_WM_VLV(wm->pipe[PIPE_B].primary, PLANEB) \|
		840	FW_WM_VLV(wm->pipe[PIPE_A].primary, PLANEA));
		841	I915_WRITE(DSPFW2,
		842	FW_WM_VLV(wm->pipe[PIPE_A].sprite[1], SPRITEB) \|
		843	FW_WM(wm->pipe[PIPE_A].cursor, CURSORA) \|
		844	FW_WM_VLV(wm->pipe[PIPE_A].sprite[0], SPRITEA));
		845	I915_WRITE(DSPFW3,
		846	FW_WM(wm->sr.cursor, CURSOR_SR));
3031	serge	847
6084	serge	848	if (IS_CHERRYVIEW(dev_priv)) {
		849	I915_WRITE(DSPFW7_CHV,
		850	FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) \|
		851	FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
		852	I915_WRITE(DSPFW8_CHV,
		853	FW_WM_VLV(wm->pipe[PIPE_C].sprite[1], SPRITEF) \|
		854	FW_WM_VLV(wm->pipe[PIPE_C].sprite[0], SPRITEE));
		855	I915_WRITE(DSPFW9_CHV,
		856	FW_WM_VLV(wm->pipe[PIPE_C].primary, PLANEC) \|
		857	FW_WM(wm->pipe[PIPE_C].cursor, CURSORC));
		858	I915_WRITE(DSPHOWM,
		859	FW_WM(wm->sr.plane >> 9, SR_HI) \|
		860	FW_WM(wm->pipe[PIPE_C].sprite[1] >> 8, SPRITEF_HI) \|
		861	FW_WM(wm->pipe[PIPE_C].sprite[0] >> 8, SPRITEE_HI) \|
		862	FW_WM(wm->pipe[PIPE_C].primary >> 8, PLANEC_HI) \|
		863	FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) \|
		864	FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) \|
		865	FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) \|
		866	FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) \|
		867	FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) \|
		868	FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
		869	} else {
		870	I915_WRITE(DSPFW7,
		871	FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) \|
		872	FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
		873	I915_WRITE(DSPHOWM,
		874	FW_WM(wm->sr.plane >> 9, SR_HI) \|
		875	FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) \|
		876	FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) \|
		877	FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) \|
		878	FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) \|
		879	FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) \|
		880	FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
		881	}
3031	serge	882
6084	serge	883	/* zero (unused) WM1 watermarks */
		884	I915_WRITE(DSPFW4, 0);
		885	I915_WRITE(DSPFW5, 0);
		886	I915_WRITE(DSPFW6, 0);
		887	I915_WRITE(DSPHOWM1, 0);
3031	serge	888
6084	serge	889	POSTING_READ(DSPFW1);
3031	serge	890	}
		891
6084	serge	892	#undef FW_WM_VLV
3031	serge	893
6084	serge	894	enum vlv_wm_level {
		895	VLV_WM_LEVEL_PM2,
		896	VLV_WM_LEVEL_PM5,
		897	VLV_WM_LEVEL_DDR_DVFS,
		898	};
		899
		900	/* latency must be in 0.1us units. */
		901	static unsigned int vlv_wm_method2(unsigned int pixel_rate,
		902	unsigned int pipe_htotal,
		903	unsigned int horiz_pixels,
		904	unsigned int bytes_per_pixel,
		905	unsigned int latency)
3031	serge	906	{
6084	serge	907	unsigned int ret;
		908
		909	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		910	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		911	ret = DIV_ROUND_UP(ret, 64);
		912
		913	return ret;
		914	}
		915
		916	static void vlv_setup_wm_latency(struct drm_device *dev)
		917	{
3031	serge	918	struct drm_i915_private *dev_priv = dev->dev_private;
		919
6084	serge	920	/* all latencies in usec */
		921	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
3031	serge	922
6084	serge	923	dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
		924
		925	if (IS_CHERRYVIEW(dev_priv)) {
		926	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
		927	dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
		928
		929	dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
3031	serge	930	}
6084	serge	931	}
3031	serge	932
6084	serge	933	static uint16_t vlv_compute_wm_level(struct intel_plane *plane,
		934	struct intel_crtc *crtc,
		935	const struct intel_plane_state *state,
		936	int level)
		937	{
		938	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
		939	int clock, htotal, pixel_size, width, wm;
		940
		941	if (dev_priv->wm.pri_latency[level] == 0)
		942	return USHRT_MAX;
		943
		944	if (!state->visible)
		945	return 0;
		946
		947	pixel_size = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		948	clock = crtc->config->base.adjusted_mode.crtc_clock;
		949	htotal = crtc->config->base.adjusted_mode.crtc_htotal;
		950	width = crtc->config->pipe_src_w;
		951	if (WARN_ON(htotal == 0))
		952	htotal = 1;
		953
		954	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		955	/*
		956	* FIXME the formula gives values that are
		957	* too big for the cursor FIFO, and hence we
		958	* would never be able to use cursors. For
		959	* now just hardcode the watermark.
		960	*/
		961	wm = 63;
		962	} else {
		963	wm = vlv_wm_method2(clock, htotal, width, pixel_size,
		964	dev_priv->wm.pri_latency[level] * 10);
5354	serge	965	}
3031	serge	966
6084	serge	967	return min_t(int, wm, USHRT_MAX);
		968	}
5354	serge	969
6084	serge	970	static void vlv_compute_fifo(struct intel_crtc *crtc)
		971	{
		972	struct drm_device *dev = crtc->base.dev;
		973	struct vlv_wm_state *wm_state = &crtc->wm_state;
		974	struct intel_plane *plane;
		975	unsigned int total_rate = 0;
		976	const int fifo_size = 512 - 1;
		977	int fifo_extra, fifo_left = fifo_size;
		978
		979	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		980	struct intel_plane_state *state =
		981	to_intel_plane_state(plane->base.state);
		982
		983	if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
		984	continue;
		985
		986	if (state->visible) {
		987	wm_state->num_active_planes++;
		988	total_rate += drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		989	}
3031	serge	990	}
5354	serge	991
6084	serge	992	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		993	struct intel_plane_state *state =
		994	to_intel_plane_state(plane->base.state);
		995	unsigned int rate;
		996
		997	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		998	plane->wm.fifo_size = 63;
		999	continue;
		1000	}
		1001
		1002	if (!state->visible) {
		1003	plane->wm.fifo_size = 0;
		1004	continue;
		1005	}
		1006
		1007	rate = drm_format_plane_cpp(state->base.fb->pixel_format, 0);
		1008	plane->wm.fifo_size = fifo_size * rate / total_rate;
		1009	fifo_left -= plane->wm.fifo_size;
		1010	}
		1011
		1012	fifo_extra = DIV_ROUND_UP(fifo_left, wm_state->num_active_planes ?: 1);
		1013
		1014	/* spread the remainder evenly */
		1015	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1016	int plane_extra;
		1017
		1018	if (fifo_left == 0)
		1019	break;
		1020
		1021	if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
		1022	continue;
		1023
		1024	/* give it all to the first plane if none are active */
		1025	if (plane->wm.fifo_size == 0 &&
		1026	wm_state->num_active_planes)
		1027	continue;
		1028
		1029	plane_extra = min(fifo_extra, fifo_left);
		1030	plane->wm.fifo_size += plane_extra;
		1031	fifo_left -= plane_extra;
		1032	}
		1033
		1034	WARN_ON(fifo_left != 0);
3031	serge	1035	}
		1036
6084	serge	1037	static void vlv_invert_wms(struct intel_crtc *crtc)
		1038	{
		1039	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1040	int level;
3031	serge	1041
6084	serge	1042	for (level = 0; level < wm_state->num_levels; level++) {
		1043	struct drm_device *dev = crtc->base.dev;
		1044	const int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
		1045	struct intel_plane *plane;
		1046
		1047	wm_state->sr[level].plane = sr_fifo_size - wm_state->sr[level].plane;
		1048	wm_state->sr[level].cursor = 63 - wm_state->sr[level].cursor;
		1049
		1050	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1051	switch (plane->base.type) {
		1052	int sprite;
		1053	case DRM_PLANE_TYPE_CURSOR:
		1054	wm_state->wm[level].cursor = plane->wm.fifo_size -
		1055	wm_state->wm[level].cursor;
		1056	break;
		1057	case DRM_PLANE_TYPE_PRIMARY:
		1058	wm_state->wm[level].primary = plane->wm.fifo_size -
		1059	wm_state->wm[level].primary;
		1060	break;
		1061	case DRM_PLANE_TYPE_OVERLAY:
		1062	sprite = plane->plane;
		1063	wm_state->wm[level].sprite[sprite] = plane->wm.fifo_size -
		1064	wm_state->wm[level].sprite[sprite];
		1065	break;
		1066	}
		1067	}
		1068	}
		1069	}
		1070
		1071	static void vlv_compute_wm(struct intel_crtc *crtc)
3031	serge	1072	{
6084	serge	1073	struct drm_device *dev = crtc->base.dev;
		1074	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1075	struct intel_plane *plane;
		1076	int sr_fifo_size = INTEL_INFO(dev)->num_pipes * 512 - 1;
		1077	int level;
3031	serge	1078
6084	serge	1079	memset(wm_state, 0, sizeof(*wm_state));
3031	serge	1080
6084	serge	1081	wm_state->cxsr = crtc->pipe != PIPE_C && crtc->wm.cxsr_allowed;
		1082	wm_state->num_levels = to_i915(dev)->wm.max_level + 1;
3031	serge	1083
6084	serge	1084	wm_state->num_active_planes = 0;
3031	serge	1085
6084	serge	1086	vlv_compute_fifo(crtc);
		1087
		1088	if (wm_state->num_active_planes != 1)
		1089	wm_state->cxsr = false;
		1090
		1091	if (wm_state->cxsr) {
		1092	for (level = 0; level < wm_state->num_levels; level++) {
		1093	wm_state->sr[level].plane = sr_fifo_size;
		1094	wm_state->sr[level].cursor = 63;
		1095	}
3243	Serge	1096	}
3031	serge	1097
6084	serge	1098	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1099	struct intel_plane_state *state =
		1100	to_intel_plane_state(plane->base.state);
3031	serge	1101
6084	serge	1102	if (!state->visible)
		1103	continue;
5060	serge	1104
6084	serge	1105	/* normal watermarks */
		1106	for (level = 0; level < wm_state->num_levels; level++) {
		1107	int wm = vlv_compute_wm_level(plane, crtc, state, level);
		1108	int max_wm = plane->base.type == DRM_PLANE_TYPE_CURSOR ? 63 : 511;
		1109
		1110	/* hack */
		1111	if (WARN_ON(level == 0 && wm > max_wm))
		1112	wm = max_wm;
		1113
		1114	if (wm > plane->wm.fifo_size)
		1115	break;
		1116
		1117	switch (plane->base.type) {
		1118	int sprite;
		1119	case DRM_PLANE_TYPE_CURSOR:
		1120	wm_state->wm[level].cursor = wm;
		1121	break;
		1122	case DRM_PLANE_TYPE_PRIMARY:
		1123	wm_state->wm[level].primary = wm;
		1124	break;
		1125	case DRM_PLANE_TYPE_OVERLAY:
		1126	sprite = plane->plane;
		1127	wm_state->wm[level].sprite[sprite] = wm;
		1128	break;
		1129	}
		1130	}
		1131
		1132	wm_state->num_levels = level;
		1133
		1134	if (!wm_state->cxsr)
		1135	continue;
		1136
		1137	/* maxfifo watermarks */
		1138	switch (plane->base.type) {
		1139	int sprite, level;
		1140	case DRM_PLANE_TYPE_CURSOR:
		1141	for (level = 0; level < wm_state->num_levels; level++)
		1142	wm_state->sr[level].cursor =
		1143	wm_state->wm[level].cursor;
		1144	break;
		1145	case DRM_PLANE_TYPE_PRIMARY:
		1146	for (level = 0; level < wm_state->num_levels; level++)
		1147	wm_state->sr[level].plane =
		1148	min(wm_state->sr[level].plane,
		1149	wm_state->wm[level].primary);
		1150	break;
		1151	case DRM_PLANE_TYPE_OVERLAY:
		1152	sprite = plane->plane;
		1153	for (level = 0; level < wm_state->num_levels; level++)
		1154	wm_state->sr[level].plane =
		1155	min(wm_state->sr[level].plane,
		1156	wm_state->wm[level].sprite[sprite]);
		1157	break;
		1158	}
		1159	}
		1160
		1161	/* clear any (partially) filled invalid levels */
		1162	for (level = wm_state->num_levels; level < to_i915(dev)->wm.max_level + 1; level++) {
		1163	memset(&wm_state->wm[level], 0, sizeof(wm_state->wm[level]));
		1164	memset(&wm_state->sr[level], 0, sizeof(wm_state->sr[level]));
		1165	}
		1166
		1167	vlv_invert_wms(crtc);
3031	serge	1168	}
		1169
6084	serge	1170	#define VLV_FIFO(plane, value) \
		1171	(((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
		1172
		1173	static void vlv_pipe_set_fifo_size(struct intel_crtc *crtc)
5354	serge	1174	{
6084	serge	1175	struct drm_device *dev = crtc->base.dev;
		1176	struct drm_i915_private *dev_priv = to_i915(dev);
		1177	struct intel_plane *plane;
		1178	int sprite0_start = 0, sprite1_start = 0, fifo_size = 0;
5354	serge	1179
6084	serge	1180	for_each_intel_plane_on_crtc(dev, crtc, plane) {
		1181	if (plane->base.type == DRM_PLANE_TYPE_CURSOR) {
		1182	WARN_ON(plane->wm.fifo_size != 63);
		1183	continue;
		1184	}
5354	serge	1185
6084	serge	1186	if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
		1187	sprite0_start = plane->wm.fifo_size;
		1188	else if (plane->plane == 0)
		1189	sprite1_start = sprite0_start + plane->wm.fifo_size;
		1190	else
		1191	fifo_size = sprite1_start + plane->wm.fifo_size;
		1192	}
5354	serge	1193
6084	serge	1194	WARN_ON(fifo_size != 512 - 1);
5354	serge	1195
6084	serge	1196	DRM_DEBUG_KMS("Pipe %c FIFO split %d / %d / %d\n",
		1197	pipe_name(crtc->pipe), sprite0_start,
		1198	sprite1_start, fifo_size);
5354	serge	1199
6084	serge	1200	switch (crtc->pipe) {
		1201	uint32_t dsparb, dsparb2, dsparb3;
		1202	case PIPE_A:
		1203	dsparb = I915_READ(DSPARB);
		1204	dsparb2 = I915_READ(DSPARB2);
		1205
		1206	dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) \|
		1207	VLV_FIFO(SPRITEB, 0xff));
		1208	dsparb \|= (VLV_FIFO(SPRITEA, sprite0_start) \|
		1209	VLV_FIFO(SPRITEB, sprite1_start));
		1210
		1211	dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) \|
		1212	VLV_FIFO(SPRITEB_HI, 0x1));
		1213	dsparb2 \|= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) \|
		1214	VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
		1215
		1216	I915_WRITE(DSPARB, dsparb);
		1217	I915_WRITE(DSPARB2, dsparb2);
		1218	break;
		1219	case PIPE_B:
		1220	dsparb = I915_READ(DSPARB);
		1221	dsparb2 = I915_READ(DSPARB2);
		1222
		1223	dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) \|
		1224	VLV_FIFO(SPRITED, 0xff));
		1225	dsparb \|= (VLV_FIFO(SPRITEC, sprite0_start) \|
		1226	VLV_FIFO(SPRITED, sprite1_start));
		1227
		1228	dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) \|
		1229	VLV_FIFO(SPRITED_HI, 0xff));
		1230	dsparb2 \|= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) \|
		1231	VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
		1232
		1233	I915_WRITE(DSPARB, dsparb);
		1234	I915_WRITE(DSPARB2, dsparb2);
		1235	break;
		1236	case PIPE_C:
		1237	dsparb3 = I915_READ(DSPARB3);
		1238	dsparb2 = I915_READ(DSPARB2);
		1239
		1240	dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) \|
		1241	VLV_FIFO(SPRITEF, 0xff));
		1242	dsparb3 \|= (VLV_FIFO(SPRITEE, sprite0_start) \|
		1243	VLV_FIFO(SPRITEF, sprite1_start));
		1244
		1245	dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) \|
		1246	VLV_FIFO(SPRITEF_HI, 0xff));
		1247	dsparb2 \|= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) \|
		1248	VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
		1249
		1250	I915_WRITE(DSPARB3, dsparb3);
		1251	I915_WRITE(DSPARB2, dsparb2);
		1252	break;
		1253	default:
		1254	break;
5354	serge	1255	}
6084	serge	1256	}
5354	serge	1257
6084	serge	1258	#undef VLV_FIFO
5354	serge	1259
6084	serge	1260	static void vlv_merge_wm(struct drm_device *dev,
		1261	struct vlv_wm_values *wm)
		1262	{
		1263	struct intel_crtc *crtc;
		1264	int num_active_crtcs = 0;
5354	serge	1265
6084	serge	1266	wm->level = to_i915(dev)->wm.max_level;
		1267	wm->cxsr = true;
		1268
		1269	for_each_intel_crtc(dev, crtc) {
		1270	const struct vlv_wm_state *wm_state = &crtc->wm_state;
		1271
		1272	if (!crtc->active)
		1273	continue;
		1274
		1275	if (!wm_state->cxsr)
		1276	wm->cxsr = false;
		1277
		1278	num_active_crtcs++;
		1279	wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
		1280	}
		1281
		1282	if (num_active_crtcs != 1)
		1283	wm->cxsr = false;
		1284
		1285	if (num_active_crtcs > 1)
		1286	wm->level = VLV_WM_LEVEL_PM2;
		1287
		1288	for_each_intel_crtc(dev, crtc) {
		1289	struct vlv_wm_state *wm_state = &crtc->wm_state;
		1290	enum pipe pipe = crtc->pipe;
		1291
		1292	if (!crtc->active)
		1293	continue;
		1294
		1295	wm->pipe[pipe] = wm_state->wm[wm->level];
		1296	if (wm->cxsr)
		1297	wm->sr = wm_state->sr[wm->level];
		1298
		1299	wm->ddl[pipe].primary = DDL_PRECISION_HIGH \| 2;
		1300	wm->ddl[pipe].sprite[0] = DDL_PRECISION_HIGH \| 2;
		1301	wm->ddl[pipe].sprite[1] = DDL_PRECISION_HIGH \| 2;
		1302	wm->ddl[pipe].cursor = DDL_PRECISION_HIGH \| 2;
		1303	}
5354	serge	1304	}
		1305
6084	serge	1306	static void vlv_update_wm(struct drm_crtc *crtc)
5354	serge	1307	{
		1308	struct drm_device *dev = crtc->dev;
		1309	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	1310	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1311	enum pipe pipe = intel_crtc->pipe;
		1312	struct vlv_wm_values wm = {};
5354	serge	1313
6084	serge	1314	vlv_compute_wm(intel_crtc);
		1315	vlv_merge_wm(dev, &wm);
5354	serge	1316
6084	serge	1317	if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) {
		1318	/* FIXME should be part of crtc atomic commit */
		1319	vlv_pipe_set_fifo_size(intel_crtc);
		1320	return;
5354	serge	1321	}
		1322
6084	serge	1323	if (wm.level < VLV_WM_LEVEL_DDR_DVFS &&
		1324	dev_priv->wm.vlv.level >= VLV_WM_LEVEL_DDR_DVFS)
		1325	chv_set_memory_dvfs(dev_priv, false);
		1326
		1327	if (wm.level < VLV_WM_LEVEL_PM5 &&
		1328	dev_priv->wm.vlv.level >= VLV_WM_LEVEL_PM5)
		1329	chv_set_memory_pm5(dev_priv, false);
		1330
		1331	if (!wm.cxsr && dev_priv->wm.vlv.cxsr)
		1332	intel_set_memory_cxsr(dev_priv, false);
		1333
		1334	/* FIXME should be part of crtc atomic commit */
		1335	vlv_pipe_set_fifo_size(intel_crtc);
		1336
		1337	vlv_write_wm_values(intel_crtc, &wm);
		1338
		1339	DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, "
		1340	"sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n",
		1341	pipe_name(pipe), wm.pipe[pipe].primary, wm.pipe[pipe].cursor,
		1342	wm.pipe[pipe].sprite[0], wm.pipe[pipe].sprite[1],
		1343	wm.sr.plane, wm.sr.cursor, wm.level, wm.cxsr);
		1344
		1345	if (wm.cxsr && !dev_priv->wm.vlv.cxsr)
		1346	intel_set_memory_cxsr(dev_priv, true);
		1347
		1348	if (wm.level >= VLV_WM_LEVEL_PM5 &&
		1349	dev_priv->wm.vlv.level < VLV_WM_LEVEL_PM5)
		1350	chv_set_memory_pm5(dev_priv, true);
		1351
		1352	if (wm.level >= VLV_WM_LEVEL_DDR_DVFS &&
		1353	dev_priv->wm.vlv.level < VLV_WM_LEVEL_DDR_DVFS)
		1354	chv_set_memory_dvfs(dev_priv, true);
		1355
		1356	dev_priv->wm.vlv = wm;
5354	serge	1357	}
		1358
6084	serge	1359	#define single_plane_enabled(mask) is_power_of_2(mask)
		1360
4560	Serge	1361	static void g4x_update_wm(struct drm_crtc *crtc)
3031	serge	1362	{
4560	Serge	1363	struct drm_device *dev = crtc->dev;
3031	serge	1364	static const int sr_latency_ns = 12000;
		1365	struct drm_i915_private *dev_priv = dev->dev_private;
		1366	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1367	int plane_sr, cursor_sr;
		1368	unsigned int enabled = 0;
5060	serge	1369	bool cxsr_enabled;
3031	serge	1370
3746	Serge	1371	if (g4x_compute_wm0(dev, PIPE_A,
5354	serge	1372	&g4x_wm_info, pessimal_latency_ns,
		1373	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1374	&planea_wm, &cursora_wm))
3746	Serge	1375	enabled \|= 1 << PIPE_A;
3031	serge	1376
3746	Serge	1377	if (g4x_compute_wm0(dev, PIPE_B,
5354	serge	1378	&g4x_wm_info, pessimal_latency_ns,
		1379	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1380	&planeb_wm, &cursorb_wm))
3746	Serge	1381	enabled \|= 1 << PIPE_B;
3031	serge	1382
		1383	if (single_plane_enabled(enabled) &&
		1384	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1385	sr_latency_ns,
		1386	&g4x_wm_info,
		1387	&g4x_cursor_wm_info,
3243	Serge	1388	&plane_sr, &cursor_sr)) {
5060	serge	1389	cxsr_enabled = true;
3243	Serge	1390	} else {
5060	serge	1391	cxsr_enabled = false;
		1392	intel_set_memory_cxsr(dev_priv, false);
3243	Serge	1393	plane_sr = cursor_sr = 0;
		1394	}
3031	serge	1395
5354	serge	1396	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
		1397	"B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
3031	serge	1398	planea_wm, cursora_wm,
		1399	planeb_wm, cursorb_wm,
		1400	plane_sr, cursor_sr);
		1401
		1402	I915_WRITE(DSPFW1,
6084	serge	1403	FW_WM(plane_sr, SR) \|
		1404	FW_WM(cursorb_wm, CURSORB) \|
		1405	FW_WM(planeb_wm, PLANEB) \|
		1406	FW_WM(planea_wm, PLANEA));
3031	serge	1407	I915_WRITE(DSPFW2,
3243	Serge	1408	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
6084	serge	1409	FW_WM(cursora_wm, CURSORA));
3031	serge	1410	/* HPLL off in SR has some issues on G4x... disable it */
		1411	I915_WRITE(DSPFW3,
3243	Serge	1412	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
6084	serge	1413	FW_WM(cursor_sr, CURSOR_SR));
5060	serge	1414
		1415	if (cxsr_enabled)
		1416	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1417	}
		1418
4560	Serge	1419	static void i965_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1420	{
4560	Serge	1421	struct drm_device *dev = unused_crtc->dev;
3031	serge	1422	struct drm_i915_private *dev_priv = dev->dev_private;
		1423	struct drm_crtc *crtc;
		1424	int srwm = 1;
		1425	int cursor_sr = 16;
5060	serge	1426	bool cxsr_enabled;
3031	serge	1427
		1428	/* Calc sr entries for one plane configs */
		1429	crtc = single_enabled_crtc(dev);
		1430	if (crtc) {
		1431	/* self-refresh has much higher latency */
		1432	static const int sr_latency_ns = 12000;
6084	serge	1433	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1434	int clock = adjusted_mode->crtc_clock;
		1435	int htotal = adjusted_mode->crtc_htotal;
6084	serge	1436	int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
		1437	int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
3031	serge	1438	unsigned long line_time_us;
		1439	int entries;
		1440
5060	serge	1441	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1442
		1443	/* Use ns/us then divide to preserve precision */
		1444	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1445	pixel_size * hdisplay;
		1446	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1447	srwm = I965_FIFO_SIZE - entries;
		1448	if (srwm < 0)
		1449	srwm = 1;
		1450	srwm &= 0x1ff;
		1451	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1452	entries, srwm);
		1453
		1454	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
6084	serge	1455	pixel_size * crtc->cursor->state->crtc_w;
3031	serge	1456	entries = DIV_ROUND_UP(entries,
		1457	i965_cursor_wm_info.cacheline_size);
		1458	cursor_sr = i965_cursor_wm_info.fifo_size -
		1459	(entries + i965_cursor_wm_info.guard_size);
		1460
		1461	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1462	cursor_sr = i965_cursor_wm_info.max_wm;
		1463
		1464	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1465	"cursor %d\n", srwm, cursor_sr);
		1466
5060	serge	1467	cxsr_enabled = true;
3031	serge	1468	} else {
5060	serge	1469	cxsr_enabled = false;
3031	serge	1470	/* Turn off self refresh if both pipes are enabled */
5060	serge	1471	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1472	}
		1473
		1474	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1475	srwm);
		1476
		1477	/* 965 has limitations... */
6084	serge	1478	I915_WRITE(DSPFW1, FW_WM(srwm, SR) \|
		1479	FW_WM(8, CURSORB) \|
		1480	FW_WM(8, PLANEB) \|
		1481	FW_WM(8, PLANEA));
		1482	I915_WRITE(DSPFW2, FW_WM(8, CURSORA) \|
		1483	FW_WM(8, PLANEC_OLD));
3031	serge	1484	/* update cursor SR watermark */
6084	serge	1485	I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
5060	serge	1486
		1487	if (cxsr_enabled)
		1488	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1489	}
		1490
6084	serge	1491	#undef FW_WM
		1492
4560	Serge	1493	static void i9xx_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1494	{
4560	Serge	1495	struct drm_device *dev = unused_crtc->dev;
3031	serge	1496	struct drm_i915_private *dev_priv = dev->dev_private;
		1497	const struct intel_watermark_params *wm_info;
		1498	uint32_t fwater_lo;
		1499	uint32_t fwater_hi;
		1500	int cwm, srwm = 1;
		1501	int fifo_size;
		1502	int planea_wm, planeb_wm;
		1503	struct drm_crtc crtc, enabled = NULL;
		1504
		1505	if (IS_I945GM(dev))
		1506	wm_info = &i945_wm_info;
		1507	else if (!IS_GEN2(dev))
		1508	wm_info = &i915_wm_info;
		1509	else
5354	serge	1510	wm_info = &i830_a_wm_info;
3031	serge	1511
		1512	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1513	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1514	if (intel_crtc_active(crtc)) {
4560	Serge	1515	const struct drm_display_mode *adjusted_mode;
6084	serge	1516	int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
3243	Serge	1517	if (IS_GEN2(dev))
		1518	cpp = 4;
		1519
6084	serge	1520	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1521	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1522	wm_info, fifo_size, cpp,
5354	serge	1523	pessimal_latency_ns);
3031	serge	1524	enabled = crtc;
5354	serge	1525	} else {
3031	serge	1526	planea_wm = fifo_size - wm_info->guard_size;
5354	serge	1527	if (planea_wm > (long)wm_info->max_wm)
		1528	planea_wm = wm_info->max_wm;
		1529	}
3031	serge	1530
5354	serge	1531	if (IS_GEN2(dev))
		1532	wm_info = &i830_bc_wm_info;
		1533
3031	serge	1534	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1535	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1536	if (intel_crtc_active(crtc)) {
4560	Serge	1537	const struct drm_display_mode *adjusted_mode;
6084	serge	1538	int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
3243	Serge	1539	if (IS_GEN2(dev))
		1540	cpp = 4;
		1541
6084	serge	1542	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1543	planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1544	wm_info, fifo_size, cpp,
5354	serge	1545	pessimal_latency_ns);
3031	serge	1546	if (enabled == NULL)
		1547	enabled = crtc;
		1548	else
		1549	enabled = NULL;
5354	serge	1550	} else {
3031	serge	1551	planeb_wm = fifo_size - wm_info->guard_size;
5354	serge	1552	if (planeb_wm > (long)wm_info->max_wm)
		1553	planeb_wm = wm_info->max_wm;
		1554	}
3031	serge	1555
		1556	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1557
5060	serge	1558	if (IS_I915GM(dev) && enabled) {
		1559	struct drm_i915_gem_object *obj;
		1560
6084	serge	1561	obj = intel_fb_obj(enabled->primary->state->fb);
5060	serge	1562
		1563	/* self-refresh seems busted with untiled */
		1564	if (obj->tiling_mode == I915_TILING_NONE)
		1565	enabled = NULL;
		1566	}
		1567
3031	serge	1568	/*
		1569	* Overlay gets an aggressive default since video jitter is bad.
		1570	*/
		1571	cwm = 2;
		1572
		1573	/* Play safe and disable self-refresh before adjusting watermarks. */
5060	serge	1574	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1575
		1576	/* Calc sr entries for one plane configs */
		1577	if (HAS_FW_BLC(dev) && enabled) {
		1578	/* self-refresh has much higher latency */
		1579	static const int sr_latency_ns = 6000;
6084	serge	1580	const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode;
4560	Serge	1581	int clock = adjusted_mode->crtc_clock;
		1582	int htotal = adjusted_mode->crtc_htotal;
6084	serge	1583	int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
		1584	int pixel_size = enabled->primary->state->fb->bits_per_pixel / 8;
3031	serge	1585	unsigned long line_time_us;
		1586	int entries;
		1587
5060	serge	1588	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1589
		1590	/* Use ns/us then divide to preserve precision */
		1591	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1592	pixel_size * hdisplay;
		1593	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1594	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1595	srwm = wm_info->fifo_size - entries;
		1596	if (srwm < 0)
		1597	srwm = 1;
		1598
		1599	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1600	I915_WRITE(FW_BLC_SELF,
		1601	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1602	else if (IS_I915GM(dev))
		1603	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1604	}
		1605
		1606	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1607	planea_wm, planeb_wm, cwm, srwm);
		1608
		1609	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1610	fwater_hi = (cwm & 0x1f);
		1611
		1612	/* Set request length to 8 cachelines per fetch */
		1613	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1614	fwater_hi = fwater_hi \| (1 << 8);
		1615
		1616	I915_WRITE(FW_BLC, fwater_lo);
		1617	I915_WRITE(FW_BLC2, fwater_hi);
		1618
5060	serge	1619	if (enabled)
		1620	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1621	}
		1622
4560	Serge	1623	static void i845_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1624	{
4560	Serge	1625	struct drm_device *dev = unused_crtc->dev;
3031	serge	1626	struct drm_i915_private *dev_priv = dev->dev_private;
		1627	struct drm_crtc *crtc;
4560	Serge	1628	const struct drm_display_mode *adjusted_mode;
3031	serge	1629	uint32_t fwater_lo;
		1630	int planea_wm;
		1631
		1632	crtc = single_enabled_crtc(dev);
		1633	if (crtc == NULL)
		1634	return;
		1635
6084	serge	1636	adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
4560	Serge	1637	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
		1638	&i845_wm_info,
3031	serge	1639	dev_priv->display.get_fifo_size(dev, 0),
5354	serge	1640	4, pessimal_latency_ns);
3031	serge	1641	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1642	fwater_lo \|= (3<<8) \| planea_wm;
		1643
		1644	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1645
		1646	I915_WRITE(FW_BLC, fwater_lo);
		1647	}
		1648
6084	serge	1649	uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config)
3031	serge	1650	{
4104	Serge	1651	uint32_t pixel_rate;
		1652
6084	serge	1653	pixel_rate = pipe_config->base.adjusted_mode.crtc_clock;
4104	Serge	1654
		1655	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
		1656	* adjust the pixel_rate here. */
		1657
6084	serge	1658	if (pipe_config->pch_pfit.enabled) {
4104	Serge	1659	uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
6084	serge	1660	uint32_t pfit_size = pipe_config->pch_pfit.size;
4104	Serge	1661
6084	serge	1662	pipe_w = pipe_config->pipe_src_w;
		1663	pipe_h = pipe_config->pipe_src_h;
		1664
4104	Serge	1665	pfit_w = (pfit_size >> 16) & 0xFFFF;
		1666	pfit_h = pfit_size & 0xFFFF;
		1667	if (pipe_w < pfit_w)
		1668	pipe_w = pfit_w;
		1669	if (pipe_h < pfit_h)
		1670	pipe_h = pfit_h;
		1671
		1672	pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
		1673	pfit_w * pfit_h);
		1674	}
		1675
		1676	return pixel_rate;
		1677	}
		1678
		1679	/* latency must be in 0.1us units. */
		1680	static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		1681	uint32_t latency)
		1682	{
		1683	uint64_t ret;
		1684
		1685	if (WARN(latency == 0, "Latency value missing\n"))
		1686	return UINT_MAX;
		1687
		1688	ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
		1689	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
		1690
		1691	return ret;
		1692	}
		1693
		1694	/* latency must be in 0.1us units. */
		1695	static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		1696	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
		1697	uint32_t latency)
		1698	{
		1699	uint32_t ret;
		1700
		1701	if (WARN(latency == 0, "Latency value missing\n"))
		1702	return UINT_MAX;
		1703
		1704	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		1705	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		1706	ret = DIV_ROUND_UP(ret, 64) + 2;
		1707	return ret;
		1708	}
		1709
		1710	static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
		1711	uint8_t bytes_per_pixel)
		1712	{
		1713	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
		1714	}
		1715
5354	serge	1716	struct skl_pipe_wm_parameters {
		1717	bool active;
		1718	uint32_t pipe_htotal;
		1719	uint32_t pixel_rate; /* in KHz */
		1720	struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
		1721	};
		1722
4560	Serge	1723	struct ilk_wm_maximums {
4104	Serge	1724	uint16_t pri;
		1725	uint16_t spr;
		1726	uint16_t cur;
		1727	uint16_t fbc;
		1728	};
		1729
		1730	/* used in computing the new watermarks state */
		1731	struct intel_wm_config {
		1732	unsigned int num_pipes_active;
		1733	bool sprites_enabled;
		1734	bool sprites_scaled;
		1735	};
		1736
		1737	/*
		1738	* For both WM_PIPE and WM_LP.
		1739	* mem_value must be in 0.1us units.
		1740	*/
6084	serge	1741	static uint32_t ilk_compute_pri_wm(const struct intel_crtc_state *cstate,
		1742	const struct intel_plane_state *pstate,
4104	Serge	1743	uint32_t mem_value,
		1744	bool is_lp)
		1745	{
6084	serge	1746	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
4104	Serge	1747	uint32_t method1, method2;
		1748
6084	serge	1749	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1750	return 0;
		1751
6084	serge	1752	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value);
4104	Serge	1753
		1754	if (!is_lp)
		1755	return method1;
		1756
6084	serge	1757	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1758	cstate->base.adjusted_mode.crtc_htotal,
		1759	drm_rect_width(&pstate->dst),
		1760	bpp,
4104	Serge	1761	mem_value);
		1762
		1763	return min(method1, method2);
		1764	}
		1765
		1766	/*
		1767	* For both WM_PIPE and WM_LP.
		1768	* mem_value must be in 0.1us units.
		1769	*/
6084	serge	1770	static uint32_t ilk_compute_spr_wm(const struct intel_crtc_state *cstate,
		1771	const struct intel_plane_state *pstate,
4104	Serge	1772	uint32_t mem_value)
		1773	{
6084	serge	1774	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
4104	Serge	1775	uint32_t method1, method2;
		1776
6084	serge	1777	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1778	return 0;
		1779
6084	serge	1780	method1 = ilk_wm_method1(ilk_pipe_pixel_rate(cstate), bpp, mem_value);
		1781	method2 = ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1782	cstate->base.adjusted_mode.crtc_htotal,
		1783	drm_rect_width(&pstate->dst),
		1784	bpp,
4104	Serge	1785	mem_value);
		1786	return min(method1, method2);
		1787	}
		1788
		1789	/*
		1790	* For both WM_PIPE and WM_LP.
		1791	* mem_value must be in 0.1us units.
		1792	*/
6084	serge	1793	static uint32_t ilk_compute_cur_wm(const struct intel_crtc_state *cstate,
		1794	const struct intel_plane_state *pstate,
4104	Serge	1795	uint32_t mem_value)
		1796	{
6660	serge	1797	/*
		1798	* We treat the cursor plane as always-on for the purposes of watermark
		1799	* calculation. Until we have two-stage watermark programming merged,
		1800	* this is necessary to avoid flickering.
		1801	*/
		1802	int cpp = 4;
		1803	int width = pstate->visible ? pstate->base.crtc_w : 64;
6084	serge	1804
6660	serge	1805	if (!cstate->base.active)
4104	Serge	1806	return 0;
		1807
6084	serge	1808	return ilk_wm_method2(ilk_pipe_pixel_rate(cstate),
		1809	cstate->base.adjusted_mode.crtc_htotal,
6660	serge	1810	width, cpp, mem_value);
4104	Serge	1811	}
		1812
		1813	/* Only for WM_LP. */
6084	serge	1814	static uint32_t ilk_compute_fbc_wm(const struct intel_crtc_state *cstate,
		1815	const struct intel_plane_state *pstate,
4104	Serge	1816	uint32_t pri_val)
		1817	{
6084	serge	1818	int bpp = pstate->base.fb ? pstate->base.fb->bits_per_pixel / 8 : 0;
		1819
		1820	if (!cstate->base.active \|\| !pstate->visible)
4104	Serge	1821	return 0;
		1822
6084	serge	1823	return ilk_wm_fbc(pri_val, drm_rect_width(&pstate->dst), bpp);
4104	Serge	1824	}
		1825
		1826	static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
		1827	{
4560	Serge	1828	if (INTEL_INFO(dev)->gen >= 8)
		1829	return 3072;
		1830	else if (INTEL_INFO(dev)->gen >= 7)
4104	Serge	1831	return 768;
		1832	else
		1833	return 512;
		1834	}
		1835
5060	serge	1836	static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
		1837	int level, bool is_sprite)
		1838	{
		1839	if (INTEL_INFO(dev)->gen >= 8)
		1840	/* BDW primary/sprite plane watermarks */
		1841	return level == 0 ? 255 : 2047;
		1842	else if (INTEL_INFO(dev)->gen >= 7)
		1843	/* IVB/HSW primary/sprite plane watermarks */
		1844	return level == 0 ? 127 : 1023;
		1845	else if (!is_sprite)
		1846	/* ILK/SNB primary plane watermarks */
		1847	return level == 0 ? 127 : 511;
		1848	else
		1849	/* ILK/SNB sprite plane watermarks */
		1850	return level == 0 ? 63 : 255;
		1851	}
		1852
		1853	static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
		1854	int level)
		1855	{
		1856	if (INTEL_INFO(dev)->gen >= 7)
		1857	return level == 0 ? 63 : 255;
		1858	else
		1859	return level == 0 ? 31 : 63;
		1860	}
		1861
		1862	static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
		1863	{
		1864	if (INTEL_INFO(dev)->gen >= 8)
		1865	return 31;
		1866	else
		1867	return 15;
		1868	}
		1869
4104	Serge	1870	/* Calculate the maximum primary/sprite plane watermark */
		1871	static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
		1872	int level,
		1873	const struct intel_wm_config *config,
		1874	enum intel_ddb_partitioning ddb_partitioning,
		1875	bool is_sprite)
		1876	{
		1877	unsigned int fifo_size = ilk_display_fifo_size(dev);
		1878
		1879	/* if sprites aren't enabled, sprites get nothing */
		1880	if (is_sprite && !config->sprites_enabled)
		1881	return 0;
		1882
		1883	/* HSW allows LP1+ watermarks even with multiple pipes */
		1884	if (level == 0 \|\| config->num_pipes_active > 1) {
		1885	fifo_size /= INTEL_INFO(dev)->num_pipes;
		1886
		1887	/*
		1888	* For some reason the non self refresh
		1889	* FIFO size is only half of the self
		1890	* refresh FIFO size on ILK/SNB.
		1891	*/
		1892	if (INTEL_INFO(dev)->gen <= 6)
		1893	fifo_size /= 2;
		1894	}
		1895
		1896	if (config->sprites_enabled) {
		1897	/* level 0 is always calculated with 1:1 split */
		1898	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
		1899	if (is_sprite)
		1900	fifo_size *= 5;
		1901	fifo_size /= 6;
		1902	} else {
		1903	fifo_size /= 2;
		1904	}
		1905	}
		1906
		1907	/* clamp to max that the registers can hold */
5060	serge	1908	return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
4104	Serge	1909	}
		1910
		1911	/* Calculate the maximum cursor plane watermark */
		1912	static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
		1913	int level,
		1914	const struct intel_wm_config *config)
		1915	{
		1916	/* HSW LP1+ watermarks w/ multiple pipes */
		1917	if (level > 0 && config->num_pipes_active > 1)
		1918	return 64;
		1919
		1920	/* otherwise just report max that registers can hold */
5060	serge	1921	return ilk_cursor_wm_reg_max(dev, level);
4539	Serge	1922	}
4104	Serge	1923
5060	serge	1924	static void ilk_compute_wm_maximums(const struct drm_device *dev,
6084	serge	1925	int level,
		1926	const struct intel_wm_config *config,
		1927	enum intel_ddb_partitioning ddb_partitioning,
4560	Serge	1928	struct ilk_wm_maximums *max)
4104	Serge	1929	{
		1930	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
		1931	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
		1932	max->cur = ilk_cursor_wm_max(dev, level, config);
5060	serge	1933	max->fbc = ilk_fbc_wm_reg_max(dev);
4539	Serge	1934	}
4104	Serge	1935
5060	serge	1936	static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
		1937	int level,
		1938	struct ilk_wm_maximums *max)
		1939	{
		1940	max->pri = ilk_plane_wm_reg_max(dev, level, false);
		1941	max->spr = ilk_plane_wm_reg_max(dev, level, true);
		1942	max->cur = ilk_cursor_wm_reg_max(dev, level);
		1943	max->fbc = ilk_fbc_wm_reg_max(dev);
		1944	}
		1945
4560	Serge	1946	static bool ilk_validate_wm_level(int level,
		1947	const struct ilk_wm_maximums *max,
6084	serge	1948	struct intel_wm_level *result)
4104	Serge	1949	{
		1950	bool ret;
		1951
		1952	/* already determined to be invalid? */
		1953	if (!result->enable)
		1954	return false;
		1955
		1956	result->enable = result->pri_val <= max->pri &&
		1957	result->spr_val <= max->spr &&
		1958	result->cur_val <= max->cur;
		1959
		1960	ret = result->enable;
		1961
		1962	/*
		1963	* HACK until we can pre-compute everything,
		1964	* and thus fail gracefully if LP0 watermarks
		1965	* are exceeded...
		1966	*/
		1967	if (level == 0 && !result->enable) {
		1968	if (result->pri_val > max->pri)
		1969	DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
		1970	level, result->pri_val, max->pri);
		1971	if (result->spr_val > max->spr)
		1972	DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
		1973	level, result->spr_val, max->spr);
		1974	if (result->cur_val > max->cur)
		1975	DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
		1976	level, result->cur_val, max->cur);
		1977
		1978	result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
		1979	result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
		1980	result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
		1981	result->enable = true;
		1982	}
		1983
		1984	return ret;
		1985	}
		1986
5060	serge	1987	static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
6084	serge	1988	const struct intel_crtc *intel_crtc,
4104	Serge	1989	int level,
6084	serge	1990	struct intel_crtc_state *cstate,
4104	Serge	1991	struct intel_wm_level *result)
		1992	{
6084	serge	1993	struct intel_plane *intel_plane;
4104	Serge	1994	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
		1995	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
		1996	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
		1997
		1998	/* WM1+ latency values stored in 0.5us units */
		1999	if (level > 0) {
		2000	pri_latency *= 5;
		2001	spr_latency *= 5;
		2002	cur_latency *= 5;
		2003	}
		2004
6084	serge	2005	for_each_intel_plane_on_crtc(dev_priv->dev, intel_crtc, intel_plane) {
		2006	struct intel_plane_state *pstate =
		2007	to_intel_plane_state(intel_plane->base.state);
		2008
		2009	switch (intel_plane->base.type) {
		2010	case DRM_PLANE_TYPE_PRIMARY:
		2011	result->pri_val = ilk_compute_pri_wm(cstate, pstate,
		2012	pri_latency,
		2013	level);
		2014	result->fbc_val = ilk_compute_fbc_wm(cstate, pstate,
		2015	result->pri_val);
		2016	break;
		2017	case DRM_PLANE_TYPE_OVERLAY:
		2018	result->spr_val = ilk_compute_spr_wm(cstate, pstate,
		2019	spr_latency);
		2020	break;
		2021	case DRM_PLANE_TYPE_CURSOR:
		2022	result->cur_val = ilk_compute_cur_wm(cstate, pstate,
		2023	cur_latency);
		2024	break;
		2025	}
		2026	}
		2027
4104	Serge	2028	result->enable = true;
		2029	}
		2030
		2031	static uint32_t
		2032	hsw_compute_linetime_wm(struct drm_device dev, struct drm_crtc crtc)
		2033	{
3031	serge	2034	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2035	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6084	serge	2036	const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
4104	Serge	2037	u32 linetime, ips_linetime;
3031	serge	2038
6084	serge	2039	if (!intel_crtc->active)
4104	Serge	2040	return 0;
3031	serge	2041
		2042	/* The WM are computed with base on how long it takes to fill a single
		2043	* row at the given clock rate, multiplied by 8.
		2044	* */
6084	serge	2045	linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
		2046	adjusted_mode->crtc_clock);
		2047	ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
		2048	dev_priv->cdclk_freq);
3031	serge	2049
4104	Serge	2050	return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) \|
		2051	PIPE_WM_LINETIME_TIME(linetime);
		2052	}
		2053
5354	serge	2054	static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
4104	Serge	2055	{
		2056	struct drm_i915_private *dev_priv = dev->dev_private;
		2057
5354	serge	2058	if (IS_GEN9(dev)) {
		2059	uint32_t val;
		2060	int ret, i;
		2061	int level, max_level = ilk_wm_max_level(dev);
		2062
		2063	/* read the first set of memory latencies[0:3] */
		2064	val = 0; /* data0 to be programmed to 0 for first set */
		2065	mutex_lock(&dev_priv->rps.hw_lock);
		2066	ret = sandybridge_pcode_read(dev_priv,
		2067	GEN9_PCODE_READ_MEM_LATENCY,
		2068	&val);
		2069	mutex_unlock(&dev_priv->rps.hw_lock);
		2070
		2071	if (ret) {
		2072	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2073	return;
		2074	}
		2075
		2076	wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2077	wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2078	GEN9_MEM_LATENCY_LEVEL_MASK;
		2079	wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2080	GEN9_MEM_LATENCY_LEVEL_MASK;
		2081	wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2082	GEN9_MEM_LATENCY_LEVEL_MASK;
		2083
		2084	/* read the second set of memory latencies[4:7] */
		2085	val = 1; /* data0 to be programmed to 1 for second set */
		2086	mutex_lock(&dev_priv->rps.hw_lock);
		2087	ret = sandybridge_pcode_read(dev_priv,
		2088	GEN9_PCODE_READ_MEM_LATENCY,
		2089	&val);
		2090	mutex_unlock(&dev_priv->rps.hw_lock);
		2091	if (ret) {
		2092	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2093	return;
		2094	}
		2095
		2096	wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2097	wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2098	GEN9_MEM_LATENCY_LEVEL_MASK;
		2099	wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2100	GEN9_MEM_LATENCY_LEVEL_MASK;
		2101	wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2102	GEN9_MEM_LATENCY_LEVEL_MASK;
		2103
		2104	/*
6935	serge	2105	* If a level n (n > 1) has a 0us latency, all levels m (m >= n)
		2106	* need to be disabled. We make sure to sanitize the values out
		2107	* of the punit to satisfy this requirement.
		2108	*/
		2109	for (level = 1; level <= max_level; level++) {
		2110	if (wm[level] == 0) {
		2111	for (i = level + 1; i <= max_level; i++)
		2112	wm[i] = 0;
		2113	break;
		2114	}
		2115	}
		2116
		2117	/*
6084	serge	2118	* WaWmMemoryReadLatency:skl
		2119	*
5354	serge	2120	* punit doesn't take into account the read latency so we need
6935	serge	2121	* to add 2us to the various latency levels we retrieve from the
		2122	* punit when level 0 response data us 0us.
5354	serge	2123	*/
6935	serge	2124	if (wm[0] == 0) {
5354	serge	2125	wm[0] += 2;
6935	serge	2126	for (level = 1; level <= max_level; level++) {
		2127	if (wm[level] == 0)
		2128	break;
5354	serge	2129	wm[level] += 2;
6935	serge	2130	}
		2131	}
5354	serge	2132
		2133	} else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2134	uint64_t sskpd = I915_READ64(MCH_SSKPD);
		2135
		2136	wm[0] = (sskpd >> 56) & 0xFF;
		2137	if (wm[0] == 0)
		2138	wm[0] = sskpd & 0xF;
		2139	wm[1] = (sskpd >> 4) & 0xFF;
		2140	wm[2] = (sskpd >> 12) & 0xFF;
		2141	wm[3] = (sskpd >> 20) & 0x1FF;
		2142	wm[4] = (sskpd >> 32) & 0x1FF;
		2143	} else if (INTEL_INFO(dev)->gen >= 6) {
		2144	uint32_t sskpd = I915_READ(MCH_SSKPD);
		2145
		2146	wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		2147	wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		2148	wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		2149	wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
		2150	} else if (INTEL_INFO(dev)->gen >= 5) {
		2151	uint32_t mltr = I915_READ(MLTR_ILK);
		2152
		2153	/* ILK primary LP0 latency is 700 ns */
		2154	wm[0] = 7;
		2155	wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		2156	wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
		2157	}
		2158	}
		2159
		2160	static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2161	{
		2162	/* ILK sprite LP0 latency is 1300 ns */
		2163	if (INTEL_INFO(dev)->gen == 5)
		2164	wm[0] = 13;
		2165	}
		2166
		2167	static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2168	{
		2169	/* ILK cursor LP0 latency is 1300 ns */
		2170	if (INTEL_INFO(dev)->gen == 5)
		2171	wm[0] = 13;
		2172
		2173	/* WaDoubleCursorLP3Latency:ivb */
		2174	if (IS_IVYBRIDGE(dev))
		2175	wm[3] *= 2;
		2176	}
		2177
5060	serge	2178	int ilk_wm_max_level(const struct drm_device *dev)
4560	Serge	2179	{
		2180	/* how many WM levels are we expecting */
6084	serge	2181	if (INTEL_INFO(dev)->gen >= 9)
5354	serge	2182	return 7;
		2183	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
4560	Serge	2184	return 4;
		2185	else if (INTEL_INFO(dev)->gen >= 6)
		2186	return 3;
		2187	else
		2188	return 2;
		2189	}
		2190
4104	Serge	2191	static void intel_print_wm_latency(struct drm_device *dev,
		2192	const char *name,
5354	serge	2193	const uint16_t wm[8])
4104	Serge	2194	{
4560	Serge	2195	int level, max_level = ilk_wm_max_level(dev);
4104	Serge	2196
		2197	for (level = 0; level <= max_level; level++) {
		2198	unsigned int latency = wm[level];
		2199
		2200	if (latency == 0) {
		2201	DRM_ERROR("%s WM%d latency not provided\n",
		2202	name, level);
		2203	continue;
		2204	}
		2205
5354	serge	2206	/*
		2207	* - latencies are in us on gen9.
		2208	* - before then, WM1+ latency values are in 0.5us units
		2209	*/
		2210	if (IS_GEN9(dev))
		2211	latency *= 10;
		2212	else if (level > 0)
4104	Serge	2213	latency *= 5;
		2214
		2215	DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
		2216	name, level, wm[level],
		2217	latency / 10, latency % 10);
		2218	}
		2219	}
		2220
5060	serge	2221	static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
		2222	uint16_t wm[5], uint16_t min)
4104	Serge	2223	{
5060	serge	2224	int level, max_level = ilk_wm_max_level(dev_priv->dev);
		2225
		2226	if (wm[0] >= min)
		2227	return false;
		2228
		2229	wm[0] = max(wm[0], min);
		2230	for (level = 1; level <= max_level; level++)
		2231	wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
		2232
		2233	return true;
		2234	}
		2235
		2236	static void snb_wm_latency_quirk(struct drm_device *dev)
		2237	{
4104	Serge	2238	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2239	bool changed;
4104	Serge	2240
5060	serge	2241	/*
		2242	* The BIOS provided WM memory latency values are often
		2243	* inadequate for high resolution displays. Adjust them.
		2244	*/
		2245	changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) \|
		2246	ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) \|
		2247	ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
		2248
		2249	if (!changed)
		2250	return;
		2251
		2252	DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
		2253	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2254	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2255	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
		2256	}
		2257
		2258	static void ilk_setup_wm_latency(struct drm_device *dev)
		2259	{
		2260	struct drm_i915_private *dev_priv = dev->dev_private;
		2261
4104	Serge	2262	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
		2263
		2264	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
		2265	sizeof(dev_priv->wm.pri_latency));
		2266	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
		2267	sizeof(dev_priv->wm.pri_latency));
		2268
		2269	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
		2270	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
		2271
		2272	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2273	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2274	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
5060	serge	2275
		2276	if (IS_GEN6(dev))
		2277	snb_wm_latency_quirk(dev);
4104	Serge	2278	}
		2279
5354	serge	2280	static void skl_setup_wm_latency(struct drm_device *dev)
		2281	{
		2282	struct drm_i915_private *dev_priv = dev->dev_private;
		2283
		2284	intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
		2285	intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
		2286	}
		2287
5060	serge	2288	static void ilk_compute_wm_config(struct drm_device *dev,
		2289	struct intel_wm_config *config)
		2290	{
		2291	struct intel_crtc *intel_crtc;
		2292
		2293	/* Compute the currently _active_ config */
		2294	for_each_intel_crtc(dev, intel_crtc) {
		2295	const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
		2296
		2297	if (!wm->pipe_enabled)
		2298	continue;
		2299
		2300	config->sprites_enabled \|= wm->sprites_enabled;
		2301	config->sprites_scaled \|= wm->sprites_scaled;
		2302	config->num_pipes_active++;
4104	Serge	2303	}
4560	Serge	2304	}
4104	Serge	2305
4560	Serge	2306	/* Compute new watermarks for the pipe */
6084	serge	2307	static bool intel_compute_pipe_wm(struct intel_crtc_state *cstate,
4560	Serge	2308	struct intel_pipe_wm *pipe_wm)
		2309	{
6084	serge	2310	struct drm_crtc *crtc = cstate->base.crtc;
4560	Serge	2311	struct drm_device *dev = crtc->dev;
5060	serge	2312	const struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	2313	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2314	struct intel_plane *intel_plane;
		2315	struct intel_plane_state *sprstate = NULL;
4560	Serge	2316	int level, max_level = ilk_wm_max_level(dev);
		2317	/* LP0 watermark maximums depend on this pipe alone */
		2318	struct intel_wm_config config = {
		2319	.num_pipes_active = 1,
		2320	};
		2321	struct ilk_wm_maximums max;
4104	Serge	2322
6084	serge	2323	for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
		2324	if (intel_plane->base.type == DRM_PLANE_TYPE_OVERLAY) {
		2325	sprstate = to_intel_plane_state(intel_plane->base.state);
		2326	break;
		2327	}
		2328	}
4560	Serge	2329
6084	serge	2330	config.sprites_enabled = sprstate->visible;
		2331	config.sprites_scaled = sprstate->visible &&
		2332	(drm_rect_width(&sprstate->dst) != drm_rect_width(&sprstate->src) >> 16 \|\|
		2333	drm_rect_height(&sprstate->dst) != drm_rect_height(&sprstate->src) >> 16);
		2334
		2335	pipe_wm->pipe_enabled = cstate->base.active;
		2336	pipe_wm->sprites_enabled = sprstate->visible;
		2337	pipe_wm->sprites_scaled = config.sprites_scaled;
		2338
4560	Serge	2339	/* ILK/SNB: LP2+ watermarks only w/o sprites */
6084	serge	2340	if (INTEL_INFO(dev)->gen <= 6 && sprstate->visible)
4560	Serge	2341	max_level = 1;
		2342
		2343	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
6084	serge	2344	if (config.sprites_scaled)
4560	Serge	2345	max_level = 0;
		2346
6084	serge	2347	ilk_compute_wm_level(dev_priv, intel_crtc, 0, cstate, &pipe_wm->wm[0]);
4560	Serge	2348
		2349	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2350	pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
		2351
5060	serge	2352	/* LP0 watermarks always use 1/2 DDB partitioning */
		2353	ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
		2354
4560	Serge	2355	/* At least LP0 must be valid */
5060	serge	2356	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
		2357	return false;
		2358
		2359	ilk_compute_wm_reg_maximums(dev, 1, &max);
		2360
		2361	for (level = 1; level <= max_level; level++) {
		2362	struct intel_wm_level wm = {};
		2363
6084	serge	2364	ilk_compute_wm_level(dev_priv, intel_crtc, level, cstate, &wm);
5060	serge	2365
		2366	/*
		2367	* Disable any watermark level that exceeds the
		2368	* register maximums since such watermarks are
		2369	* always invalid.
		2370	*/
		2371	if (!ilk_validate_wm_level(level, &max, &wm))
		2372	break;
		2373
		2374	pipe_wm->wm[level] = wm;
		2375	}
		2376
		2377	return true;
4104	Serge	2378	}
		2379
4560	Serge	2380	/*
		2381	* Merge the watermarks from all active pipes for a specific level.
		2382	*/
		2383	static void ilk_merge_wm_level(struct drm_device *dev,
		2384	int level,
		2385	struct intel_wm_level *ret_wm)
4104	Serge	2386	{
4560	Serge	2387	const struct intel_crtc *intel_crtc;
4104	Serge	2388
5060	serge	2389	ret_wm->enable = true;
4104	Serge	2390
5060	serge	2391	for_each_intel_crtc(dev, intel_crtc) {
		2392	const struct intel_pipe_wm *active = &intel_crtc->wm.active;
		2393	const struct intel_wm_level *wm = &active->wm[level];
		2394
		2395	if (!active->pipe_enabled)
		2396	continue;
		2397
		2398	/*
		2399	* The watermark values may have been used in the past,
		2400	* so we must maintain them in the registers for some
		2401	* time even if the level is now disabled.
		2402	*/
4560	Serge	2403	if (!wm->enable)
5060	serge	2404	ret_wm->enable = false;
4104	Serge	2405
4560	Serge	2406	ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
		2407	ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
		2408	ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
		2409	ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
		2410	}
		2411	}
		2412
		2413	/*
		2414	* Merge all low power watermarks for all active pipes.
		2415	*/
		2416	static void ilk_wm_merge(struct drm_device *dev,
		2417	const struct intel_wm_config *config,
		2418	const struct ilk_wm_maximums *max,
		2419	struct intel_pipe_wm *merged)
		2420	{
6084	serge	2421	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2422	int level, max_level = ilk_wm_max_level(dev);
5060	serge	2423	int last_enabled_level = max_level;
4560	Serge	2424
		2425	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
		2426	if ((INTEL_INFO(dev)->gen <= 6 \|\| IS_IVYBRIDGE(dev)) &&
		2427	config->num_pipes_active > 1)
		2428	return;
		2429
		2430	/* ILK: FBC WM must be disabled always */
		2431	merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
		2432
		2433	/* merge each WM1+ level */
4104	Serge	2434	for (level = 1; level <= max_level; level++) {
4560	Serge	2435	struct intel_wm_level *wm = &merged->wm[level];
		2436
		2437	ilk_merge_wm_level(dev, level, wm);
		2438
5060	serge	2439	if (level > last_enabled_level)
		2440	wm->enable = false;
		2441	else if (!ilk_validate_wm_level(level, max, wm))
		2442	/* make sure all following levels get disabled */
		2443	last_enabled_level = level - 1;
4560	Serge	2444
		2445	/*
		2446	* The spec says it is preferred to disable
		2447	* FBC WMs instead of disabling a WM level.
		2448	*/
		2449	if (wm->fbc_val > max->fbc) {
5060	serge	2450	if (wm->enable)
6084	serge	2451	merged->fbc_wm_enabled = false;
4560	Serge	2452	wm->fbc_val = 0;
4104	Serge	2453	}
		2454	}
		2455
4560	Serge	2456	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
		2457	/*
		2458	* FIXME this is racy. FBC might get enabled later.
		2459	* What we should check here is whether FBC can be
		2460	* enabled sometime later.
		2461	*/
6084	serge	2462	if (IS_GEN5(dev) && !merged->fbc_wm_enabled &&
		2463	intel_fbc_enabled(dev_priv)) {
4560	Serge	2464	for (level = 2; level <= max_level; level++) {
		2465	struct intel_wm_level *wm = &merged->wm[level];
		2466
		2467	wm->enable = false;
		2468	}
		2469	}
		2470	}
		2471
		2472	static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
		2473	{
		2474	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
		2475	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
		2476	}
		2477
		2478	/* The value we need to program into the WM_LPx latency field */
		2479	static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
		2480	{
		2481	struct drm_i915_private *dev_priv = dev->dev_private;
		2482
		2483	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2484	return 2 * level;
		2485	else
		2486	return dev_priv->wm.pri_latency[level];
		2487	}
		2488
		2489	static void ilk_compute_wm_results(struct drm_device *dev,
		2490	const struct intel_pipe_wm *merged,
		2491	enum intel_ddb_partitioning partitioning,
		2492	struct ilk_wm_values *results)
		2493	{
		2494	struct intel_crtc *intel_crtc;
		2495	int level, wm_lp;
		2496
		2497	results->enable_fbc_wm = merged->fbc_wm_enabled;
		2498	results->partitioning = partitioning;
		2499
		2500	/* LP1+ register values */
4104	Serge	2501	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2502	const struct intel_wm_level *r;
		2503
4560	Serge	2504	level = ilk_wm_lp_to_level(wm_lp, merged);
		2505
		2506	r = &merged->wm[level];
4104	Serge	2507
5060	serge	2508	/*
		2509	* Maintain the watermark values even if the level is
		2510	* disabled. Doing otherwise could cause underruns.
		2511	*/
		2512	results->wm_lp[wm_lp - 1] =
4560	Serge	2513	(ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) \|
		2514	(r->pri_val << WM1_LP_SR_SHIFT) \|
		2515	r->cur_val;
		2516
5060	serge	2517	if (r->enable)
		2518	results->wm_lp[wm_lp - 1] \|= WM1_LP_SR_EN;
		2519
4560	Serge	2520	if (INTEL_INFO(dev)->gen >= 8)
		2521	results->wm_lp[wm_lp - 1] \|=
		2522	r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
		2523	else
		2524	results->wm_lp[wm_lp - 1] \|=
		2525	r->fbc_val << WM1_LP_FBC_SHIFT;
		2526
5060	serge	2527	/*
		2528	* Always set WM1S_LP_EN when spr_val != 0, even if the
		2529	* level is disabled. Doing otherwise could cause underruns.
		2530	*/
4560	Serge	2531	if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
		2532	WARN_ON(wm_lp != 1);
		2533	results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN \| r->spr_val;
		2534	} else
6084	serge	2535	results->wm_lp_spr[wm_lp - 1] = r->spr_val;
4104	Serge	2536	}
		2537
4560	Serge	2538	/* LP0 register values */
5060	serge	2539	for_each_intel_crtc(dev, intel_crtc) {
4560	Serge	2540	enum pipe pipe = intel_crtc->pipe;
		2541	const struct intel_wm_level *r =
		2542	&intel_crtc->wm.active.wm[0];
4104	Serge	2543
4560	Serge	2544	if (WARN_ON(!r->enable))
		2545	continue;
		2546
		2547	results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
		2548
		2549	results->wm_pipe[pipe] =
		2550	(r->pri_val << WM0_PIPE_PLANE_SHIFT) \|
		2551	(r->spr_val << WM0_PIPE_SPRITE_SHIFT) \|
		2552	r->cur_val;
4104	Serge	2553	}
		2554	}
		2555
		2556	/* Find the result with the highest level enabled. Check for enable_fbc_wm in
		2557	* case both are at the same level. Prefer r1 in case they're the same. */
4560	Serge	2558	static struct intel_pipe_wm ilk_find_best_result(struct drm_device dev,
		2559	struct intel_pipe_wm *r1,
		2560	struct intel_pipe_wm *r2)
4104	Serge	2561	{
4560	Serge	2562	int level, max_level = ilk_wm_max_level(dev);
		2563	int level1 = 0, level2 = 0;
4104	Serge	2564
4560	Serge	2565	for (level = 1; level <= max_level; level++) {
		2566	if (r1->wm[level].enable)
		2567	level1 = level;
		2568	if (r2->wm[level].enable)
		2569	level2 = level;
4104	Serge	2570	}
		2571
4560	Serge	2572	if (level1 == level2) {
		2573	if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
4104	Serge	2574	return r2;
		2575	else
		2576	return r1;
4560	Serge	2577	} else if (level1 > level2) {
4104	Serge	2578	return r1;
		2579	} else {
		2580	return r2;
		2581	}
		2582	}
		2583
4560	Serge	2584	/* dirty bits used to track which watermarks need changes */
		2585	#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
		2586	#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
		2587	#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
		2588	#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) \| WM_DIRTY_LP(2) \| WM_DIRTY_LP(3))
		2589	#define WM_DIRTY_FBC (1 << 24)
		2590	#define WM_DIRTY_DDB (1 << 25)
		2591
5354	serge	2592	static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
4560	Serge	2593	const struct ilk_wm_values *old,
		2594	const struct ilk_wm_values *new)
		2595	{
		2596	unsigned int dirty = 0;
		2597	enum pipe pipe;
		2598	int wm_lp;
		2599
5354	serge	2600	for_each_pipe(dev_priv, pipe) {
4560	Serge	2601	if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
		2602	dirty \|= WM_DIRTY_LINETIME(pipe);
		2603	/* Must disable LP1+ watermarks too */
		2604	dirty \|= WM_DIRTY_LP_ALL;
		2605	}
		2606
		2607	if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
		2608	dirty \|= WM_DIRTY_PIPE(pipe);
		2609	/* Must disable LP1+ watermarks too */
		2610	dirty \|= WM_DIRTY_LP_ALL;
		2611	}
		2612	}
		2613
		2614	if (old->enable_fbc_wm != new->enable_fbc_wm) {
		2615	dirty \|= WM_DIRTY_FBC;
		2616	/* Must disable LP1+ watermarks too */
		2617	dirty \|= WM_DIRTY_LP_ALL;
		2618	}
		2619
		2620	if (old->partitioning != new->partitioning) {
		2621	dirty \|= WM_DIRTY_DDB;
		2622	/* Must disable LP1+ watermarks too */
		2623	dirty \|= WM_DIRTY_LP_ALL;
		2624	}
		2625
		2626	/* LP1+ watermarks already deemed dirty, no need to continue */
		2627	if (dirty & WM_DIRTY_LP_ALL)
		2628	return dirty;
		2629
		2630	/* Find the lowest numbered LP1+ watermark in need of an update... */
		2631	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2632	if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] \|\|
		2633	old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
		2634	break;
		2635	}
		2636
		2637	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
		2638	for (; wm_lp <= 3; wm_lp++)
		2639	dirty \|= WM_DIRTY_LP(wm_lp);
		2640
		2641	return dirty;
		2642	}
		2643
		2644	static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
		2645	unsigned int dirty)
		2646	{
		2647	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2648	bool changed = false;
		2649
		2650	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
		2651	previous->wm_lp[2] &= ~WM1_LP_SR_EN;
		2652	I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
		2653	changed = true;
		2654	}
		2655	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
		2656	previous->wm_lp[1] &= ~WM1_LP_SR_EN;
		2657	I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
		2658	changed = true;
		2659	}
		2660	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
		2661	previous->wm_lp[0] &= ~WM1_LP_SR_EN;
		2662	I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
		2663	changed = true;
		2664	}
		2665
		2666	/*
		2667	* Don't touch WM1S_LP_EN here.
		2668	* Doing so could cause underruns.
		2669	*/
		2670
		2671	return changed;
		2672	}
		2673
4104	Serge	2674	/*
		2675	* The spec says we shouldn't write when we don't need, because every write
		2676	* causes WMs to be re-evaluated, expending some power.
6084	serge	2677	*/
4560	Serge	2678	static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
		2679	struct ilk_wm_values *results)
4104	Serge	2680	{
4560	Serge	2681	struct drm_device *dev = dev_priv->dev;
		2682	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2683	unsigned int dirty;
4104	Serge	2684	uint32_t val;
3031	serge	2685
5354	serge	2686	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
4560	Serge	2687	if (!dirty)
4104	Serge	2688	return;
		2689
4560	Serge	2690	_ilk_disable_lp_wm(dev_priv, dirty);
4104	Serge	2691
4560	Serge	2692	if (dirty & WM_DIRTY_PIPE(PIPE_A))
4104	Serge	2693	I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
4560	Serge	2694	if (dirty & WM_DIRTY_PIPE(PIPE_B))
4104	Serge	2695	I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
4560	Serge	2696	if (dirty & WM_DIRTY_PIPE(PIPE_C))
4104	Serge	2697	I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
		2698
4560	Serge	2699	if (dirty & WM_DIRTY_LINETIME(PIPE_A))
4104	Serge	2700	I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
4560	Serge	2701	if (dirty & WM_DIRTY_LINETIME(PIPE_B))
4104	Serge	2702	I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
4560	Serge	2703	if (dirty & WM_DIRTY_LINETIME(PIPE_C))
4104	Serge	2704	I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
		2705
4560	Serge	2706	if (dirty & WM_DIRTY_DDB) {
		2707	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
6084	serge	2708	val = I915_READ(WM_MISC);
4560	Serge	2709	if (results->partitioning == INTEL_DDB_PART_1_2)
6084	serge	2710	val &= ~WM_MISC_DATA_PARTITION_5_6;
		2711	else
		2712	val \|= WM_MISC_DATA_PARTITION_5_6;
		2713	I915_WRITE(WM_MISC, val);
4560	Serge	2714	} else {
		2715	val = I915_READ(DISP_ARB_CTL2);
		2716	if (results->partitioning == INTEL_DDB_PART_1_2)
		2717	val &= ~DISP_DATA_PARTITION_5_6;
		2718	else
		2719	val \|= DISP_DATA_PARTITION_5_6;
		2720	I915_WRITE(DISP_ARB_CTL2, val);
		2721	}
4104	Serge	2722	}
		2723
4560	Serge	2724	if (dirty & WM_DIRTY_FBC) {
4104	Serge	2725	val = I915_READ(DISP_ARB_CTL);
		2726	if (results->enable_fbc_wm)
		2727	val &= ~DISP_FBC_WM_DIS;
		2728	else
		2729	val \|= DISP_FBC_WM_DIS;
		2730	I915_WRITE(DISP_ARB_CTL, val);
		2731	}
		2732
4560	Serge	2733	if (dirty & WM_DIRTY_LP(1) &&
		2734	previous->wm_lp_spr[0] != results->wm_lp_spr[0])
4104	Serge	2735	I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
4560	Serge	2736
		2737	if (INTEL_INFO(dev)->gen >= 7) {
		2738	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
6084	serge	2739	I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
4560	Serge	2740	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
6084	serge	2741	I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
4560	Serge	2742	}
4104	Serge	2743
4560	Serge	2744	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
4104	Serge	2745	I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
4560	Serge	2746	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
4104	Serge	2747	I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
4560	Serge	2748	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
4104	Serge	2749	I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
4560	Serge	2750
		2751	dev_priv->wm.hw = *results;
3031	serge	2752	}
		2753
4560	Serge	2754	static bool ilk_disable_lp_wm(struct drm_device *dev)
4104	Serge	2755	{
		2756	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2757
		2758	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
		2759	}
		2760
5354	serge	2761	/*
		2762	* On gen9, we need to allocate Display Data Buffer (DDB) portions to the
		2763	* different active planes.
		2764	*/
		2765
		2766	#define SKL_DDB_SIZE 896 /* in blocks */
6084	serge	2767	#define BXT_DDB_SIZE 512
5354	serge	2768
		2769	static void
		2770	skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
		2771	struct drm_crtc *for_crtc,
		2772	const struct intel_wm_config *config,
		2773	const struct skl_pipe_wm_parameters *params,
		2774	struct skl_ddb_entry alloc / out */)
		2775	{
		2776	struct drm_crtc *crtc;
		2777	unsigned int pipe_size, ddb_size;
		2778	int nth_active_pipe;
		2779
		2780	if (!params->active) {
		2781	alloc->start = 0;
		2782	alloc->end = 0;
		2783	return;
		2784	}
		2785
6084	serge	2786	if (IS_BROXTON(dev))
		2787	ddb_size = BXT_DDB_SIZE;
		2788	else
		2789	ddb_size = SKL_DDB_SIZE;
5354	serge	2790
		2791	ddb_size -= 4; /* 4 blocks for bypass path allocation */
		2792
		2793	nth_active_pipe = 0;
		2794	for_each_crtc(dev, crtc) {
6084	serge	2795	if (!to_intel_crtc(crtc)->active)
5354	serge	2796	continue;
		2797
		2798	if (crtc == for_crtc)
		2799	break;
		2800
		2801	nth_active_pipe++;
		2802	}
		2803
		2804	pipe_size = ddb_size / config->num_pipes_active;
		2805	alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
		2806	alloc->end = alloc->start + pipe_size;
		2807	}
		2808
		2809	static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
		2810	{
		2811	if (config->num_pipes_active == 1)
		2812	return 32;
		2813
		2814	return 8;
		2815	}
		2816
		2817	static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
		2818	{
		2819	entry->start = reg & 0x3ff;
		2820	entry->end = (reg >> 16) & 0x3ff;
		2821	if (entry->end)
		2822	entry->end += 1;
		2823	}
		2824
		2825	void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
		2826	struct skl_ddb_allocation ddb / out */)
		2827	{
		2828	enum pipe pipe;
		2829	int plane;
		2830	u32 val;
		2831
6084	serge	2832	memset(ddb, 0, sizeof(*ddb));
		2833
5354	serge	2834	for_each_pipe(dev_priv, pipe) {
6084	serge	2835	if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_PIPE(pipe)))
		2836	continue;
		2837
		2838	for_each_plane(dev_priv, pipe, plane) {
5354	serge	2839	val = I915_READ(PLANE_BUF_CFG(pipe, plane));
		2840	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
		2841	val);
		2842	}
		2843
		2844	val = I915_READ(CUR_BUF_CFG(pipe));
6084	serge	2845	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][PLANE_CURSOR],
		2846	val);
5354	serge	2847	}
		2848	}
		2849
		2850	static unsigned int
6084	serge	2851	skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p, int y)
5354	serge	2852	{
6084	serge	2853
		2854	/* for planar format */
		2855	if (p->y_bytes_per_pixel) {
		2856	if (y) /* y-plane data rate */
		2857	return p->horiz_pixels * p->vert_pixels * p->y_bytes_per_pixel;
		2858	else /* uv-plane data rate */
		2859	return (p->horiz_pixels/2) * (p->vert_pixels/2) * p->bytes_per_pixel;
		2860	}
		2861
		2862	/* for packed formats */
5354	serge	2863	return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
		2864	}
		2865
		2866	/*
		2867	* We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
		2868	* a 8192x4096@32bpp framebuffer:
		2869	* 3 * 4096 * 8192 * 4 < 2^32
		2870	*/
		2871	static unsigned int
		2872	skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
		2873	const struct skl_pipe_wm_parameters *params)
		2874	{
		2875	unsigned int total_data_rate = 0;
		2876	int plane;
		2877
		2878	for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
		2879	const struct intel_plane_wm_parameters *p;
		2880
		2881	p = ¶ms->plane[plane];
		2882	if (!p->enabled)
		2883	continue;
		2884
6084	serge	2885	total_data_rate += skl_plane_relative_data_rate(p, 0); /* packed/uv */
		2886	if (p->y_bytes_per_pixel) {
		2887	total_data_rate += skl_plane_relative_data_rate(p, 1); /* y-plane */
		2888	}
5354	serge	2889	}
		2890
		2891	return total_data_rate;
		2892	}
		2893
		2894	static void
		2895	skl_allocate_pipe_ddb(struct drm_crtc *crtc,
		2896	const struct intel_wm_config *config,
		2897	const struct skl_pipe_wm_parameters *params,
		2898	struct skl_ddb_allocation ddb / out */)
		2899	{
		2900	struct drm_device *dev = crtc->dev;
6084	serge	2901	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	2902	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2903	enum pipe pipe = intel_crtc->pipe;
		2904	struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
		2905	uint16_t alloc_size, start, cursor_blocks;
6084	serge	2906	uint16_t minimum[I915_MAX_PLANES];
		2907	uint16_t y_minimum[I915_MAX_PLANES];
5354	serge	2908	unsigned int total_data_rate;
		2909	int plane;
		2910
		2911	skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
		2912	alloc_size = skl_ddb_entry_size(alloc);
		2913	if (alloc_size == 0) {
		2914	memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
6084	serge	2915	memset(&ddb->plane[pipe][PLANE_CURSOR], 0,
		2916	sizeof(ddb->plane[pipe][PLANE_CURSOR]));
5354	serge	2917	return;
		2918	}
		2919
		2920	cursor_blocks = skl_cursor_allocation(config);
6084	serge	2921	ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks;
		2922	ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
5354	serge	2923
		2924	alloc_size -= cursor_blocks;
		2925	alloc->end -= cursor_blocks;
		2926
6084	serge	2927	/* 1. Allocate the mininum required blocks for each active plane */
		2928	for_each_plane(dev_priv, pipe, plane) {
		2929	const struct intel_plane_wm_parameters *p;
		2930
		2931	p = ¶ms->plane[plane];
		2932	if (!p->enabled)
		2933	continue;
		2934
		2935	minimum[plane] = 8;
		2936	alloc_size -= minimum[plane];
		2937	y_minimum[plane] = p->y_bytes_per_pixel ? 8 : 0;
		2938	alloc_size -= y_minimum[plane];
		2939	}
		2940
		2941	/*
		2942	* 2. Distribute the remaining space in proportion to the amount of
		2943	* data each plane needs to fetch from memory.
5354	serge	2944	*
		2945	* FIXME: we may not allocate every single block here.
6084	serge	2946	*/
5354	serge	2947	total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
		2948
		2949	start = alloc->start;
		2950	for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
		2951	const struct intel_plane_wm_parameters *p;
6084	serge	2952	unsigned int data_rate, y_data_rate;
		2953	uint16_t plane_blocks, y_plane_blocks = 0;
5354	serge	2954
		2955	p = ¶ms->plane[plane];
		2956	if (!p->enabled)
		2957	continue;
		2958
6084	serge	2959	data_rate = skl_plane_relative_data_rate(p, 0);
5354	serge	2960
		2961	/*
6084	serge	2962	* allocation for (packed formats) or (uv-plane part of planar format):
5354	serge	2963	* promote the expression to 64 bits to avoid overflowing, the
		2964	* result is < available as data_rate / total_data_rate < 1
		2965	*/
6084	serge	2966	plane_blocks = minimum[plane];
		2967	plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
		2968	total_data_rate);
5354	serge	2969
		2970	ddb->plane[pipe][plane].start = start;
		2971	ddb->plane[pipe][plane].end = start + plane_blocks;
		2972
		2973	start += plane_blocks;
6084	serge	2974
		2975	/*
		2976	* allocation for y_plane part of planar format:
		2977	*/
		2978	if (p->y_bytes_per_pixel) {
		2979	y_data_rate = skl_plane_relative_data_rate(p, 1);
		2980	y_plane_blocks = y_minimum[plane];
		2981	y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
		2982	total_data_rate);
		2983
		2984	ddb->y_plane[pipe][plane].start = start;
		2985	ddb->y_plane[pipe][plane].end = start + y_plane_blocks;
		2986
		2987	start += y_plane_blocks;
		2988	}
		2989
5354	serge	2990	}
		2991
		2992	}
		2993
6084	serge	2994	static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
5354	serge	2995	{
		2996	/* TODO: Take into account the scalers once we support them */
6084	serge	2997	return config->base.adjusted_mode.crtc_clock;
5354	serge	2998	}
		2999
		3000	/*
		3001	* The max latency should be 257 (max the punit can code is 255 and we add 2us
		3002	* for the read latency) and bytes_per_pixel should always be <= 8, so that
		3003	* should allow pixel_rate up to ~2 GHz which seems sufficient since max
		3004	* 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
		3005	*/
		3006	static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		3007	uint32_t latency)
		3008	{
		3009	uint32_t wm_intermediate_val, ret;
		3010
		3011	if (latency == 0)
		3012	return UINT_MAX;
		3013
6084	serge	3014	wm_intermediate_val = latency * pixel_rate * bytes_per_pixel / 512;
5354	serge	3015	ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
		3016
		3017	return ret;
		3018	}
		3019
		3020	static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		3021	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
6084	serge	3022	uint64_t tiling, uint32_t latency)
5354	serge	3023	{
6084	serge	3024	uint32_t ret;
		3025	uint32_t plane_bytes_per_line, plane_blocks_per_line;
		3026	uint32_t wm_intermediate_val;
5354	serge	3027
		3028	if (latency == 0)
		3029	return UINT_MAX;
		3030
		3031	plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
6084	serge	3032
		3033	if (tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3034	tiling == I915_FORMAT_MOD_Yf_TILED) {
		3035	plane_bytes_per_line *= 4;
		3036	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
		3037	plane_blocks_per_line /= 4;
		3038	} else {
		3039	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
		3040	}
		3041
5354	serge	3042	wm_intermediate_val = latency * pixel_rate;
		3043	ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
6084	serge	3044	plane_blocks_per_line;
5354	serge	3045
		3046	return ret;
		3047	}
		3048
		3049	static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
		3050	const struct intel_crtc *intel_crtc)
		3051	{
		3052	struct drm_device *dev = intel_crtc->base.dev;
		3053	struct drm_i915_private *dev_priv = dev->dev_private;
		3054	const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3055	enum pipe pipe = intel_crtc->pipe;
		3056
		3057	if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
		3058	sizeof(new_ddb->plane[pipe])))
		3059	return true;
		3060
6084	serge	3061	if (memcmp(&new_ddb->plane[pipe][PLANE_CURSOR], &cur_ddb->plane[pipe][PLANE_CURSOR],
		3062	sizeof(new_ddb->plane[pipe][PLANE_CURSOR])))
5354	serge	3063	return true;
		3064
		3065	return false;
		3066	}
		3067
		3068	static void skl_compute_wm_global_parameters(struct drm_device *dev,
		3069	struct intel_wm_config *config)
		3070	{
		3071	struct drm_crtc *crtc;
		3072	struct drm_plane *plane;
		3073
		3074	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
6084	serge	3075	config->num_pipes_active += to_intel_crtc(crtc)->active;
5354	serge	3076
		3077	/* FIXME: I don't think we need those two global parameters on SKL */
		3078	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		3079	struct intel_plane *intel_plane = to_intel_plane(plane);
		3080
		3081	config->sprites_enabled \|= intel_plane->wm.enabled;
		3082	config->sprites_scaled \|= intel_plane->wm.scaled;
		3083	}
		3084	}
		3085
		3086	static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
		3087	struct skl_pipe_wm_parameters *p)
		3088	{
		3089	struct drm_device *dev = crtc->dev;
		3090	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3091	enum pipe pipe = intel_crtc->pipe;
		3092	struct drm_plane *plane;
6084	serge	3093	struct drm_framebuffer *fb;
5354	serge	3094	int i = 1; /* Index for sprite planes start */
		3095
6084	serge	3096	p->active = intel_crtc->active;
5354	serge	3097	if (p->active) {
6084	serge	3098	p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
		3099	p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config);
5354	serge	3100
6084	serge	3101	fb = crtc->primary->state->fb;
		3102	/* For planar: Bpp is for uv plane, y_Bpp is for y plane */
		3103	if (fb) {
		3104	p->plane[0].enabled = true;
		3105	p->plane[0].bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
		3106	drm_format_plane_cpp(fb->pixel_format, 1) :
		3107	drm_format_plane_cpp(fb->pixel_format, 0);
		3108	p->plane[0].y_bytes_per_pixel = fb->pixel_format == DRM_FORMAT_NV12 ?
		3109	drm_format_plane_cpp(fb->pixel_format, 0) : 0;
		3110	p->plane[0].tiling = fb->modifier[0];
		3111	} else {
		3112	p->plane[0].enabled = false;
		3113	p->plane[0].bytes_per_pixel = 0;
		3114	p->plane[0].y_bytes_per_pixel = 0;
		3115	p->plane[0].tiling = DRM_FORMAT_MOD_NONE;
		3116	}
		3117	p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w;
		3118	p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h;
		3119	p->plane[0].rotation = crtc->primary->state->rotation;
5354	serge	3120
6084	serge	3121	fb = crtc->cursor->state->fb;
		3122	p->plane[PLANE_CURSOR].y_bytes_per_pixel = 0;
		3123	if (fb) {
		3124	p->plane[PLANE_CURSOR].enabled = true;
		3125	p->plane[PLANE_CURSOR].bytes_per_pixel = fb->bits_per_pixel / 8;
		3126	p->plane[PLANE_CURSOR].horiz_pixels = crtc->cursor->state->crtc_w;
		3127	p->plane[PLANE_CURSOR].vert_pixels = crtc->cursor->state->crtc_h;
		3128	} else {
		3129	p->plane[PLANE_CURSOR].enabled = false;
		3130	p->plane[PLANE_CURSOR].bytes_per_pixel = 0;
		3131	p->plane[PLANE_CURSOR].horiz_pixels = 64;
		3132	p->plane[PLANE_CURSOR].vert_pixels = 64;
		3133	}
5354	serge	3134	}
		3135
		3136	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		3137	struct intel_plane *intel_plane = to_intel_plane(plane);
		3138
6084	serge	3139	if (intel_plane->pipe == pipe &&
		3140	plane->type == DRM_PLANE_TYPE_OVERLAY)
5354	serge	3141	p->plane[i++] = intel_plane->wm;
		3142	}
		3143	}
		3144
6084	serge	3145	static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
		3146	struct skl_pipe_wm_parameters *p,
5354	serge	3147	struct intel_plane_wm_parameters *p_params,
		3148	uint16_t ddb_allocation,
6084	serge	3149	int level,
5354	serge	3150	uint16_t out_blocks, / out */
		3151	uint8_t out_lines / out */)
		3152	{
6084	serge	3153	uint32_t latency = dev_priv->wm.skl_latency[level];
		3154	uint32_t method1, method2;
		3155	uint32_t plane_bytes_per_line, plane_blocks_per_line;
		3156	uint32_t res_blocks, res_lines;
		3157	uint32_t selected_result;
		3158	uint8_t bytes_per_pixel;
5354	serge	3159
6084	serge	3160	if (latency == 0 \|\| !p->active \|\| !p_params->enabled)
5354	serge	3161	return false;
		3162
6084	serge	3163	bytes_per_pixel = p_params->y_bytes_per_pixel ?
		3164	p_params->y_bytes_per_pixel :
		3165	p_params->bytes_per_pixel;
5354	serge	3166	method1 = skl_wm_method1(p->pixel_rate,
6084	serge	3167	bytes_per_pixel,
		3168	latency);
5354	serge	3169	method2 = skl_wm_method2(p->pixel_rate,
		3170	p->pipe_htotal,
		3171	p_params->horiz_pixels,
6084	serge	3172	bytes_per_pixel,
		3173	p_params->tiling,
		3174	latency);
5354	serge	3175
6084	serge	3176	plane_bytes_per_line = p_params->horiz_pixels * bytes_per_pixel;
		3177	plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
5354	serge	3178
6084	serge	3179	if (p_params->tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3180	p_params->tiling == I915_FORMAT_MOD_Yf_TILED) {
		3181	uint32_t min_scanlines = 4;
		3182	uint32_t y_tile_minimum;
		3183	if (intel_rotation_90_or_270(p_params->rotation)) {
		3184	switch (p_params->bytes_per_pixel) {
		3185	case 1:
		3186	min_scanlines = 16;
		3187	break;
		3188	case 2:
		3189	min_scanlines = 8;
		3190	break;
		3191	case 8:
		3192	WARN(1, "Unsupported pixel depth for rotation");
		3193	}
		3194	}
		3195	y_tile_minimum = plane_blocks_per_line * min_scanlines;
		3196	selected_result = max(method2, y_tile_minimum);
		3197	} else {
		3198	if ((ddb_allocation / plane_blocks_per_line) >= 1)
		3199	selected_result = min(method1, method2);
		3200	else
		3201	selected_result = method1;
		3202	}
5354	serge	3203
6084	serge	3204	res_blocks = selected_result + 1;
		3205	res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
5354	serge	3206
6084	serge	3207	if (level >= 1 && level <= 7) {
		3208	if (p_params->tiling == I915_FORMAT_MOD_Y_TILED \|\|
		3209	p_params->tiling == I915_FORMAT_MOD_Yf_TILED)
		3210	res_lines += 4;
		3211	else
		3212	res_blocks++;
		3213	}
		3214
		3215	if (res_blocks >= ddb_allocation \|\| res_lines > 31)
5354	serge	3216	return false;
		3217
		3218	*out_blocks = res_blocks;
		3219	*out_lines = res_lines;
		3220
		3221	return true;
		3222	}
		3223
		3224	static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
		3225	struct skl_ddb_allocation *ddb,
		3226	struct skl_pipe_wm_parameters *p,
		3227	enum pipe pipe,
		3228	int level,
		3229	int num_planes,
		3230	struct skl_wm_level *result)
		3231	{
		3232	uint16_t ddb_blocks;
		3233	int i;
		3234
		3235	for (i = 0; i < num_planes; i++) {
		3236	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
		3237
6084	serge	3238	result->plane_en[i] = skl_compute_plane_wm(dev_priv,
		3239	p, &p->plane[i],
5354	serge	3240	ddb_blocks,
6084	serge	3241	level,
5354	serge	3242	&result->plane_res_b[i],
		3243	&result->plane_res_l[i]);
		3244	}
		3245
6084	serge	3246	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][PLANE_CURSOR]);
		3247	result->plane_en[PLANE_CURSOR] = skl_compute_plane_wm(dev_priv, p,
		3248	&p->plane[PLANE_CURSOR],
		3249	ddb_blocks, level,
		3250	&result->plane_res_b[PLANE_CURSOR],
		3251	&result->plane_res_l[PLANE_CURSOR]);
5354	serge	3252	}
		3253
		3254	static uint32_t
		3255	skl_compute_linetime_wm(struct drm_crtc crtc, struct skl_pipe_wm_parameters p)
		3256	{
6084	serge	3257	if (!to_intel_crtc(crtc)->active)
5354	serge	3258	return 0;
		3259
6084	serge	3260	if (WARN_ON(p->pixel_rate == 0))
		3261	return 0;
		3262
5354	serge	3263	return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
		3264	}
		3265
		3266	static void skl_compute_transition_wm(struct drm_crtc *crtc,
		3267	struct skl_pipe_wm_parameters *params,
		3268	struct skl_wm_level trans_wm / out */)
		3269	{
		3270	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3271	int i;
		3272
		3273	if (!params->active)
		3274	return;
		3275
		3276	/* Until we know more, just disable transition WMs */
		3277	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3278	trans_wm->plane_en[i] = false;
6084	serge	3279	trans_wm->plane_en[PLANE_CURSOR] = false;
5354	serge	3280	}
		3281
		3282	static void skl_compute_pipe_wm(struct drm_crtc *crtc,
		3283	struct skl_ddb_allocation *ddb,
		3284	struct skl_pipe_wm_parameters *params,
		3285	struct skl_pipe_wm *pipe_wm)
		3286	{
		3287	struct drm_device *dev = crtc->dev;
		3288	const struct drm_i915_private *dev_priv = dev->dev_private;
		3289	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3290	int level, max_level = ilk_wm_max_level(dev);
		3291
		3292	for (level = 0; level <= max_level; level++) {
		3293	skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
		3294	level, intel_num_planes(intel_crtc),
		3295	&pipe_wm->wm[level]);
		3296	}
		3297	pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
		3298
		3299	skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
		3300	}
		3301
		3302	static void skl_compute_wm_results(struct drm_device *dev,
		3303	struct skl_pipe_wm_parameters *p,
		3304	struct skl_pipe_wm *p_wm,
		3305	struct skl_wm_values *r,
		3306	struct intel_crtc *intel_crtc)
		3307	{
		3308	int level, max_level = ilk_wm_max_level(dev);
		3309	enum pipe pipe = intel_crtc->pipe;
		3310	uint32_t temp;
		3311	int i;
		3312
		3313	for (level = 0; level <= max_level; level++) {
		3314	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3315	temp = 0;
		3316
		3317	temp \|= p_wm->wm[level].plane_res_l[i] <<
		3318	PLANE_WM_LINES_SHIFT;
		3319	temp \|= p_wm->wm[level].plane_res_b[i];
		3320	if (p_wm->wm[level].plane_en[i])
		3321	temp \|= PLANE_WM_EN;
		3322
		3323	r->plane[pipe][i][level] = temp;
		3324	}
		3325
		3326	temp = 0;
		3327
6084	serge	3328	temp \|= p_wm->wm[level].plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
		3329	temp \|= p_wm->wm[level].plane_res_b[PLANE_CURSOR];
5354	serge	3330
6084	serge	3331	if (p_wm->wm[level].plane_en[PLANE_CURSOR])
5354	serge	3332	temp \|= PLANE_WM_EN;
		3333
6084	serge	3334	r->plane[pipe][PLANE_CURSOR][level] = temp;
5354	serge	3335
		3336	}
		3337
		3338	/* transition WMs */
		3339	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3340	temp = 0;
		3341	temp \|= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
		3342	temp \|= p_wm->trans_wm.plane_res_b[i];
		3343	if (p_wm->trans_wm.plane_en[i])
		3344	temp \|= PLANE_WM_EN;
		3345
		3346	r->plane_trans[pipe][i] = temp;
		3347	}
		3348
		3349	temp = 0;
6084	serge	3350	temp \|= p_wm->trans_wm.plane_res_l[PLANE_CURSOR] << PLANE_WM_LINES_SHIFT;
		3351	temp \|= p_wm->trans_wm.plane_res_b[PLANE_CURSOR];
		3352	if (p_wm->trans_wm.plane_en[PLANE_CURSOR])
5354	serge	3353	temp \|= PLANE_WM_EN;
		3354
6084	serge	3355	r->plane_trans[pipe][PLANE_CURSOR] = temp;
5354	serge	3356
		3357	r->wm_linetime[pipe] = p_wm->linetime;
		3358	}
		3359
		3360	static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
		3361	const struct skl_ddb_entry *entry)
		3362	{
		3363	if (entry->end)
		3364	I915_WRITE(reg, (entry->end - 1) << 16 \| entry->start);
		3365	else
		3366	I915_WRITE(reg, 0);
		3367	}
		3368
		3369	static void skl_write_wm_values(struct drm_i915_private *dev_priv,
		3370	const struct skl_wm_values *new)
		3371	{
		3372	struct drm_device *dev = dev_priv->dev;
		3373	struct intel_crtc *crtc;
		3374
		3375	list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
		3376	int i, level, max_level = ilk_wm_max_level(dev);
		3377	enum pipe pipe = crtc->pipe;
		3378
		3379	if (!new->dirty[pipe])
		3380	continue;
		3381
		3382	I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
		3383
		3384	for (level = 0; level <= max_level; level++) {
		3385	for (i = 0; i < intel_num_planes(crtc); i++)
		3386	I915_WRITE(PLANE_WM(pipe, i, level),
		3387	new->plane[pipe][i][level]);
		3388	I915_WRITE(CUR_WM(pipe, level),
6084	serge	3389	new->plane[pipe][PLANE_CURSOR][level]);
5354	serge	3390	}
		3391	for (i = 0; i < intel_num_planes(crtc); i++)
		3392	I915_WRITE(PLANE_WM_TRANS(pipe, i),
		3393	new->plane_trans[pipe][i]);
6084	serge	3394	I915_WRITE(CUR_WM_TRANS(pipe),
		3395	new->plane_trans[pipe][PLANE_CURSOR]);
5354	serge	3396
6084	serge	3397	for (i = 0; i < intel_num_planes(crtc); i++) {
5354	serge	3398	skl_ddb_entry_write(dev_priv,
		3399	PLANE_BUF_CFG(pipe, i),
		3400	&new->ddb.plane[pipe][i]);
6084	serge	3401	skl_ddb_entry_write(dev_priv,
		3402	PLANE_NV12_BUF_CFG(pipe, i),
		3403	&new->ddb.y_plane[pipe][i]);
		3404	}
5354	serge	3405
		3406	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
6084	serge	3407	&new->ddb.plane[pipe][PLANE_CURSOR]);
5354	serge	3408	}
		3409	}
		3410
		3411	/*
		3412	* When setting up a new DDB allocation arrangement, we need to correctly
		3413	* sequence the times at which the new allocations for the pipes are taken into
		3414	* account or we'll have pipes fetching from space previously allocated to
		3415	* another pipe.
		3416	*
		3417	* Roughly the sequence looks like:
		3418	* 1. re-allocate the pipe(s) with the allocation being reduced and not
		3419	* overlapping with a previous light-up pipe (another way to put it is:
		3420	* pipes with their new allocation strickly included into their old ones).
		3421	* 2. re-allocate the other pipes that get their allocation reduced
		3422	* 3. allocate the pipes having their allocation increased
		3423	*
		3424	* Steps 1. and 2. are here to take care of the following case:
		3425	* - Initially DDB looks like this:
		3426	* \| B \| C \|
		3427	* - enable pipe A.
		3428	* - pipe B has a reduced DDB allocation that overlaps with the old pipe C
		3429	* allocation
		3430	* \| A \| B \| C \|
		3431	*
		3432	* We need to sequence the re-allocation: C, B, A (and not B, C, A).
		3433	*/
		3434
		3435	static void
		3436	skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
		3437	{
		3438	int plane;
		3439
		3440	DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
		3441
6084	serge	3442	for_each_plane(dev_priv, pipe, plane) {
5354	serge	3443	I915_WRITE(PLANE_SURF(pipe, plane),
		3444	I915_READ(PLANE_SURF(pipe, plane)));
		3445	}
		3446	I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
		3447	}
		3448
		3449	static bool
		3450	skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
		3451	const struct skl_ddb_allocation *new,
		3452	enum pipe pipe)
		3453	{
		3454	uint16_t old_size, new_size;
		3455
		3456	old_size = skl_ddb_entry_size(&old->pipe[pipe]);
		3457	new_size = skl_ddb_entry_size(&new->pipe[pipe]);
		3458
		3459	return old_size != new_size &&
		3460	new->pipe[pipe].start >= old->pipe[pipe].start &&
		3461	new->pipe[pipe].end <= old->pipe[pipe].end;
		3462	}
		3463
		3464	static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
		3465	struct skl_wm_values *new_values)
		3466	{
		3467	struct drm_device *dev = dev_priv->dev;
		3468	struct skl_ddb_allocation cur_ddb, new_ddb;
6084	serge	3469	bool reallocated[I915_MAX_PIPES] = {};
5354	serge	3470	struct intel_crtc *crtc;
		3471	enum pipe pipe;
		3472
		3473	new_ddb = &new_values->ddb;
		3474	cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3475
		3476	/*
		3477	* First pass: flush the pipes with the new allocation contained into
		3478	* the old space.
		3479	*
		3480	* We'll wait for the vblank on those pipes to ensure we can safely
		3481	* re-allocate the freed space without this pipe fetching from it.
		3482	*/
		3483	for_each_intel_crtc(dev, crtc) {
		3484	if (!crtc->active)
		3485	continue;
		3486
		3487	pipe = crtc->pipe;
		3488
		3489	if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
		3490	continue;
		3491
		3492	skl_wm_flush_pipe(dev_priv, pipe, 1);
		3493	intel_wait_for_vblank(dev, pipe);
		3494
		3495	reallocated[pipe] = true;
		3496	}
		3497
		3498
		3499	/*
		3500	* Second pass: flush the pipes that are having their allocation
		3501	* reduced, but overlapping with a previous allocation.
		3502	*
		3503	* Here as well we need to wait for the vblank to make sure the freed
		3504	* space is not used anymore.
		3505	*/
		3506	for_each_intel_crtc(dev, crtc) {
		3507	if (!crtc->active)
		3508	continue;
		3509
		3510	pipe = crtc->pipe;
		3511
		3512	if (reallocated[pipe])
		3513	continue;
		3514
		3515	if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
		3516	skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
		3517	skl_wm_flush_pipe(dev_priv, pipe, 2);
		3518	intel_wait_for_vblank(dev, pipe);
6084	serge	3519	reallocated[pipe] = true;
5354	serge	3520	}
		3521	}
		3522
		3523	/*
		3524	* Third pass: flush the pipes that got more space allocated.
		3525	*
		3526	* We don't need to actively wait for the update here, next vblank
		3527	* will just get more DDB space with the correct WM values.
		3528	*/
		3529	for_each_intel_crtc(dev, crtc) {
		3530	if (!crtc->active)
		3531	continue;
		3532
		3533	pipe = crtc->pipe;
		3534
		3535	/*
		3536	* At this point, only the pipes more space than before are
		3537	* left to re-allocate.
		3538	*/
		3539	if (reallocated[pipe])
		3540	continue;
		3541
		3542	skl_wm_flush_pipe(dev_priv, pipe, 3);
		3543	}
		3544	}
		3545
		3546	static bool skl_update_pipe_wm(struct drm_crtc *crtc,
		3547	struct skl_pipe_wm_parameters *params,
		3548	struct intel_wm_config *config,
		3549	struct skl_ddb_allocation ddb, / out */
		3550	struct skl_pipe_wm pipe_wm / out */)
		3551	{
		3552	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3553
		3554	skl_compute_wm_pipe_parameters(crtc, params);
		3555	skl_allocate_pipe_ddb(crtc, config, params, ddb);
		3556	skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
		3557
		3558	if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
		3559	return false;
		3560
		3561	intel_crtc->wm.skl_active = *pipe_wm;
6084	serge	3562
5354	serge	3563	return true;
		3564	}
		3565
		3566	static void skl_update_other_pipe_wm(struct drm_device *dev,
		3567	struct drm_crtc *crtc,
		3568	struct intel_wm_config *config,
		3569	struct skl_wm_values *r)
		3570	{
		3571	struct intel_crtc *intel_crtc;
		3572	struct intel_crtc *this_crtc = to_intel_crtc(crtc);
		3573
		3574	/*
		3575	* If the WM update hasn't changed the allocation for this_crtc (the
		3576	* crtc we are currently computing the new WM values for), other
		3577	* enabled crtcs will keep the same allocation and we don't need to
		3578	* recompute anything for them.
		3579	*/
		3580	if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
		3581	return;
		3582
		3583	/*
		3584	* Otherwise, because of this_crtc being freshly enabled/disabled, the
		3585	* other active pipes need new DDB allocation and WM values.
		3586	*/
		3587	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
		3588	base.head) {
		3589	struct skl_pipe_wm_parameters params = {};
		3590	struct skl_pipe_wm pipe_wm = {};
		3591	bool wm_changed;
		3592
		3593	if (this_crtc->pipe == intel_crtc->pipe)
		3594	continue;
		3595
		3596	if (!intel_crtc->active)
		3597	continue;
		3598
		3599	wm_changed = skl_update_pipe_wm(&intel_crtc->base,
		3600	¶ms, config,
		3601	&r->ddb, &pipe_wm);
		3602
		3603	/*
		3604	* If we end up re-computing the other pipe WM values, it's
		3605	* because it was really needed, so we expect the WM values to
		3606	* be different.
6084	serge	3607	*/
5354	serge	3608	WARN_ON(!wm_changed);
		3609
		3610	skl_compute_wm_results(dev, ¶ms, &pipe_wm, r, intel_crtc);
		3611	r->dirty[intel_crtc->pipe] = true;
		3612	}
		3613	}
		3614
6084	serge	3615	static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe)
		3616	{
		3617	watermarks->wm_linetime[pipe] = 0;
		3618	memset(watermarks->plane[pipe], 0,
		3619	sizeof(uint32_t) * 8 * I915_MAX_PLANES);
		3620	memset(watermarks->plane_trans[pipe],
		3621	0, sizeof(uint32_t) * I915_MAX_PLANES);
		3622	watermarks->plane_trans[pipe][PLANE_CURSOR] = 0;
		3623
		3624	/* Clear ddb entries for pipe */
		3625	memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry));
		3626	memset(&watermarks->ddb.plane[pipe], 0,
		3627	sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
		3628	memset(&watermarks->ddb.y_plane[pipe], 0,
		3629	sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
		3630	memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0,
		3631	sizeof(struct skl_ddb_entry));
		3632
		3633	}
		3634
5354	serge	3635	static void skl_update_wm(struct drm_crtc *crtc)
		3636	{
		3637	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3638	struct drm_device *dev = crtc->dev;
		3639	struct drm_i915_private *dev_priv = dev->dev_private;
		3640	struct skl_pipe_wm_parameters params = {};
		3641	struct skl_wm_values *results = &dev_priv->wm.skl_results;
		3642	struct skl_pipe_wm pipe_wm = {};
		3643	struct intel_wm_config config = {};
		3644
		3645
6084	serge	3646	/* Clear all dirty flags */
		3647	memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES);
		3648
		3649	skl_clear_wm(results, intel_crtc->pipe);
		3650
5354	serge	3651	skl_compute_wm_global_parameters(dev, &config);
		3652
		3653	if (!skl_update_pipe_wm(crtc, ¶ms, &config,
		3654	&results->ddb, &pipe_wm))
		3655	return;
		3656
		3657	skl_compute_wm_results(dev, ¶ms, &pipe_wm, results, intel_crtc);
		3658	results->dirty[intel_crtc->pipe] = true;
		3659
		3660	skl_update_other_pipe_wm(dev, crtc, &config, results);
		3661	skl_write_wm_values(dev_priv, results);
		3662	skl_flush_wm_values(dev_priv, results);
		3663
		3664	/* store the new configuration */
		3665	dev_priv->wm.skl_hw = *results;
		3666	}
		3667
		3668	static void
		3669	skl_update_sprite_wm(struct drm_plane plane, struct drm_crtc crtc,
		3670	uint32_t sprite_width, uint32_t sprite_height,
		3671	int pixel_size, bool enabled, bool scaled)
		3672	{
		3673	struct intel_plane *intel_plane = to_intel_plane(plane);
6084	serge	3674	struct drm_framebuffer *fb = plane->state->fb;
5354	serge	3675
		3676	intel_plane->wm.enabled = enabled;
		3677	intel_plane->wm.scaled = scaled;
		3678	intel_plane->wm.horiz_pixels = sprite_width;
		3679	intel_plane->wm.vert_pixels = sprite_height;
6084	serge	3680	intel_plane->wm.tiling = DRM_FORMAT_MOD_NONE;
5354	serge	3681
6084	serge	3682	/* For planar: Bpp is for UV plane, y_Bpp is for Y plane */
		3683	intel_plane->wm.bytes_per_pixel =
		3684	(fb && fb->pixel_format == DRM_FORMAT_NV12) ?
		3685	drm_format_plane_cpp(plane->state->fb->pixel_format, 1) : pixel_size;
		3686	intel_plane->wm.y_bytes_per_pixel =
		3687	(fb && fb->pixel_format == DRM_FORMAT_NV12) ?
		3688	drm_format_plane_cpp(plane->state->fb->pixel_format, 0) : 0;
		3689
		3690	/*
		3691	* Framebuffer can be NULL on plane disable, but it does not
		3692	* matter for watermarks if we assume no tiling in that case.
		3693	*/
		3694	if (fb)
		3695	intel_plane->wm.tiling = fb->modifier[0];
		3696	intel_plane->wm.rotation = plane->state->rotation;
		3697
5354	serge	3698	skl_update_wm(crtc);
		3699	}
		3700
4560	Serge	3701	static void ilk_update_wm(struct drm_crtc *crtc)
		3702	{
		3703	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6084	serge	3704	struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
4560	Serge	3705	struct drm_device *dev = crtc->dev;
		3706	struct drm_i915_private *dev_priv = dev->dev_private;
		3707	struct ilk_wm_maximums max;
		3708	struct ilk_wm_values results = {};
4104	Serge	3709	enum intel_ddb_partitioning partitioning;
4560	Serge	3710	struct intel_pipe_wm pipe_wm = {};
		3711	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
		3712	struct intel_wm_config config = {};
4104	Serge	3713
6084	serge	3714	WARN_ON(cstate->base.active != intel_crtc->active);
4104	Serge	3715
6084	serge	3716	intel_compute_pipe_wm(cstate, &pipe_wm);
4560	Serge	3717
		3718	if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
		3719	return;
		3720
		3721	intel_crtc->wm.active = pipe_wm;
		3722
5060	serge	3723	ilk_compute_wm_config(dev, &config);
		3724
4560	Serge	3725	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
		3726	ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
		3727
		3728	/* 5/6 split only in single pipe config on IVB+ */
		3729	if (INTEL_INFO(dev)->gen >= 7 &&
		3730	config.num_pipes_active == 1 && config.sprites_enabled) {
		3731	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
		3732	ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
		3733
		3734	best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
4104	Serge	3735	} else {
4560	Serge	3736	best_lp_wm = &lp_wm_1_2;
4104	Serge	3737	}
		3738
4560	Serge	3739	partitioning = (best_lp_wm == &lp_wm_1_2) ?
4104	Serge	3740	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
		3741
4560	Serge	3742	ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
		3743
		3744	ilk_write_wm_values(dev_priv, &results);
4104	Serge	3745	}
		3746
5060	serge	3747	static void
		3748	ilk_update_sprite_wm(struct drm_plane *plane,
5354	serge	3749	struct drm_crtc *crtc,
5060	serge	3750	uint32_t sprite_width, uint32_t sprite_height,
		3751	int pixel_size, bool enabled, bool scaled)
4104	Serge	3752	{
4560	Serge	3753	struct drm_device *dev = plane->dev;
5354	serge	3754	struct intel_plane *intel_plane = to_intel_plane(plane);
4104	Serge	3755
4560	Serge	3756	/*
		3757	* IVB workaround: must disable low power watermarks for at least
		3758	* one frame before enabling scaling. LP watermarks can be re-enabled
		3759	* when scaling is disabled.
		3760	*
		3761	* WaCxSRDisabledForSpriteScaling:ivb
		3762	*/
		3763	if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
		3764	intel_wait_for_vblank(dev, intel_plane->pipe);
		3765
		3766	ilk_update_wm(crtc);
4104	Serge	3767	}
		3768
5354	serge	3769	static void skl_pipe_wm_active_state(uint32_t val,
		3770	struct skl_pipe_wm *active,
		3771	bool is_transwm,
		3772	bool is_cursor,
		3773	int i,
		3774	int level)
		3775	{
		3776	bool is_enabled = (val & PLANE_WM_EN) != 0;
		3777
		3778	if (!is_transwm) {
		3779	if (!is_cursor) {
		3780	active->wm[level].plane_en[i] = is_enabled;
		3781	active->wm[level].plane_res_b[i] =
		3782	val & PLANE_WM_BLOCKS_MASK;
		3783	active->wm[level].plane_res_l[i] =
		3784	(val >> PLANE_WM_LINES_SHIFT) &
		3785	PLANE_WM_LINES_MASK;
		3786	} else {
6084	serge	3787	active->wm[level].plane_en[PLANE_CURSOR] = is_enabled;
		3788	active->wm[level].plane_res_b[PLANE_CURSOR] =
5354	serge	3789	val & PLANE_WM_BLOCKS_MASK;
6084	serge	3790	active->wm[level].plane_res_l[PLANE_CURSOR] =
5354	serge	3791	(val >> PLANE_WM_LINES_SHIFT) &
		3792	PLANE_WM_LINES_MASK;
		3793	}
		3794	} else {
		3795	if (!is_cursor) {
		3796	active->trans_wm.plane_en[i] = is_enabled;
		3797	active->trans_wm.plane_res_b[i] =
		3798	val & PLANE_WM_BLOCKS_MASK;
		3799	active->trans_wm.plane_res_l[i] =
		3800	(val >> PLANE_WM_LINES_SHIFT) &
		3801	PLANE_WM_LINES_MASK;
		3802	} else {
6084	serge	3803	active->trans_wm.plane_en[PLANE_CURSOR] = is_enabled;
		3804	active->trans_wm.plane_res_b[PLANE_CURSOR] =
5354	serge	3805	val & PLANE_WM_BLOCKS_MASK;
6084	serge	3806	active->trans_wm.plane_res_l[PLANE_CURSOR] =
5354	serge	3807	(val >> PLANE_WM_LINES_SHIFT) &
		3808	PLANE_WM_LINES_MASK;
		3809	}
		3810	}
		3811	}
		3812
		3813	static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
		3814	{
		3815	struct drm_device *dev = crtc->dev;
		3816	struct drm_i915_private *dev_priv = dev->dev_private;
		3817	struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
		3818	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3819	struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
		3820	enum pipe pipe = intel_crtc->pipe;
		3821	int level, i, max_level;
		3822	uint32_t temp;
		3823
		3824	max_level = ilk_wm_max_level(dev);
		3825
		3826	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
		3827
		3828	for (level = 0; level <= max_level; level++) {
		3829	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3830	hw->plane[pipe][i][level] =
		3831	I915_READ(PLANE_WM(pipe, i, level));
6084	serge	3832	hw->plane[pipe][PLANE_CURSOR][level] = I915_READ(CUR_WM(pipe, level));
5354	serge	3833	}
		3834
		3835	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3836	hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
6084	serge	3837	hw->plane_trans[pipe][PLANE_CURSOR] = I915_READ(CUR_WM_TRANS(pipe));
5354	serge	3838
6084	serge	3839	if (!intel_crtc->active)
5354	serge	3840	return;
		3841
		3842	hw->dirty[pipe] = true;
		3843
		3844	active->linetime = hw->wm_linetime[pipe];
		3845
		3846	for (level = 0; level <= max_level; level++) {
		3847	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3848	temp = hw->plane[pipe][i][level];
		3849	skl_pipe_wm_active_state(temp, active, false,
		3850	false, i, level);
		3851	}
6084	serge	3852	temp = hw->plane[pipe][PLANE_CURSOR][level];
5354	serge	3853	skl_pipe_wm_active_state(temp, active, false, true, i, level);
		3854	}
		3855
		3856	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3857	temp = hw->plane_trans[pipe][i];
		3858	skl_pipe_wm_active_state(temp, active, true, false, i, 0);
		3859	}
		3860
6084	serge	3861	temp = hw->plane_trans[pipe][PLANE_CURSOR];
5354	serge	3862	skl_pipe_wm_active_state(temp, active, true, true, i, 0);
		3863	}
		3864
		3865	void skl_wm_get_hw_state(struct drm_device *dev)
		3866	{
		3867	struct drm_i915_private *dev_priv = dev->dev_private;
		3868	struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
		3869	struct drm_crtc *crtc;
		3870
		3871	skl_ddb_get_hw_state(dev_priv, ddb);
		3872	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
		3873	skl_pipe_wm_get_hw_state(crtc);
		3874	}
		3875
4560	Serge	3876	static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3031	serge	3877	{
4560	Serge	3878	struct drm_device *dev = crtc->dev;
		3879	struct drm_i915_private *dev_priv = dev->dev_private;
		3880	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		3881	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3882	struct intel_pipe_wm *active = &intel_crtc->wm.active;
		3883	enum pipe pipe = intel_crtc->pipe;
		3884	static const unsigned int wm0_pipe_reg[] = {
		3885	[PIPE_A] = WM0_PIPEA_ILK,
		3886	[PIPE_B] = WM0_PIPEB_ILK,
		3887	[PIPE_C] = WM0_PIPEC_IVB,
		3888	};
3031	serge	3889
4560	Serge	3890	hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
		3891	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		3892	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3031	serge	3893
6660	serge	3894	memset(active, 0, sizeof(*active));
		3895
6084	serge	3896	active->pipe_enabled = intel_crtc->active;
5060	serge	3897
		3898	if (active->pipe_enabled) {
4560	Serge	3899	u32 tmp = hw->wm_pipe[pipe];
3031	serge	3900
4560	Serge	3901	/*
		3902	* For active pipes LP0 watermark is marked as
		3903	* enabled, and LP1+ watermaks as disabled since
		3904	* we can't really reverse compute them in case
		3905	* multiple pipes are active.
		3906	*/
		3907	active->wm[0].enable = true;
		3908	active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
		3909	active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
		3910	active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
		3911	active->linetime = hw->wm_linetime[pipe];
		3912	} else {
		3913	int level, max_level = ilk_wm_max_level(dev);
3031	serge	3914
4560	Serge	3915	/*
		3916	* For inactive pipes, all watermark levels
		3917	* should be marked as enabled but zeroed,
		3918	* which is what we'd compute them to.
		3919	*/
		3920	for (level = 0; level <= max_level; level++)
		3921	active->wm[level].enable = true;
3031	serge	3922	}
		3923	}
		3924
6084	serge	3925	#define _FW_WM(value, plane) \
		3926	(((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
		3927	#define _FW_WM_VLV(value, plane) \
		3928	(((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
		3929
		3930	static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
		3931	struct vlv_wm_values *wm)
		3932	{
		3933	enum pipe pipe;
		3934	uint32_t tmp;
		3935
		3936	for_each_pipe(dev_priv, pipe) {
		3937	tmp = I915_READ(VLV_DDL(pipe));
		3938
		3939	wm->ddl[pipe].primary =
		3940	(tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3941	wm->ddl[pipe].cursor =
		3942	(tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3943	wm->ddl[pipe].sprite[0] =
		3944	(tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3945	wm->ddl[pipe].sprite[1] =
		3946	(tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH \| DRAIN_LATENCY_MASK);
		3947	}
		3948
		3949	tmp = I915_READ(DSPFW1);
		3950	wm->sr.plane = _FW_WM(tmp, SR);
		3951	wm->pipe[PIPE_B].cursor = _FW_WM(tmp, CURSORB);
		3952	wm->pipe[PIPE_B].primary = _FW_WM_VLV(tmp, PLANEB);
		3953	wm->pipe[PIPE_A].primary = _FW_WM_VLV(tmp, PLANEA);
		3954
		3955	tmp = I915_READ(DSPFW2);
		3956	wm->pipe[PIPE_A].sprite[1] = _FW_WM_VLV(tmp, SPRITEB);
		3957	wm->pipe[PIPE_A].cursor = _FW_WM(tmp, CURSORA);
		3958	wm->pipe[PIPE_A].sprite[0] = _FW_WM_VLV(tmp, SPRITEA);
		3959
		3960	tmp = I915_READ(DSPFW3);
		3961	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
		3962
		3963	if (IS_CHERRYVIEW(dev_priv)) {
		3964	tmp = I915_READ(DSPFW7_CHV);
		3965	wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
		3966	wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
		3967
		3968	tmp = I915_READ(DSPFW8_CHV);
		3969	wm->pipe[PIPE_C].sprite[1] = _FW_WM_VLV(tmp, SPRITEF);
		3970	wm->pipe[PIPE_C].sprite[0] = _FW_WM_VLV(tmp, SPRITEE);
		3971
		3972	tmp = I915_READ(DSPFW9_CHV);
		3973	wm->pipe[PIPE_C].primary = _FW_WM_VLV(tmp, PLANEC);
		3974	wm->pipe[PIPE_C].cursor = _FW_WM(tmp, CURSORC);
		3975
		3976	tmp = I915_READ(DSPHOWM);
		3977	wm->sr.plane \|= _FW_WM(tmp, SR_HI) << 9;
		3978	wm->pipe[PIPE_C].sprite[1] \|= _FW_WM(tmp, SPRITEF_HI) << 8;
		3979	wm->pipe[PIPE_C].sprite[0] \|= _FW_WM(tmp, SPRITEE_HI) << 8;
		3980	wm->pipe[PIPE_C].primary \|= _FW_WM(tmp, PLANEC_HI) << 8;
		3981	wm->pipe[PIPE_B].sprite[1] \|= _FW_WM(tmp, SPRITED_HI) << 8;
		3982	wm->pipe[PIPE_B].sprite[0] \|= _FW_WM(tmp, SPRITEC_HI) << 8;
		3983	wm->pipe[PIPE_B].primary \|= _FW_WM(tmp, PLANEB_HI) << 8;
		3984	wm->pipe[PIPE_A].sprite[1] \|= _FW_WM(tmp, SPRITEB_HI) << 8;
		3985	wm->pipe[PIPE_A].sprite[0] \|= _FW_WM(tmp, SPRITEA_HI) << 8;
		3986	wm->pipe[PIPE_A].primary \|= _FW_WM(tmp, PLANEA_HI) << 8;
		3987	} else {
		3988	tmp = I915_READ(DSPFW7);
		3989	wm->pipe[PIPE_B].sprite[1] = _FW_WM_VLV(tmp, SPRITED);
		3990	wm->pipe[PIPE_B].sprite[0] = _FW_WM_VLV(tmp, SPRITEC);
		3991
		3992	tmp = I915_READ(DSPHOWM);
		3993	wm->sr.plane \|= _FW_WM(tmp, SR_HI) << 9;
		3994	wm->pipe[PIPE_B].sprite[1] \|= _FW_WM(tmp, SPRITED_HI) << 8;
		3995	wm->pipe[PIPE_B].sprite[0] \|= _FW_WM(tmp, SPRITEC_HI) << 8;
		3996	wm->pipe[PIPE_B].primary \|= _FW_WM(tmp, PLANEB_HI) << 8;
		3997	wm->pipe[PIPE_A].sprite[1] \|= _FW_WM(tmp, SPRITEB_HI) << 8;
		3998	wm->pipe[PIPE_A].sprite[0] \|= _FW_WM(tmp, SPRITEA_HI) << 8;
		3999	wm->pipe[PIPE_A].primary \|= _FW_WM(tmp, PLANEA_HI) << 8;
		4000	}
		4001	}
		4002
		4003	#undef _FW_WM
		4004	#undef _FW_WM_VLV
		4005
		4006	void vlv_wm_get_hw_state(struct drm_device *dev)
		4007	{
		4008	struct drm_i915_private *dev_priv = to_i915(dev);
		4009	struct vlv_wm_values *wm = &dev_priv->wm.vlv;
		4010	struct intel_plane *plane;
		4011	enum pipe pipe;
		4012	u32 val;
		4013
		4014	vlv_read_wm_values(dev_priv, wm);
		4015
		4016	for_each_intel_plane(dev, plane) {
		4017	switch (plane->base.type) {
		4018	int sprite;
		4019	case DRM_PLANE_TYPE_CURSOR:
		4020	plane->wm.fifo_size = 63;
		4021	break;
		4022	case DRM_PLANE_TYPE_PRIMARY:
		4023	plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0);
		4024	break;
		4025	case DRM_PLANE_TYPE_OVERLAY:
		4026	sprite = plane->plane;
		4027	plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1);
		4028	break;
		4029	}
		4030	}
		4031
		4032	wm->cxsr = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
		4033	wm->level = VLV_WM_LEVEL_PM2;
		4034
		4035	if (IS_CHERRYVIEW(dev_priv)) {
		4036	mutex_lock(&dev_priv->rps.hw_lock);
		4037
		4038	val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
		4039	if (val & DSP_MAXFIFO_PM5_ENABLE)
		4040	wm->level = VLV_WM_LEVEL_PM5;
		4041
		4042	/*
		4043	* If DDR DVFS is disabled in the BIOS, Punit
		4044	* will never ack the request. So if that happens
		4045	* assume we don't have to enable/disable DDR DVFS
		4046	* dynamically. To test that just set the REQ_ACK
		4047	* bit to poke the Punit, but don't change the
		4048	* HIGH/LOW bits so that we don't actually change
		4049	* the current state.
		4050	*/
		4051	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		4052	val \|= FORCE_DDR_FREQ_REQ_ACK;
		4053	vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
		4054
		4055	if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
		4056	FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
		4057	DRM_DEBUG_KMS("Punit not acking DDR DVFS request, "
		4058	"assuming DDR DVFS is disabled\n");
		4059	dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
		4060	} else {
		4061	val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
		4062	if ((val & FORCE_DDR_HIGH_FREQ) == 0)
		4063	wm->level = VLV_WM_LEVEL_DDR_DVFS;
		4064	}
		4065
		4066	mutex_unlock(&dev_priv->rps.hw_lock);
		4067	}
		4068
		4069	for_each_pipe(dev_priv, pipe)
		4070	DRM_DEBUG_KMS("Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
		4071	pipe_name(pipe), wm->pipe[pipe].primary, wm->pipe[pipe].cursor,
		4072	wm->pipe[pipe].sprite[0], wm->pipe[pipe].sprite[1]);
		4073
		4074	DRM_DEBUG_KMS("Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
		4075	wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
		4076	}
		4077
4560	Serge	4078	void ilk_wm_get_hw_state(struct drm_device *dev)
3031	serge	4079	{
		4080	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4081	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		4082	struct drm_crtc *crtc;
3031	serge	4083
5060	serge	4084	for_each_crtc(dev, crtc)
4560	Serge	4085	ilk_pipe_wm_get_hw_state(crtc);
4104	Serge	4086
4560	Serge	4087	hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
		4088	hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
		4089	hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3031	serge	4090
4560	Serge	4091	hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
5060	serge	4092	if (INTEL_INFO(dev)->gen >= 7) {
5354	serge	4093	hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		4094	hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
5060	serge	4095	}
3031	serge	4096
4560	Serge	4097	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4098	hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
		4099	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
		4100	else if (IS_IVYBRIDGE(dev))
		4101	hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
		4102	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3031	serge	4103
4560	Serge	4104	hw->enable_fbc_wm =
		4105	!(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3031	serge	4106	}
		4107
		4108	/**
		4109	* intel_update_watermarks - update FIFO watermark values based on current modes
		4110	*
		4111	* Calculate watermark values for the various WM regs based on current mode
		4112	* and plane configuration.
		4113	*
		4114	* There are several cases to deal with here:
		4115	* - normal (i.e. non-self-refresh)
		4116	* - self-refresh (SR) mode
		4117	* - lines are large relative to FIFO size (buffer can hold up to 2)
		4118	* - lines are small relative to FIFO size (buffer can hold more than 2
		4119	* lines), so need to account for TLB latency
		4120	*
		4121	* The normal calculation is:
		4122	* watermark = dotclock * bytes per pixel * latency
		4123	* where latency is platform & configuration dependent (we assume pessimal
		4124	* values here).
		4125	*
		4126	* The SR calculation is:
		4127	* watermark = (trunc(latency/line time)+1) * surface width *
		4128	* bytes per pixel
		4129	* where
		4130	* line time = htotal / dotclock
		4131	* surface width = hdisplay for normal plane and 64 for cursor
		4132	* and latency is assumed to be high, as above.
		4133	*
		4134	* The final value programmed to the register should always be rounded up,
		4135	* and include an extra 2 entries to account for clock crossings.
		4136	*
		4137	* We don't use the sprite, so we can ignore that. And on Crestline we have
		4138	* to set the non-SR watermarks to 8.
		4139	*/
4560	Serge	4140	void intel_update_watermarks(struct drm_crtc *crtc)
3031	serge	4141	{
4560	Serge	4142	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3031	serge	4143
		4144	if (dev_priv->display.update_wm)
4560	Serge	4145	dev_priv->display.update_wm(crtc);
3031	serge	4146	}
		4147
4104	Serge	4148	void intel_update_sprite_watermarks(struct drm_plane *plane,
		4149	struct drm_crtc *crtc,
5060	serge	4150	uint32_t sprite_width,
		4151	uint32_t sprite_height,
		4152	int pixel_size,
4104	Serge	4153	bool enabled, bool scaled)
3031	serge	4154	{
4104	Serge	4155	struct drm_i915_private *dev_priv = plane->dev->dev_private;
3031	serge	4156
		4157	if (dev_priv->display.update_sprite_wm)
5060	serge	4158	dev_priv->display.update_sprite_wm(plane, crtc,
		4159	sprite_width, sprite_height,
4104	Serge	4160	pixel_size, enabled, scaled);
3031	serge	4161	}
		4162
		4163	/**
		4164	* Lock protecting IPS related data structures
		4165	*/
		4166	DEFINE_SPINLOCK(mchdev_lock);
		4167
		4168	/* Global for IPS driver to get at the current i915 device. Protected by
		4169	* mchdev_lock. */
		4170	static struct drm_i915_private *i915_mch_dev;
		4171
		4172	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		4173	{
		4174	struct drm_i915_private *dev_priv = dev->dev_private;
		4175	u16 rgvswctl;
		4176
		4177	assert_spin_locked(&mchdev_lock);
		4178
		4179	rgvswctl = I915_READ16(MEMSWCTL);
		4180	if (rgvswctl & MEMCTL_CMD_STS) {
		4181	DRM_DEBUG("gpu busy, RCS change rejected\n");
		4182	return false; /* still busy with another command */
		4183	}
		4184
		4185	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		4186	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		4187	I915_WRITE16(MEMSWCTL, rgvswctl);
		4188	POSTING_READ16(MEMSWCTL);
		4189
		4190	rgvswctl \|= MEMCTL_CMD_STS;
		4191	I915_WRITE16(MEMSWCTL, rgvswctl);
		4192
		4193	return true;
		4194	}
		4195
		4196	static void ironlake_enable_drps(struct drm_device *dev)
		4197	{
		4198	struct drm_i915_private *dev_priv = dev->dev_private;
		4199	u32 rgvmodectl = I915_READ(MEMMODECTL);
		4200	u8 fmax, fmin, fstart, vstart;
		4201
		4202	spin_lock_irq(&mchdev_lock);
		4203
		4204	/* Enable temp reporting */
		4205	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		4206	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		4207
		4208	/* 100ms RC evaluation intervals */
		4209	I915_WRITE(RCUPEI, 100000);
		4210	I915_WRITE(RCDNEI, 100000);
		4211
		4212	/* Set max/min thresholds to 90ms and 80ms respectively */
		4213	I915_WRITE(RCBMAXAVG, 90000);
		4214	I915_WRITE(RCBMINAVG, 80000);
		4215
		4216	I915_WRITE(MEMIHYST, 1);
		4217
		4218	/* Set up min, max, and cur for interrupt handling */
		4219	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		4220	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		4221	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		4222	MEMMODE_FSTART_SHIFT;
		4223
6084	serge	4224	vstart = (I915_READ(PXVFREQ(fstart)) & PXVFREQ_PX_MASK) >>
3031	serge	4225	PXVFREQ_PX_SHIFT;
		4226
		4227	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		4228	dev_priv->ips.fstart = fstart;
		4229
		4230	dev_priv->ips.max_delay = fstart;
		4231	dev_priv->ips.min_delay = fmin;
		4232	dev_priv->ips.cur_delay = fstart;
		4233
		4234	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		4235	fmax, fmin, fstart);
		4236
		4237	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		4238
		4239	/*
		4240	* Interrupts will be enabled in ironlake_irq_postinstall
		4241	*/
		4242
		4243	I915_WRITE(VIDSTART, vstart);
		4244	POSTING_READ(VIDSTART);
		4245
		4246	rgvmodectl \|= MEMMODE_SWMODE_EN;
		4247	I915_WRITE(MEMMODECTL, rgvmodectl);
		4248
		4249	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		4250	DRM_ERROR("stuck trying to change perf mode\n");
		4251	mdelay(1);
		4252
		4253	ironlake_set_drps(dev, fstart);
		4254
6084	serge	4255	dev_priv->ips.last_count1 = I915_READ(DMIEC) +
		4256	I915_READ(DDREC) + I915_READ(CSIEC);
5060	serge	4257	dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
6084	serge	4258	dev_priv->ips.last_count2 = I915_READ(GFXEC);
5060	serge	4259	dev_priv->ips.last_time2 = ktime_get_raw_ns();
3031	serge	4260
		4261	spin_unlock_irq(&mchdev_lock);
		4262	}
		4263
		4264	static void ironlake_disable_drps(struct drm_device *dev)
		4265	{
		4266	struct drm_i915_private *dev_priv = dev->dev_private;
		4267	u16 rgvswctl;
		4268
		4269	spin_lock_irq(&mchdev_lock);
		4270
		4271	rgvswctl = I915_READ16(MEMSWCTL);
		4272
		4273	/* Ack interrupts, disable EFC interrupt */
		4274	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		4275	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		4276	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		4277	I915_WRITE(DEIIR, DE_PCU_EVENT);
		4278	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		4279
		4280	/* Go back to the starting frequency */
		4281	ironlake_set_drps(dev, dev_priv->ips.fstart);
		4282	mdelay(1);
		4283	rgvswctl \|= MEMCTL_CMD_STS;
		4284	I915_WRITE(MEMSWCTL, rgvswctl);
		4285	mdelay(1);
		4286
		4287	spin_unlock_irq(&mchdev_lock);
		4288	}
		4289
		4290	/* There's a funny hw issue where the hw returns all 0 when reading from
		4291	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		4292	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		4293	* all limits and the gpu stuck at whatever frequency it is at atm).
		4294	*/
6084	serge	4295	static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3031	serge	4296	{
		4297	u32 limits;
		4298
		4299	/* Only set the down limit when we've reached the lowest level to avoid
		4300	* getting more interrupts, otherwise leave this clear. This prevents a
		4301	* race in the hw when coming out of rc6: There's a tiny window where
		4302	* the hw runs at the minimal clock before selecting the desired
		4303	* frequency, if the down threshold expires in that window we will not
		4304	* receive a down interrupt. */
6084	serge	4305	if (IS_GEN9(dev_priv->dev)) {
		4306	limits = (dev_priv->rps.max_freq_softlimit) << 23;
		4307	if (val <= dev_priv->rps.min_freq_softlimit)
		4308	limits \|= (dev_priv->rps.min_freq_softlimit) << 14;
		4309	} else {
		4310	limits = dev_priv->rps.max_freq_softlimit << 24;
		4311	if (val <= dev_priv->rps.min_freq_softlimit)
		4312	limits \|= dev_priv->rps.min_freq_softlimit << 16;
		4313	}
3031	serge	4314
		4315	return limits;
		4316	}
		4317
4560	Serge	4318	static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
		4319	{
		4320	int new_power;
6084	serge	4321	u32 threshold_up = 0, threshold_down = 0; /* in % */
		4322	u32 ei_up = 0, ei_down = 0;
4560	Serge	4323
		4324	new_power = dev_priv->rps.power;
		4325	switch (dev_priv->rps.power) {
		4326	case LOW_POWER:
5060	serge	4327	if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
4560	Serge	4328	new_power = BETWEEN;
		4329	break;
		4330
		4331	case BETWEEN:
5060	serge	4332	if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
4560	Serge	4333	new_power = LOW_POWER;
5060	serge	4334	else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
4560	Serge	4335	new_power = HIGH_POWER;
		4336	break;
		4337
		4338	case HIGH_POWER:
5060	serge	4339	if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
4560	Serge	4340	new_power = BETWEEN;
		4341	break;
		4342	}
		4343	/* Max/min bins are special */
6084	serge	4344	if (val <= dev_priv->rps.min_freq_softlimit)
4560	Serge	4345	new_power = LOW_POWER;
6084	serge	4346	if (val >= dev_priv->rps.max_freq_softlimit)
4560	Serge	4347	new_power = HIGH_POWER;
		4348	if (new_power == dev_priv->rps.power)
		4349	return;
		4350
		4351	/* Note the units here are not exactly 1us, but 1280ns. */
		4352	switch (new_power) {
		4353	case LOW_POWER:
		4354	/* Upclock if more than 95% busy over 16ms */
6084	serge	4355	ei_up = 16000;
		4356	threshold_up = 95;
4560	Serge	4357
		4358	/* Downclock if less than 85% busy over 32ms */
6084	serge	4359	ei_down = 32000;
		4360	threshold_down = 85;
4560	Serge	4361	break;
		4362
		4363	case BETWEEN:
		4364	/* Upclock if more than 90% busy over 13ms */
6084	serge	4365	ei_up = 13000;
		4366	threshold_up = 90;
4560	Serge	4367
		4368	/* Downclock if less than 75% busy over 32ms */
6084	serge	4369	ei_down = 32000;
		4370	threshold_down = 75;
4560	Serge	4371	break;
		4372
		4373	case HIGH_POWER:
		4374	/* Upclock if more than 85% busy over 10ms */
6084	serge	4375	ei_up = 10000;
		4376	threshold_up = 85;
4560	Serge	4377
		4378	/* Downclock if less than 60% busy over 32ms */
6084	serge	4379	ei_down = 32000;
		4380	threshold_down = 60;
4560	Serge	4381	break;
		4382	}
		4383
6935	serge	4384	/* When byt can survive without system hang with dynamic
		4385	* sw freq adjustments, this restriction can be lifted.
		4386	*/
		4387	if (IS_VALLEYVIEW(dev_priv))
		4388	goto skip_hw_write;
		4389
6084	serge	4390	I915_WRITE(GEN6_RP_UP_EI,
		4391	GT_INTERVAL_FROM_US(dev_priv, ei_up));
		4392	I915_WRITE(GEN6_RP_UP_THRESHOLD,
		4393	GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100)));
		4394
		4395	I915_WRITE(GEN6_RP_DOWN_EI,
		4396	GT_INTERVAL_FROM_US(dev_priv, ei_down));
		4397	I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
		4398	GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100)));
		4399
		4400	I915_WRITE(GEN6_RP_CONTROL,
		4401	GEN6_RP_MEDIA_TURBO \|
		4402	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4403	GEN6_RP_MEDIA_IS_GFX \|
		4404	GEN6_RP_ENABLE \|
		4405	GEN6_RP_UP_BUSY_AVG \|
		4406	GEN6_RP_DOWN_IDLE_AVG);
		4407
6935	serge	4408	skip_hw_write:
4560	Serge	4409	dev_priv->rps.power = new_power;
6084	serge	4410	dev_priv->rps.up_threshold = threshold_up;
		4411	dev_priv->rps.down_threshold = threshold_down;
4560	Serge	4412	dev_priv->rps.last_adj = 0;
		4413	}
		4414
5060	serge	4415	static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
		4416	{
		4417	u32 mask = 0;
		4418
6935	serge	4419	/* We use UP_EI_EXPIRED interupts for both up/down in manual mode */
5060	serge	4420	if (val > dev_priv->rps.min_freq_softlimit)
6935	serge	4421	mask \|= GEN6_PM_RP_UP_EI_EXPIRED \| GEN6_PM_RP_DOWN_THRESHOLD \| GEN6_PM_RP_DOWN_TIMEOUT;
5060	serge	4422	if (val < dev_priv->rps.max_freq_softlimit)
6084	serge	4423	mask \|= GEN6_PM_RP_UP_EI_EXPIRED \| GEN6_PM_RP_UP_THRESHOLD;
5060	serge	4424
		4425	mask &= dev_priv->pm_rps_events;
		4426
6084	serge	4427	return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
5060	serge	4428	}
		4429
		4430	/* gen6_set_rps is called to update the frequency request, but should also be
		4431	* called when the range (min_delay and max_delay) is modified so that we can
		4432	* update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
6084	serge	4433	static void gen6_set_rps(struct drm_device *dev, u8 val)
3031	serge	4434	{
		4435	struct drm_i915_private *dev_priv = dev->dev_private;
		4436
6084	serge	4437	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
		4438	if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0))
		4439	return;
		4440
3243	Serge	4441	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6084	serge	4442	WARN_ON(val > dev_priv->rps.max_freq);
		4443	WARN_ON(val < dev_priv->rps.min_freq);
3031	serge	4444
5060	serge	4445	/* min/max delay may still have been modified so be sure to
		4446	* write the limits value.
		4447	*/
		4448	if (val != dev_priv->rps.cur_freq) {
5354	serge	4449	gen6_set_rps_thresholds(dev_priv, val);
4560	Serge	4450
6084	serge	4451	if (IS_GEN9(dev))
5354	serge	4452	I915_WRITE(GEN6_RPNSWREQ,
6084	serge	4453	GEN9_FREQUENCY(val));
		4454	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4455	I915_WRITE(GEN6_RPNSWREQ,
5354	serge	4456	HSW_FREQUENCY(val));
		4457	else
		4458	I915_WRITE(GEN6_RPNSWREQ,
		4459	GEN6_FREQUENCY(val) \|
		4460	GEN6_OFFSET(0) \|
		4461	GEN6_AGGRESSIVE_TURBO);
5060	serge	4462	}
3031	serge	4463
		4464	/* Make sure we continue to get interrupts
		4465	* until we hit the minimum or maximum frequencies.
		4466	*/
6084	serge	4467	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
5060	serge	4468	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3031	serge	4469
		4470	POSTING_READ(GEN6_RPNSWREQ);
		4471
5060	serge	4472	dev_priv->rps.cur_freq = val;
6084	serge	4473	trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
3031	serge	4474	}
		4475
6084	serge	4476	static void valleyview_set_rps(struct drm_device *dev, u8 val)
5060	serge	4477	{
6084	serge	4478	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4479
6084	serge	4480	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4481	WARN_ON(val > dev_priv->rps.max_freq);
		4482	WARN_ON(val < dev_priv->rps.min_freq);
5060	serge	4483
6084	serge	4484	if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
		4485	"Odd GPU freq value\n"))
		4486	val &= ~1;
5060	serge	4487
6084	serge	4488	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
5060	serge	4489
6084	serge	4490	if (val != dev_priv->rps.cur_freq) {
		4491	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
		4492	if (!IS_CHERRYVIEW(dev_priv))
		4493	gen6_set_rps_thresholds(dev_priv, val);
		4494	}
5060	serge	4495
6084	serge	4496	dev_priv->rps.cur_freq = val;
		4497	trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
		4498	}
5060	serge	4499
6084	serge	4500	/* vlv_set_rps_idle: Set the frequency to idle, if Gfx clocks are down
		4501	*
		4502	* * If Gfx is Idle, then
		4503	* 1. Forcewake Media well.
		4504	* 2. Request idle freq.
		4505	* 3. Release Forcewake of Media well.
		4506	*/
		4507	static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
		4508	{
		4509	u32 val = dev_priv->rps.idle_freq;
5060	serge	4510
6084	serge	4511	if (dev_priv->rps.cur_freq <= val)
		4512	return;
5060	serge	4513
6084	serge	4514	/* Wake up the media well, as that takes a lot less
		4515	* power than the Render well. */
		4516	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
		4517	valleyview_set_rps(dev_priv->dev, val);
		4518	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
5060	serge	4519	}
		4520
6084	serge	4521	void gen6_rps_busy(struct drm_i915_private *dev_priv)
3031	serge	4522	{
4560	Serge	4523	mutex_lock(&dev_priv->rps.hw_lock);
		4524	if (dev_priv->rps.enabled) {
6935	serge	4525	if (dev_priv->pm_rps_events & GEN6_PM_RP_UP_EI_EXPIRED)
6084	serge	4526	gen6_rps_reset_ei(dev_priv);
		4527	I915_WRITE(GEN6_PMINTRMSK,
		4528	gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
4560	Serge	4529	}
		4530	mutex_unlock(&dev_priv->rps.hw_lock);
		4531	}
4104	Serge	4532
6084	serge	4533	void gen6_rps_idle(struct drm_i915_private *dev_priv)
4560	Serge	4534	{
		4535	struct drm_device *dev = dev_priv->dev;
4104	Serge	4536
4560	Serge	4537	mutex_lock(&dev_priv->rps.hw_lock);
		4538	if (dev_priv->rps.enabled) {
		4539	if (IS_VALLEYVIEW(dev))
6084	serge	4540	vlv_set_rps_idle(dev_priv);
4560	Serge	4541	else
6084	serge	4542	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4560	Serge	4543	dev_priv->rps.last_adj = 0;
6660	serge	4544	I915_WRITE(GEN6_PMINTRMSK,
		4545	gen6_sanitize_rps_pm_mask(dev_priv, ~0));
4560	Serge	4546	}
		4547	mutex_unlock(&dev_priv->rps.hw_lock);
6084	serge	4548
		4549	spin_lock(&dev_priv->rps.client_lock);
		4550	while (!list_empty(&dev_priv->rps.clients))
		4551	list_del_init(dev_priv->rps.clients.next);
		4552	spin_unlock(&dev_priv->rps.client_lock);
4104	Serge	4553	}
		4554
6084	serge	4555	void gen6_rps_boost(struct drm_i915_private *dev_priv,
		4556	struct intel_rps_client *rps,
		4557	unsigned long submitted)
4104	Serge	4558	{
6084	serge	4559	/* This is intentionally racy! We peek at the state here, then
		4560	* validate inside the RPS worker.
		4561	*/
		4562	if (!(dev_priv->mm.busy &&
		4563	dev_priv->rps.enabled &&
		4564	dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit))
		4565	return;
3031	serge	4566
6084	serge	4567	/* Force a RPS boost (and don't count it against the client) if
		4568	* the GPU is severely congested.
		4569	*/
		4570	if (rps && time_after(jiffies, submitted + DRM_I915_THROTTLE_JIFFIES))
		4571	rps = NULL;
4104	Serge	4572
6084	serge	4573	spin_lock(&dev_priv->rps.client_lock);
		4574	if (rps == NULL \|\| list_empty(&rps->link)) {
		4575	spin_lock_irq(&dev_priv->irq_lock);
		4576	if (dev_priv->rps.interrupts_enabled) {
		4577	dev_priv->rps.client_boost = true;
		4578	queue_work(dev_priv->wq, &dev_priv->rps.work);
		4579	}
		4580	spin_unlock_irq(&dev_priv->irq_lock);
4104	Serge	4581
6084	serge	4582	if (rps != NULL) {
		4583	list_add(&rps->link, &dev_priv->rps.clients);
		4584	rps->boosts++;
		4585	} else
		4586	dev_priv->rps.boosts++;
		4587	}
		4588	spin_unlock(&dev_priv->rps.client_lock);
		4589	}
4104	Serge	4590
6084	serge	4591	void intel_set_rps(struct drm_device *dev, u8 val)
		4592	{
		4593	if (IS_VALLEYVIEW(dev))
		4594	valleyview_set_rps(dev, val);
		4595	else
		4596	gen6_set_rps(dev, val);
4104	Serge	4597	}
		4598
5354	serge	4599	static void gen9_disable_rps(struct drm_device *dev)
5060	serge	4600	{
		4601	struct drm_i915_private *dev_priv = dev->dev_private;
		4602
5354	serge	4603	I915_WRITE(GEN6_RC_CONTROL, 0);
6084	serge	4604	I915_WRITE(GEN9_PG_ENABLE, 0);
5060	serge	4605	}
		4606
4104	Serge	4607	static void gen6_disable_rps(struct drm_device *dev)
		4608	{
		4609	struct drm_i915_private *dev_priv = dev->dev_private;
		4610
		4611	I915_WRITE(GEN6_RC_CONTROL, 0);
		4612	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		4613	}
		4614
5060	serge	4615	static void cherryview_disable_rps(struct drm_device *dev)
		4616	{
		4617	struct drm_i915_private *dev_priv = dev->dev_private;
		4618
		4619	I915_WRITE(GEN6_RC_CONTROL, 0);
		4620	}
		4621
4104	Serge	4622	static void valleyview_disable_rps(struct drm_device *dev)
		4623	{
		4624	struct drm_i915_private *dev_priv = dev->dev_private;
		4625
5354	serge	4626	/* we're doing forcewake before Disabling RC6,
		4627	* This what the BIOS expects when going into suspend */
6084	serge	4628	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4629
4104	Serge	4630	I915_WRITE(GEN6_RC_CONTROL, 0);
		4631
6084	serge	4632	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	4633	}
		4634
4560	Serge	4635	static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
		4636	{
5060	serge	4637	if (IS_VALLEYVIEW(dev)) {
		4638	if (mode & (GEN7_RC_CTL_TO_MODE \| GEN6_RC_CTL_EI_MODE(1)))
		4639	mode = GEN6_RC_CTL_RC6_ENABLE;
		4640	else
		4641	mode = 0;
		4642	}
5354	serge	4643	if (HAS_RC6p(dev))
		4644	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
6084	serge	4645	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		4646	(mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		4647	(mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
5354	serge	4648
		4649	else
		4650	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
		4651	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
4560	Serge	4652	}
		4653
5060	serge	4654	static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
3031	serge	4655	{
6084	serge	4656	/* No RC6 before Ironlake and code is gone for ilk. */
		4657	if (INTEL_INFO(dev)->gen < 6)
4104	Serge	4658	return 0;
		4659
3031	serge	4660	/* Respect the kernel parameter if it is set */
5060	serge	4661	if (enable_rc6 >= 0) {
		4662	int mask;
3031	serge	4663
5354	serge	4664	if (HAS_RC6p(dev))
5060	serge	4665	mask = INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE \|
		4666	INTEL_RC6pp_ENABLE;
		4667	else
		4668	mask = INTEL_RC6_ENABLE;
		4669
		4670	if ((enable_rc6 & mask) != enable_rc6)
		4671	DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
6084	serge	4672	enable_rc6 & mask, enable_rc6, mask);
5060	serge	4673
		4674	return enable_rc6 & mask;
		4675	}
		4676
		4677	if (IS_IVYBRIDGE(dev))
		4678	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
3031	serge	4679
6084	serge	4680	return INTEL_RC6_ENABLE;
5060	serge	4681	}
3031	serge	4682
5060	serge	4683	int intel_enable_rc6(const struct drm_device *dev)
		4684	{
		4685	return i915.enable_rc6;
3031	serge	4686	}
		4687
5354	serge	4688	static void gen6_init_rps_frequencies(struct drm_device *dev)
5060	serge	4689	{
		4690	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4691	uint32_t rp_state_cap;
		4692	u32 ddcc_status = 0;
		4693	int ret;
5060	serge	4694
		4695	/* All of these values are in units of 50MHz */
		4696	dev_priv->rps.cur_freq = 0;
5354	serge	4697	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
6084	serge	4698	if (IS_BROXTON(dev)) {
		4699	rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
		4700	dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
		4701	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4702	dev_priv->rps.min_freq = (rp_state_cap >> 0) & 0xff;
		4703	} else {
		4704	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		4705	dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
		4706	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4707	dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
		4708	}
		4709
5060	serge	4710	/* hw_max = RP0 until we check for overclocking */
		4711	dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
4104	Serge	4712
5354	serge	4713	dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
6084	serge	4714	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev) \|\| IS_SKYLAKE(dev)) {
5354	serge	4715	ret = sandybridge_pcode_read(dev_priv,
		4716	HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
		4717	&ddcc_status);
		4718	if (0 == ret)
		4719	dev_priv->rps.efficient_freq =
6084	serge	4720	clamp_t(u8,
		4721	((ddcc_status >> 8) & 0xff),
		4722	dev_priv->rps.min_freq,
		4723	dev_priv->rps.max_freq);
5354	serge	4724	}
		4725
6084	serge	4726	if (IS_SKYLAKE(dev)) {
		4727	/* Store the frequency values in 16.66 MHZ units, which is
		4728	the natural hardware unit for SKL */
		4729	dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
		4730	dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER;
		4731	dev_priv->rps.min_freq *= GEN9_FREQ_SCALER;
		4732	dev_priv->rps.max_freq *= GEN9_FREQ_SCALER;
		4733	dev_priv->rps.efficient_freq *= GEN9_FREQ_SCALER;
		4734	}
		4735
		4736	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		4737
5060	serge	4738	/* Preserve min/max settings in case of re-init */
		4739	if (dev_priv->rps.max_freq_softlimit == 0)
		4740	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4104	Serge	4741
5354	serge	4742	if (dev_priv->rps.min_freq_softlimit == 0) {
		4743	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4744	dev_priv->rps.min_freq_softlimit =
6084	serge	4745	max_t(int, dev_priv->rps.efficient_freq,
		4746	intel_freq_opcode(dev_priv, 450));
5354	serge	4747	else
		4748	dev_priv->rps.min_freq_softlimit =
		4749	dev_priv->rps.min_freq;
		4750	}
4104	Serge	4751	}
		4752
6084	serge	4753	/* See the Gen9_GT_PM_Programming_Guide doc for the below */
5354	serge	4754	static void gen9_enable_rps(struct drm_device *dev)
		4755	{
		4756	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	4757
		4758	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
		4759
		4760	gen6_init_rps_frequencies(dev);
		4761
		4762	/* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
		4763	if (IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) {
		4764	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4765	return;
		4766	}
		4767
		4768	/* Program defaults and thresholds for RPS*/
		4769	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4770	GEN9_FREQUENCY(dev_priv->rps.rp1_freq));
		4771
		4772	/* 1 second timeout*/
		4773	I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
		4774	GT_INTERVAL_FROM_US(dev_priv, 1000000));
		4775
		4776	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
		4777
		4778	/* Leaning on the below call to gen6_set_rps to program/setup the
		4779	* Up/Down EI & threshold registers, as well as the RP_CONTROL,
		4780	* RP_INTERRUPT_LIMITS & RPNSWREQ registers */
		4781	dev_priv->rps.power = HIGH_POWER; /* force a reset */
		4782	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
		4783
		4784	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4785	}
		4786
		4787	static void gen9_enable_rc6(struct drm_device *dev)
		4788	{
		4789	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4790	struct intel_engine_cs *ring;
		4791	uint32_t rc6_mask = 0;
		4792	int unused;
		4793
		4794	/* 1a: Software RC state - RC0 */
		4795	I915_WRITE(GEN6_RC_STATE, 0);
		4796
		4797	/* 1b: Get forcewake during program sequence. Although the driver
		4798	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	4799	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5354	serge	4800
		4801	/* 2a: Disable RC states. */
		4802	I915_WRITE(GEN6_RC_CONTROL, 0);
		4803
		4804	/* 2b: Program RC6 thresholds.*/
6084	serge	4805
		4806	/* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
		4807	if (IS_SKYLAKE(dev))
		4808	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
		4809	else
		4810	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
5354	serge	4811	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4812	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4813	for_each_ring(ring, dev_priv, unused)
		4814	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
6084	serge	4815
		4816	if (HAS_GUC_UCODE(dev))
		4817	I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
		4818
5354	serge	4819	I915_WRITE(GEN6_RC_SLEEP, 0);
		4820
6084	serge	4821	/* 2c: Program Coarse Power Gating Policies. */
		4822	I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
		4823	I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
		4824
5354	serge	4825	/* 3a: Enable RC6 */
		4826	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4827	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
		4828	DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
		4829	"on" : "off");
6084	serge	4830	/* WaRsUseTimeoutMode */
		4831	if ((IS_SKYLAKE(dev) && INTEL_REVID(dev) <= SKL_REVID_D0) \|\|
		4832	(IS_BROXTON(dev) && INTEL_REVID(dev) <= BXT_REVID_A0)) {
		4833	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
		4834	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4835	GEN7_RC_CTL_TO_MODE \|
		4836	rc6_mask);
		4837	} else {
		4838	I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
		4839	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4840	GEN6_RC_CTL_EI_MODE(1) \|
		4841	rc6_mask);
		4842	}
5354	serge	4843
6084	serge	4844	/*
		4845	* 3b: Enable Coarse Power Gating only when RC6 is enabled.
		4846	* WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6.
		4847	*/
		4848	if ((IS_BROXTON(dev) && (INTEL_REVID(dev) < BXT_REVID_B0)) \|\|
		4849	((IS_SKL_GT3(dev) \|\| IS_SKL_GT4(dev)) && (INTEL_REVID(dev) <= SKL_REVID_F0)))
		4850	I915_WRITE(GEN9_PG_ENABLE, 0);
		4851	else
		4852	I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
		4853	(GEN9_RENDER_PG_ENABLE \| GEN9_MEDIA_PG_ENABLE) : 0);
5354	serge	4854
6084	serge	4855	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		4856
5354	serge	4857	}
		4858
4560	Serge	4859	static void gen8_enable_rps(struct drm_device *dev)
		4860	{
		4861	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4862	struct intel_engine_cs *ring;
5354	serge	4863	uint32_t rc6_mask = 0;
4560	Serge	4864	int unused;
		4865
		4866	/* 1a: Software RC state - RC0 */
		4867	I915_WRITE(GEN6_RC_STATE, 0);
		4868
		4869	/* 1c & 1d: Get forcewake during program sequence. Although the driver
		4870	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	4871	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4560	Serge	4872
		4873	/* 2a: Disable RC states. */
		4874	I915_WRITE(GEN6_RC_CONTROL, 0);
		4875
5354	serge	4876	/* Initialize rps frequencies */
		4877	gen6_init_rps_frequencies(dev);
4560	Serge	4878
		4879	/* 2b: Program RC6 thresholds.*/
		4880	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		4881	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4882	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4883	for_each_ring(ring, dev_priv, unused)
		4884	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4885	I915_WRITE(GEN6_RC_SLEEP, 0);
5060	serge	4886	if (IS_BROADWELL(dev))
		4887	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
		4888	else
6084	serge	4889	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
4560	Serge	4890
		4891	/* 3: Enable RC6 */
		4892	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4893	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
5060	serge	4894	intel_print_rc6_info(dev, rc6_mask);
		4895	if (IS_BROADWELL(dev))
		4896	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4897	GEN7_RC_CTL_TO_MODE \|
		4898	rc6_mask);
		4899	else
6084	serge	4900	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4901	GEN6_RC_CTL_EI_MODE(1) \|
		4902	rc6_mask);
4560	Serge	4903
		4904	/* 4 Program defaults and thresholds for RPS*/
5060	serge	4905	I915_WRITE(GEN6_RPNSWREQ,
		4906	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
		4907	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4908	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4560	Serge	4909	/* NB: Docs say 1s, and 1000000 - which aren't equivalent */
		4910	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
		4911
		4912	/* Docs recommend 900MHz, and 300 MHz respectively */
		4913	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
5060	serge	4914	dev_priv->rps.max_freq_softlimit << 24 \|
		4915	dev_priv->rps.min_freq_softlimit << 16);
4560	Serge	4916
		4917	I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
		4918	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
		4919	I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
		4920	I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
		4921
		4922	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		4923
		4924	/* 5: Enable RPS */
		4925	I915_WRITE(GEN6_RP_CONTROL,
		4926	GEN6_RP_MEDIA_TURBO \|
		4927	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4928	GEN6_RP_MEDIA_IS_GFX \|
		4929	GEN6_RP_ENABLE \|
		4930	GEN6_RP_UP_BUSY_AVG \|
		4931	GEN6_RP_DOWN_IDLE_AVG);
		4932
		4933	/* 6: Ring frequency + overclocking (our driver does this later */
		4934
5354	serge	4935	dev_priv->rps.power = HIGH_POWER; /* force a reset */
6084	serge	4936	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
4560	Serge	4937
6084	serge	4938	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4560	Serge	4939	}
		4940
3031	serge	4941	static void gen6_enable_rps(struct drm_device *dev)
		4942	{
		4943	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4944	struct intel_engine_cs *ring;
		4945	u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
3031	serge	4946	u32 gtfifodbg;
		4947	int rc6_mode;
3243	Serge	4948	int i, ret;
3031	serge	4949
3243	Serge	4950	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4951
		4952	/* Here begins a magic sequence of register writes to enable
		4953	* auto-downclocking.
		4954	*
		4955	* Perhaps there might be some value in exposing these to
		4956	* userspace...
		4957	*/
		4958	I915_WRITE(GEN6_RC_STATE, 0);
		4959
		4960	/* Clear the DBG now so we don't confuse earlier errors */
		4961	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		4962	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		4963	I915_WRITE(GTFIFODBG, gtfifodbg);
		4964	}
		4965
6084	serge	4966	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
3031	serge	4967
5354	serge	4968	/* Initialize rps frequencies */
		4969	gen6_init_rps_frequencies(dev);
3031	serge	4970
		4971	/* disable the counters and set deterministic thresholds */
		4972	I915_WRITE(GEN6_RC_CONTROL, 0);
		4973
		4974	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		4975	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		4976	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		4977	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		4978	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		4979
		4980	for_each_ring(ring, dev_priv, i)
		4981	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4982
		4983	I915_WRITE(GEN6_RC_SLEEP, 0);
		4984	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4560	Serge	4985	if (IS_IVYBRIDGE(dev))
4104	Serge	4986	I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
		4987	else
6084	serge	4988	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	4989	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	4990	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		4991
		4992	/* Check if we are enabling RC6 */
		4993	rc6_mode = intel_enable_rc6(dev_priv->dev);
		4994	if (rc6_mode & INTEL_RC6_ENABLE)
		4995	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		4996
		4997	/* We don't use those on Haswell */
		4998	if (!IS_HASWELL(dev)) {
		4999	if (rc6_mode & INTEL_RC6p_ENABLE)
		5000	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		5001
		5002	if (rc6_mode & INTEL_RC6pp_ENABLE)
		5003	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		5004	}
		5005
4560	Serge	5006	intel_print_rc6_info(dev, rc6_mask);
3031	serge	5007
		5008	I915_WRITE(GEN6_RC_CONTROL,
		5009	rc6_mask \|
		5010	GEN6_RC_CTL_EI_MODE(1) \|
		5011	GEN6_RC_CTL_HW_ENABLE);
		5012
4560	Serge	5013	/* Power down if completely idle for over 50ms */
		5014	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
3031	serge	5015	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5016
3243	Serge	5017	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
5060	serge	5018	if (ret)
		5019	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		5020
6084	serge	5021	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
		5022	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		5023	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
5060	serge	5024	(dev_priv->rps.max_freq_softlimit & 0xff) * 50,
6084	serge	5025	(pcu_mbox & 0xff) * 50);
5060	serge	5026	dev_priv->rps.max_freq = pcu_mbox & 0xff;
3031	serge	5027	}
		5028
4560	Serge	5029	dev_priv->rps.power = HIGH_POWER; /* force a reset */
6084	serge	5030	gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
3031	serge	5031
3243	Serge	5032	rc6vids = 0;
		5033	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		5034	if (IS_GEN6(dev) && ret) {
		5035	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		5036	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		5037	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		5038	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		5039	rc6vids &= 0xffff00;
		5040	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		5041	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		5042	if (ret)
		5043	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		5044	}
		5045
6084	serge	5046	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
3031	serge	5047	}
		5048
5060	serge	5049	static void __gen6_update_ring_freq(struct drm_device *dev)
3031	serge	5050	{
		5051	struct drm_i915_private *dev_priv = dev->dev_private;
		5052	int min_freq = 15;
3746	Serge	5053	unsigned int gpu_freq;
		5054	unsigned int max_ia_freq, min_ring_freq;
6084	serge	5055	unsigned int max_gpu_freq, min_gpu_freq;
3031	serge	5056	int scaling_factor = 180;
4560	Serge	5057	struct cpufreq_policy *policy;
3031	serge	5058
3243	Serge	5059	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	5060
		5061	max_ia_freq = cpufreq_quick_get_max(0);
		5062	/*
5060	serge	5063	* Default to measured freq if none found, PCU will ensure we
		5064	* don't go over
3031	serge	5065	*/
		5066	max_ia_freq = tsc_khz;
		5067
		5068	/* Convert from kHz to MHz */
		5069	max_ia_freq /= 1000;
		5070
4560	Serge	5071	min_ring_freq = I915_READ(DCLK) & 0xf;
		5072	/* convert DDR frequency from units of 266.6MHz to bandwidth */
		5073	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3746	Serge	5074
6084	serge	5075	if (IS_SKYLAKE(dev)) {
		5076	/* Convert GT frequency to 50 HZ units */
		5077	min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
		5078	max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
		5079	} else {
		5080	min_gpu_freq = dev_priv->rps.min_freq;
		5081	max_gpu_freq = dev_priv->rps.max_freq;
		5082	}
		5083
3031	serge	5084	/*
		5085	* For each potential GPU frequency, load a ring frequency we'd like
		5086	* to use for memory access. We do this by specifying the IA frequency
		5087	* the PCU should use as a reference to determine the ring frequency.
		5088	*/
6084	serge	5089	for (gpu_freq = max_gpu_freq; gpu_freq >= min_gpu_freq; gpu_freq--) {
		5090	int diff = max_gpu_freq - gpu_freq;
3746	Serge	5091	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	5092
6084	serge	5093	if (IS_SKYLAKE(dev)) {
		5094	/*
		5095	* ring_freq = 2 * GT. ring_freq is in 100MHz units
		5096	* No floor required for ring frequency on SKL.
		5097	*/
		5098	ring_freq = gpu_freq;
		5099	} else if (INTEL_INFO(dev)->gen >= 8) {
4560	Serge	5100	/* max(2 * GT, DDR). NB: GT is 50MHz units */
		5101	ring_freq = max(min_ring_freq, gpu_freq);
		5102	} else if (IS_HASWELL(dev)) {
		5103	ring_freq = mult_frac(gpu_freq, 5, 4);
3746	Serge	5104	ring_freq = max(min_ring_freq, ring_freq);
		5105	/* leave ia_freq as the default, chosen by cpufreq */
		5106	} else {
		5107	/* On older processors, there is no separate ring
		5108	* clock domain, so in order to boost the bandwidth
		5109	* of the ring, we need to upclock the CPU (ia_freq).
		5110	*
		5111	* For GPU frequencies less than 750MHz,
		5112	* just use the lowest ring freq.
6084	serge	5113	*/
		5114	if (gpu_freq < min_freq)
		5115	ia_freq = 800;
		5116	else
		5117	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		5118	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	5119	}
3031	serge	5120
3243	Serge	5121	sandybridge_pcode_write(dev_priv,
		5122	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	5123	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		5124	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		5125	gpu_freq);
3031	serge	5126	}
		5127	}
		5128
5060	serge	5129	void gen6_update_ring_freq(struct drm_device *dev)
4104	Serge	5130	{
5060	serge	5131	struct drm_i915_private *dev_priv = dev->dev_private;
		5132
6084	serge	5133	if (!HAS_CORE_RING_FREQ(dev))
5060	serge	5134	return;
		5135
		5136	mutex_lock(&dev_priv->rps.hw_lock);
		5137	__gen6_update_ring_freq(dev);
		5138	mutex_unlock(&dev_priv->rps.hw_lock);
		5139	}
		5140
		5141	static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
		5142	{
6084	serge	5143	struct drm_device *dev = dev_priv->dev;
4104	Serge	5144	u32 val, rp0;
		5145
6084	serge	5146	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
5060	serge	5147
6084	serge	5148	switch (INTEL_INFO(dev)->eu_total) {
		5149	case 8:
		5150	/* (2 * 4) config */
		5151	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
		5152	break;
		5153	case 12:
		5154	/* (2 * 6) config */
		5155	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
		5156	break;
		5157	case 16:
		5158	/* (2 * 8) config */
		5159	default:
		5160	/* Setting (2 * 8) Min RP0 for any other combination */
		5161	rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
		5162	break;
		5163	}
		5164
		5165	rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
		5166
5060	serge	5167	return rp0;
		5168	}
		5169
		5170	static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5171	{
		5172	u32 val, rpe;
		5173
		5174	val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
		5175	rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
		5176
		5177	return rpe;
		5178	}
		5179
		5180	static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5181	{
		5182	u32 val, rp1;
		5183
6084	serge	5184	val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
		5185	rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
5060	serge	5186
		5187	return rp1;
		5188	}
		5189
		5190	static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5191	{
		5192	u32 val, rp1;
		5193
4104	Serge	5194	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5195
5060	serge	5196	rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
		5197
		5198	return rp1;
		5199	}
		5200
		5201	static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
		5202	{
		5203	u32 val, rp0;
		5204
		5205	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5206
4104	Serge	5207	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
		5208	/* Clamp to max */
		5209	rp0 = min_t(u32, rp0, 0xea);
		5210
		5211	return rp0;
		5212	}
		5213
		5214	static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5215	{
		5216	u32 val, rpe;
		5217
		5218	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
		5219	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
		5220	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
		5221	rpe \|= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
		5222
		5223	return rpe;
		5224	}
		5225
5060	serge	5226	static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
4104	Serge	5227	{
		5228	return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
		5229	}
		5230
5060	serge	5231	/* Check that the pctx buffer wasn't move under us. */
		5232	static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
		5233	{
		5234	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5235
		5236	WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
		5237	dev_priv->vlv_pctx->stolen->start);
		5238	}
		5239
		5240
		5241	/* Check that the pcbr address is not empty. */
		5242	static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
		5243	{
		5244	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5245
		5246	WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
		5247	}
		5248
		5249	static void cherryview_setup_pctx(struct drm_device *dev)
		5250	{
		5251	struct drm_i915_private *dev_priv = dev->dev_private;
		5252	unsigned long pctx_paddr, paddr;
		5253	struct i915_gtt *gtt = &dev_priv->gtt;
		5254	u32 pcbr;
		5255	int pctx_size = 32*1024;
		5256
		5257	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5258
		5259	pcbr = I915_READ(VLV_PCBR);
		5260	if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
5354	serge	5261	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
5060	serge	5262	paddr = (dev_priv->mm.stolen_base +
		5263	(gtt->stolen_size - pctx_size));
		5264
		5265	pctx_paddr = (paddr & (~4095));
		5266	I915_WRITE(VLV_PCBR, pctx_paddr);
		5267	}
5354	serge	5268
		5269	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5060	serge	5270	}
		5271
4104	Serge	5272	static void valleyview_setup_pctx(struct drm_device *dev)
		5273	{
		5274	struct drm_i915_private *dev_priv = dev->dev_private;
		5275	struct drm_i915_gem_object *pctx;
		5276	unsigned long pctx_paddr;
		5277	u32 pcbr;
		5278	int pctx_size = 24*1024;
		5279
5060	serge	5280	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5281
4104	Serge	5282	pcbr = I915_READ(VLV_PCBR);
		5283	if (pcbr) {
		5284	/* BIOS set it up already, grab the pre-alloc'd space */
		5285	int pcbr_offset;
		5286
		5287	pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
		5288	pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
		5289	pcbr_offset,
		5290	I915_GTT_OFFSET_NONE,
		5291	pctx_size);
		5292	goto out;
		5293	}
		5294
5354	serge	5295	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
		5296
4104	Serge	5297	/*
		5298	* From the Gunit register HAS:
		5299	* The Gfx driver is expected to program this register and ensure
		5300	* proper allocation within Gfx stolen memory. For example, this
		5301	* register should be programmed such than the PCBR range does not
		5302	* overlap with other ranges, such as the frame buffer, protected
		5303	* memory, or any other relevant ranges.
		5304	*/
		5305	pctx = i915_gem_object_create_stolen(dev, pctx_size);
		5306	if (!pctx) {
		5307	DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
		5308	return;
		5309	}
		5310
		5311	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
		5312	I915_WRITE(VLV_PCBR, pctx_paddr);
		5313
		5314	out:
5354	serge	5315	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
4104	Serge	5316	dev_priv->vlv_pctx = pctx;
		5317	}
		5318
5060	serge	5319	static void valleyview_cleanup_pctx(struct drm_device *dev)
		5320	{
		5321	struct drm_i915_private *dev_priv = dev->dev_private;
		5322
		5323	if (WARN_ON(!dev_priv->vlv_pctx))
		5324	return;
		5325
		5326	drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
		5327	dev_priv->vlv_pctx = NULL;
		5328	}
		5329
		5330	static void valleyview_init_gt_powersave(struct drm_device *dev)
		5331	{
		5332	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5333	u32 val;
5060	serge	5334
		5335	valleyview_setup_pctx(dev);
		5336
		5337	mutex_lock(&dev_priv->rps.hw_lock);
		5338
5354	serge	5339	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5340	switch ((val >> 6) & 3) {
		5341	case 0:
		5342	case 1:
		5343	dev_priv->mem_freq = 800;
		5344	break;
		5345	case 2:
		5346	dev_priv->mem_freq = 1066;
		5347	break;
		5348	case 3:
		5349	dev_priv->mem_freq = 1333;
		5350	break;
		5351	}
		5352	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5353
5060	serge	5354	dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
		5355	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5356	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6084	serge	5357	intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5060	serge	5358	dev_priv->rps.max_freq);
		5359
		5360	dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
		5361	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6084	serge	5362	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5363	dev_priv->rps.efficient_freq);
		5364
		5365	dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
		5366	DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
6084	serge	5367	intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5060	serge	5368	dev_priv->rps.rp1_freq);
		5369
		5370	dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
		5371	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6084	serge	5372	intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5060	serge	5373	dev_priv->rps.min_freq);
		5374
6084	serge	5375	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		5376
5060	serge	5377	/* Preserve min/max settings in case of re-init */
		5378	if (dev_priv->rps.max_freq_softlimit == 0)
		5379	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5380
		5381	if (dev_priv->rps.min_freq_softlimit == 0)
		5382	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5383
		5384	mutex_unlock(&dev_priv->rps.hw_lock);
		5385	}
		5386
		5387	static void cherryview_init_gt_powersave(struct drm_device *dev)
		5388	{
		5389	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5390	u32 val;
5060	serge	5391
		5392	cherryview_setup_pctx(dev);
		5393
		5394	mutex_lock(&dev_priv->rps.hw_lock);
		5395
6084	serge	5396	mutex_lock(&dev_priv->sb_lock);
5354	serge	5397	val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
6084	serge	5398	mutex_unlock(&dev_priv->sb_lock);
5354	serge	5399
		5400	switch ((val >> 2) & 0x7) {
		5401	case 3:
		5402	dev_priv->mem_freq = 2000;
		5403	break;
6084	serge	5404	default:
5354	serge	5405	dev_priv->mem_freq = 1600;
		5406	break;
		5407	}
		5408	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5409
5060	serge	5410	dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
		5411	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5412	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
6084	serge	5413	intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
5060	serge	5414	dev_priv->rps.max_freq);
		5415
		5416	dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
		5417	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
6084	serge	5418	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5419	dev_priv->rps.efficient_freq);
		5420
		5421	dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
		5422	DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
6084	serge	5423	intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
5060	serge	5424	dev_priv->rps.rp1_freq);
		5425
6084	serge	5426	/* PUnit validated range is only [RPe, RP0] */
		5427	dev_priv->rps.min_freq = dev_priv->rps.efficient_freq;
5060	serge	5428	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
6084	serge	5429	intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
5060	serge	5430	dev_priv->rps.min_freq);
		5431
5354	serge	5432	WARN_ONCE((dev_priv->rps.max_freq \|
		5433	dev_priv->rps.efficient_freq \|
		5434	dev_priv->rps.rp1_freq \|
		5435	dev_priv->rps.min_freq) & 1,
		5436	"Odd GPU freq values\n");
		5437
6084	serge	5438	dev_priv->rps.idle_freq = dev_priv->rps.min_freq;
		5439
5060	serge	5440	/* Preserve min/max settings in case of re-init */
		5441	if (dev_priv->rps.max_freq_softlimit == 0)
		5442	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5443
		5444	if (dev_priv->rps.min_freq_softlimit == 0)
		5445	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5446
		5447	mutex_unlock(&dev_priv->rps.hw_lock);
		5448	}
		5449
		5450	static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
		5451	{
		5452	valleyview_cleanup_pctx(dev);
		5453	}
		5454
		5455	static void cherryview_enable_rps(struct drm_device *dev)
		5456	{
		5457	struct drm_i915_private *dev_priv = dev->dev_private;
		5458	struct intel_engine_cs *ring;
		5459	u32 gtfifodbg, val, rc6_mode = 0, pcbr;
		5460	int i;
		5461
		5462	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5463
		5464	gtfifodbg = I915_READ(GTFIFODBG);
		5465	if (gtfifodbg) {
		5466	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5467	gtfifodbg);
		5468	I915_WRITE(GTFIFODBG, gtfifodbg);
		5469	}
		5470
		5471	cherryview_check_pctx(dev_priv);
		5472
		5473	/* 1a & 1b: Get forcewake during program sequence. Although the driver
		5474	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
6084	serge	5475	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5060	serge	5476
6084	serge	5477	/* Disable RC states. */
		5478	I915_WRITE(GEN6_RC_CONTROL, 0);
		5479
5060	serge	5480	/* 2a: Program RC6 thresholds.*/
		5481	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		5482	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		5483	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		5484
		5485	for_each_ring(ring, dev_priv, i)
		5486	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5487	I915_WRITE(GEN6_RC_SLEEP, 0);
		5488
6084	serge	5489	/* TO threshold set to 500 us ( 0x186 * 1.28 us) */
		5490	I915_WRITE(GEN6_RC6_THRESHOLD, 0x186);
5060	serge	5491
		5492	/* allows RC6 residency counter to work */
		5493	I915_WRITE(VLV_COUNTER_CONTROL,
		5494	_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH \|
		5495	VLV_MEDIA_RC6_COUNT_EN \|
		5496	VLV_RENDER_RC6_COUNT_EN));
		5497
		5498	/* For now we assume BIOS is allocating and populating the PCBR */
		5499	pcbr = I915_READ(VLV_PCBR);
		5500
		5501	/* 3: Enable RC6 */
		5502	if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
		5503	(pcbr >> VLV_PCBR_ADDR_SHIFT))
6084	serge	5504	rc6_mode = GEN7_RC_CTL_TO_MODE;
5060	serge	5505
		5506	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5507
		5508	/* 4 Program defaults and thresholds for RPS*/
6084	serge	5509	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5060	serge	5510	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5511	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5512	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5513	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5514
		5515	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5516
		5517	/* 5: Enable RPS */
		5518	I915_WRITE(GEN6_RP_CONTROL,
		5519	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
6084	serge	5520	GEN6_RP_MEDIA_IS_GFX \|
5060	serge	5521	GEN6_RP_ENABLE \|
		5522	GEN6_RP_UP_BUSY_AVG \|
		5523	GEN6_RP_DOWN_IDLE_AVG);
		5524
6084	serge	5525	/* Setting Fixed Bias */
		5526	val = VLV_OVERRIDE_EN \|
		5527	VLV_SOC_TDP_EN \|
		5528	CHV_BIAS_CPU_50_SOC_50;
		5529	vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
		5530
5060	serge	5531	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5532
5354	serge	5533	/* RPS code assumes GPLL is used */
		5534	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5535
6084	serge	5536	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
5060	serge	5537	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5538
		5539	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
		5540	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
6084	serge	5541	intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5060	serge	5542	dev_priv->rps.cur_freq);
		5543
		5544	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
6084	serge	5545	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5546	dev_priv->rps.efficient_freq);
		5547
		5548	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
		5549
6084	serge	5550	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5060	serge	5551	}
		5552
4104	Serge	5553	static void valleyview_enable_rps(struct drm_device *dev)
		5554	{
		5555	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	5556	struct intel_engine_cs *ring;
4560	Serge	5557	u32 gtfifodbg, val, rc6_mode = 0;
4104	Serge	5558	int i;
		5559
		5560	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5561
5060	serge	5562	valleyview_check_pctx(dev_priv);
		5563
4104	Serge	5564	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4560	Serge	5565	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5566	gtfifodbg);
4104	Serge	5567	I915_WRITE(GTFIFODBG, gtfifodbg);
		5568	}
		5569
4560	Serge	5570	/* If VLV, Forcewake all wells, else re-direct to regular path */
6084	serge	5571	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4104	Serge	5572
6084	serge	5573	/* Disable RC states. */
		5574	I915_WRITE(GEN6_RC_CONTROL, 0);
		5575
		5576	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
4104	Serge	5577	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5578	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5579	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5580	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5581
		5582	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5583
		5584	I915_WRITE(GEN6_RP_CONTROL,
		5585	GEN6_RP_MEDIA_TURBO \|
		5586	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		5587	GEN6_RP_MEDIA_IS_GFX \|
		5588	GEN6_RP_ENABLE \|
		5589	GEN6_RP_UP_BUSY_AVG \|
		5590	GEN6_RP_DOWN_IDLE_CONT);
		5591
		5592	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
		5593	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		5594	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		5595
		5596	for_each_ring(ring, dev_priv, i)
		5597	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5598
4560	Serge	5599	I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4104	Serge	5600
		5601	/* allows RC6 residency counter to work */
4560	Serge	5602	I915_WRITE(VLV_COUNTER_CONTROL,
5060	serge	5603	_MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN \|
		5604	VLV_RENDER_RC0_COUNT_EN \|
4560	Serge	5605	VLV_MEDIA_RC6_COUNT_EN \|
		5606	VLV_RENDER_RC6_COUNT_EN));
5060	serge	5607
4560	Serge	5608	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		5609	rc6_mode = GEN7_RC_CTL_TO_MODE \| VLV_RC_CTL_CTX_RST_PARALLEL;
4104	Serge	5610
4560	Serge	5611	intel_print_rc6_info(dev, rc6_mode);
		5612
		5613	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5614
6084	serge	5615	/* Setting Fixed Bias */
		5616	val = VLV_OVERRIDE_EN \|
		5617	VLV_SOC_TDP_EN \|
		5618	VLV_BIAS_CPU_125_SOC_875;
		5619	vlv_punit_write(dev_priv, VLV_TURBO_SOC_OVERRIDE, val);
		5620
4104	Serge	5621	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5622
5354	serge	5623	/* RPS code assumes GPLL is used */
		5624	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5625
6084	serge	5626	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE));
4104	Serge	5627	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5628
5060	serge	5629	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4104	Serge	5630	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
6084	serge	5631	intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5060	serge	5632	dev_priv->rps.cur_freq);
4104	Serge	5633
		5634	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
6084	serge	5635	intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5060	serge	5636	dev_priv->rps.efficient_freq);
4104	Serge	5637
5060	serge	5638	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4104	Serge	5639
6084	serge	5640	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	5641	}
		5642
3031	serge	5643	static unsigned long intel_pxfreq(u32 vidfreq)
		5644	{
		5645	unsigned long freq;
		5646	int div = (vidfreq & 0x3f0000) >> 16;
		5647	int post = (vidfreq & 0x3000) >> 12;
		5648	int pre = (vidfreq & 0x7);
		5649
		5650	if (!pre)
		5651	return 0;
		5652
		5653	freq = ((div * 133333) / ((1<
		5654
		5655	return freq;
		5656	}
		5657
		5658	static const struct cparams {
		5659	u16 i;
		5660	u16 t;
		5661	u16 m;
		5662	u16 c;
		5663	} cparams[] = {
		5664	{ 1, 1333, 301, 28664 },
		5665	{ 1, 1066, 294, 24460 },
		5666	{ 1, 800, 294, 25192 },
		5667	{ 0, 1333, 276, 27605 },
		5668	{ 0, 1066, 276, 27605 },
		5669	{ 0, 800, 231, 23784 },
		5670	};
		5671
		5672	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		5673	{
		5674	u64 total_count, diff, ret;
		5675	u32 count1, count2, count3, m = 0, c = 0;
5060	serge	5676	unsigned long now = jiffies_to_msecs(jiffies), diff1;
3031	serge	5677	int i;
		5678
		5679	assert_spin_locked(&mchdev_lock);
		5680
		5681	diff1 = now - dev_priv->ips.last_time1;
		5682
		5683	/* Prevent division-by-zero if we are asking too fast.
		5684	* Also, we don't get interesting results if we are polling
		5685	* faster than once in 10ms, so just return the saved value
		5686	* in such cases.
		5687	*/
		5688	if (diff1 <= 10)
		5689	return dev_priv->ips.chipset_power;
		5690
		5691	count1 = I915_READ(DMIEC);
		5692	count2 = I915_READ(DDREC);
		5693	count3 = I915_READ(CSIEC);
		5694
		5695	total_count = count1 + count2 + count3;
		5696
		5697	/* FIXME: handle per-counter overflow */
		5698	if (total_count < dev_priv->ips.last_count1) {
		5699	diff = ~0UL - dev_priv->ips.last_count1;
		5700	diff += total_count;
		5701	} else {
		5702	diff = total_count - dev_priv->ips.last_count1;
		5703	}
		5704
		5705	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		5706	if (cparams[i].i == dev_priv->ips.c_m &&
		5707	cparams[i].t == dev_priv->ips.r_t) {
		5708	m = cparams[i].m;
		5709	c = cparams[i].c;
		5710	break;
		5711	}
		5712	}
		5713
		5714	diff = div_u64(diff, diff1);
		5715	ret = ((m * diff) + c);
		5716	ret = div_u64(ret, 10);
		5717
		5718	dev_priv->ips.last_count1 = total_count;
		5719	dev_priv->ips.last_time1 = now;
		5720
		5721	dev_priv->ips.chipset_power = ret;
		5722
		5723	return ret;
		5724	}
		5725
		5726	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		5727	{
5060	serge	5728	struct drm_device *dev = dev_priv->dev;
3031	serge	5729	unsigned long val;
		5730
5060	serge	5731	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5732	return 0;
		5733
		5734	spin_lock_irq(&mchdev_lock);
		5735
		5736	val = __i915_chipset_val(dev_priv);
		5737
		5738	spin_unlock_irq(&mchdev_lock);
		5739
		5740	return val;
		5741	}
		5742
		5743	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		5744	{
		5745	unsigned long m, x, b;
		5746	u32 tsfs;
		5747
		5748	tsfs = I915_READ(TSFS);
		5749
		5750	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		5751	x = I915_READ8(TR1);
		5752
		5753	b = tsfs & TSFS_INTR_MASK;
		5754
		5755	return ((m * x) / 127) - b;
		5756	}
		5757
6084	serge	5758	static int _pxvid_to_vd(u8 pxvid)
3031	serge	5759	{
6084	serge	5760	if (pxvid == 0)
		5761	return 0;
		5762
		5763	if (pxvid >= 8 && pxvid < 31)
		5764	pxvid = 31;
		5765
		5766	return (pxvid + 2) * 125;
		5767	}
		5768
		5769	static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		5770	{
5060	serge	5771	struct drm_device *dev = dev_priv->dev;
6084	serge	5772	const int vd = _pxvid_to_vd(pxvid);
		5773	const int vm = vd - 1125;
		5774
5060	serge	5775	if (INTEL_INFO(dev)->is_mobile)
6084	serge	5776	return vm > 0 ? vm : 0;
		5777
		5778	return vd;
3031	serge	5779	}
		5780
		5781	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5782	{
5060	serge	5783	u64 now, diff, diffms;
3031	serge	5784	u32 count;
		5785
		5786	assert_spin_locked(&mchdev_lock);
		5787
5060	serge	5788	now = ktime_get_raw_ns();
		5789	diffms = now - dev_priv->ips.last_time2;
		5790	do_div(diffms, NSEC_PER_MSEC);
3031	serge	5791
		5792	/* Don't divide by 0 */
		5793	if (!diffms)
		5794	return;
		5795
		5796	count = I915_READ(GFXEC);
		5797
		5798	if (count < dev_priv->ips.last_count2) {
		5799	diff = ~0UL - dev_priv->ips.last_count2;
		5800	diff += count;
		5801	} else {
		5802	diff = count - dev_priv->ips.last_count2;
		5803	}
		5804
		5805	dev_priv->ips.last_count2 = count;
		5806	dev_priv->ips.last_time2 = now;
		5807
		5808	/* More magic constants... */
		5809	diff = diff * 1181;
		5810	diff = div_u64(diff, diffms * 10);
		5811	dev_priv->ips.gfx_power = diff;
		5812	}
		5813
		5814	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5815	{
5060	serge	5816	struct drm_device *dev = dev_priv->dev;
		5817
		5818	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5819	return;
		5820
		5821	spin_lock_irq(&mchdev_lock);
		5822
		5823	__i915_update_gfx_val(dev_priv);
		5824
		5825	spin_unlock_irq(&mchdev_lock);
		5826	}
		5827
		5828	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		5829	{
		5830	unsigned long t, corr, state1, corr2, state2;
		5831	u32 pxvid, ext_v;
		5832
		5833	assert_spin_locked(&mchdev_lock);
		5834
6084	serge	5835	pxvid = I915_READ(PXVFREQ(dev_priv->rps.cur_freq));
3031	serge	5836	pxvid = (pxvid >> 24) & 0x7f;
		5837	ext_v = pvid_to_extvid(dev_priv, pxvid);
		5838
		5839	state1 = ext_v;
		5840
		5841	t = i915_mch_val(dev_priv);
		5842
		5843	/* Revel in the empirically derived constants */
		5844
		5845	/* Correction factor in 1/100000 units */
		5846	if (t > 80)
		5847	corr = ((t * 2349) + 135940);
		5848	else if (t >= 50)
		5849	corr = ((t * 964) + 29317);
		5850	else /* < 50 */
		5851	corr = ((t * 301) + 1004);
		5852
		5853	corr = corr * ((150142 * state1) / 10000 - 78642);
		5854	corr /= 100000;
		5855	corr2 = (corr * dev_priv->ips.corr);
		5856
		5857	state2 = (corr2 * state1) / 10000;
		5858	state2 /= 100; /* convert to mW */
		5859
		5860	__i915_update_gfx_val(dev_priv);
		5861
		5862	return dev_priv->ips.gfx_power + state2;
		5863	}
		5864
		5865	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		5866	{
5060	serge	5867	struct drm_device *dev = dev_priv->dev;
3031	serge	5868	unsigned long val;
		5869
5060	serge	5870	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5871	return 0;
		5872
		5873	spin_lock_irq(&mchdev_lock);
		5874
		5875	val = __i915_gfx_val(dev_priv);
		5876
		5877	spin_unlock_irq(&mchdev_lock);
		5878
		5879	return val;
		5880	}
		5881
		5882	/**
		5883	* i915_read_mch_val - return value for IPS use
		5884	*
		5885	* Calculate and return a value for the IPS driver to use when deciding whether
		5886	* we have thermal and power headroom to increase CPU or GPU power budget.
		5887	*/
		5888	unsigned long i915_read_mch_val(void)
		5889	{
		5890	struct drm_i915_private *dev_priv;
		5891	unsigned long chipset_val, graphics_val, ret = 0;
		5892
		5893	spin_lock_irq(&mchdev_lock);
		5894	if (!i915_mch_dev)
		5895	goto out_unlock;
		5896	dev_priv = i915_mch_dev;
		5897
		5898	chipset_val = __i915_chipset_val(dev_priv);
		5899	graphics_val = __i915_gfx_val(dev_priv);
		5900
		5901	ret = chipset_val + graphics_val;
		5902
		5903	out_unlock:
		5904	spin_unlock_irq(&mchdev_lock);
		5905
		5906	return ret;
		5907	}
		5908	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		5909
		5910	/**
		5911	* i915_gpu_raise - raise GPU frequency limit
		5912	*
		5913	* Raise the limit; IPS indicates we have thermal headroom.
		5914	*/
		5915	bool i915_gpu_raise(void)
		5916	{
		5917	struct drm_i915_private *dev_priv;
		5918	bool ret = true;
		5919
		5920	spin_lock_irq(&mchdev_lock);
		5921	if (!i915_mch_dev) {
		5922	ret = false;
		5923	goto out_unlock;
		5924	}
		5925	dev_priv = i915_mch_dev;
		5926
		5927	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		5928	dev_priv->ips.max_delay--;
		5929
		5930	out_unlock:
		5931	spin_unlock_irq(&mchdev_lock);
		5932
		5933	return ret;
		5934	}
		5935	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		5936
		5937	/**
		5938	* i915_gpu_lower - lower GPU frequency limit
		5939	*
		5940	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		5941	* frequency maximum.
		5942	*/
		5943	bool i915_gpu_lower(void)
		5944	{
		5945	struct drm_i915_private *dev_priv;
		5946	bool ret = true;
		5947
		5948	spin_lock_irq(&mchdev_lock);
		5949	if (!i915_mch_dev) {
		5950	ret = false;
		5951	goto out_unlock;
		5952	}
		5953	dev_priv = i915_mch_dev;
		5954
		5955	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		5956	dev_priv->ips.max_delay++;
		5957
		5958	out_unlock:
		5959	spin_unlock_irq(&mchdev_lock);
		5960
		5961	return ret;
		5962	}
		5963	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		5964
		5965	/**
		5966	* i915_gpu_busy - indicate GPU business to IPS
		5967	*
		5968	* Tell the IPS driver whether or not the GPU is busy.
		5969	*/
		5970	bool i915_gpu_busy(void)
		5971	{
		5972	struct drm_i915_private *dev_priv;
5060	serge	5973	struct intel_engine_cs *ring;
3031	serge	5974	bool ret = false;
		5975	int i;
		5976
		5977	spin_lock_irq(&mchdev_lock);
		5978	if (!i915_mch_dev)
		5979	goto out_unlock;
		5980	dev_priv = i915_mch_dev;
		5981
		5982	for_each_ring(ring, dev_priv, i)
		5983	ret \|= !list_empty(&ring->request_list);
		5984
		5985	out_unlock:
		5986	spin_unlock_irq(&mchdev_lock);
		5987
		5988	return ret;
		5989	}
		5990	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		5991
		5992	/**
		5993	* i915_gpu_turbo_disable - disable graphics turbo
		5994	*
		5995	* Disable graphics turbo by resetting the max frequency and setting the
		5996	* current frequency to the default.
		5997	*/
		5998	bool i915_gpu_turbo_disable(void)
		5999	{
		6000	struct drm_i915_private *dev_priv;
		6001	bool ret = true;
		6002
		6003	spin_lock_irq(&mchdev_lock);
		6004	if (!i915_mch_dev) {
		6005	ret = false;
		6006	goto out_unlock;
		6007	}
		6008	dev_priv = i915_mch_dev;
		6009
		6010	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		6011
		6012	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		6013	ret = false;
		6014
		6015	out_unlock:
		6016	spin_unlock_irq(&mchdev_lock);
		6017
		6018	return ret;
		6019	}
		6020	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		6021
		6022	/**
		6023	* Tells the intel_ips driver that the i915 driver is now loaded, if
		6024	* IPS got loaded first.
		6025	*
		6026	* This awkward dance is so that neither module has to depend on the
		6027	* other in order for IPS to do the appropriate communication of
		6028	* GPU turbo limits to i915.
		6029	*/
		6030	static void
		6031	ips_ping_for_i915_load(void)
		6032	{
		6033	void (*link)(void);
		6034
		6035	// link = symbol_get(ips_link_to_i915_driver);
		6036	// if (link) {
		6037	// link();
		6038	// symbol_put(ips_link_to_i915_driver);
		6039	// }
		6040	}
		6041
		6042	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		6043	{
		6044	/* We only register the i915 ips part with intel-ips once everything is
		6045	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		6046	spin_lock_irq(&mchdev_lock);
		6047	i915_mch_dev = dev_priv;
		6048	spin_unlock_irq(&mchdev_lock);
		6049
		6050	ips_ping_for_i915_load();
		6051	}
		6052
		6053	void intel_gpu_ips_teardown(void)
		6054	{
		6055	spin_lock_irq(&mchdev_lock);
		6056	i915_mch_dev = NULL;
		6057	spin_unlock_irq(&mchdev_lock);
		6058	}
5060	serge	6059
3031	serge	6060	static void intel_init_emon(struct drm_device *dev)
		6061	{
		6062	struct drm_i915_private *dev_priv = dev->dev_private;
		6063	u32 lcfuse;
		6064	u8 pxw[16];
		6065	int i;
		6066
		6067	/* Disable to program */
		6068	I915_WRITE(ECR, 0);
		6069	POSTING_READ(ECR);
		6070
		6071	/* Program energy weights for various events */
		6072	I915_WRITE(SDEW, 0x15040d00);
		6073	I915_WRITE(CSIEW0, 0x007f0000);
		6074	I915_WRITE(CSIEW1, 0x1e220004);
		6075	I915_WRITE(CSIEW2, 0x04000004);
		6076
		6077	for (i = 0; i < 5; i++)
6084	serge	6078	I915_WRITE(PEW(i), 0);
3031	serge	6079	for (i = 0; i < 3; i++)
6084	serge	6080	I915_WRITE(DEW(i), 0);
3031	serge	6081
		6082	/* Program P-state weights to account for frequency power adjustment */
		6083	for (i = 0; i < 16; i++) {
6084	serge	6084	u32 pxvidfreq = I915_READ(PXVFREQ(i));
3031	serge	6085	unsigned long freq = intel_pxfreq(pxvidfreq);
		6086	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		6087	PXVFREQ_PX_SHIFT;
		6088	unsigned long val;
		6089
		6090	val = vid * vid;
		6091	val *= (freq / 1000);
		6092	val *= 255;
		6093	val /= (127127900);
		6094	if (val > 0xff)
		6095	DRM_ERROR("bad pxval: %ld\n", val);
		6096	pxw[i] = val;
		6097	}
		6098	/* Render standby states get 0 weight */
		6099	pxw[14] = 0;
		6100	pxw[15] = 0;
		6101
		6102	for (i = 0; i < 4; i++) {
		6103	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		6104	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
6084	serge	6105	I915_WRITE(PXW(i), val);
3031	serge	6106	}
		6107
		6108	/* Adjust magic regs to magic values (more experimental results) */
		6109	I915_WRITE(OGW0, 0);
		6110	I915_WRITE(OGW1, 0);
		6111	I915_WRITE(EG0, 0x00007f00);
		6112	I915_WRITE(EG1, 0x0000000e);
		6113	I915_WRITE(EG2, 0x000e0000);
		6114	I915_WRITE(EG3, 0x68000300);
		6115	I915_WRITE(EG4, 0x42000000);
		6116	I915_WRITE(EG5, 0x00140031);
		6117	I915_WRITE(EG6, 0);
		6118	I915_WRITE(EG7, 0);
		6119
		6120	for (i = 0; i < 8; i++)
6084	serge	6121	I915_WRITE(PXWL(i), 0);
3031	serge	6122
		6123	/* Enable PMON + select events */
		6124	I915_WRITE(ECR, 0x80000019);
		6125
		6126	lcfuse = I915_READ(LCFUSE02);
		6127
		6128	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		6129	}
		6130
5060	serge	6131	void intel_init_gt_powersave(struct drm_device *dev)
		6132	{
		6133	i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
		6134
		6135	if (IS_CHERRYVIEW(dev))
		6136	cherryview_init_gt_powersave(dev);
		6137	else if (IS_VALLEYVIEW(dev))
		6138	valleyview_init_gt_powersave(dev);
		6139	}
		6140
		6141	void intel_cleanup_gt_powersave(struct drm_device *dev)
		6142	{
		6143	if (IS_CHERRYVIEW(dev))
		6144	return;
		6145	else if (IS_VALLEYVIEW(dev))
		6146	valleyview_cleanup_gt_powersave(dev);
		6147	}
		6148
5354	serge	6149	static void gen6_suspend_rps(struct drm_device *dev)
		6150	{
		6151	struct drm_i915_private *dev_priv = dev->dev_private;
		6152
		6153	// flush_delayed_work(&dev_priv->rps.delayed_resume_work);
		6154
6084	serge	6155	gen6_disable_rps_interrupts(dev);
5354	serge	6156	}
		6157
5060	serge	6158	/**
		6159	* intel_suspend_gt_powersave - suspend PM work and helper threads
		6160	* @dev: drm device
		6161	*
		6162	* We don't want to disable RC6 or other features here, we just want
		6163	* to make sure any work we've queued has finished and won't bother
		6164	* us while we're suspended.
		6165	*/
		6166	void intel_suspend_gt_powersave(struct drm_device *dev)
		6167	{
		6168	struct drm_i915_private *dev_priv = dev->dev_private;
		6169
5354	serge	6170	if (INTEL_INFO(dev)->gen < 6)
		6171	return;
5060	serge	6172
5354	serge	6173	gen6_suspend_rps(dev);
5060	serge	6174
		6175	/* Force GPU to min freq during suspend */
		6176	gen6_rps_idle(dev_priv);
		6177	}
		6178
3031	serge	6179	void intel_disable_gt_powersave(struct drm_device *dev)
		6180	{
3243	Serge	6181	struct drm_i915_private *dev_priv = dev->dev_private;
		6182
3031	serge	6183	if (IS_IRONLAKE_M(dev)) {
		6184	ironlake_disable_drps(dev);
4293	Serge	6185	} else if (INTEL_INFO(dev)->gen >= 6) {
5060	serge	6186	intel_suspend_gt_powersave(dev);
		6187
3482	Serge	6188	mutex_lock(&dev_priv->rps.hw_lock);
5354	serge	6189	if (INTEL_INFO(dev)->gen >= 9)
		6190	gen9_disable_rps(dev);
		6191	else if (IS_CHERRYVIEW(dev))
5060	serge	6192	cherryview_disable_rps(dev);
		6193	else if (IS_VALLEYVIEW(dev))
4104	Serge	6194	valleyview_disable_rps(dev);
		6195	else
6084	serge	6196	gen6_disable_rps(dev);
5354	serge	6197
4560	Serge	6198	dev_priv->rps.enabled = false;
3480	Serge	6199	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	6200	}
		6201	}
		6202
3482	Serge	6203	static void intel_gen6_powersave_work(struct work_struct *work)
		6204	{
		6205	struct drm_i915_private *dev_priv =
		6206	container_of(work, struct drm_i915_private,
		6207	rps.delayed_resume_work.work);
		6208	struct drm_device *dev = dev_priv->dev;
		6209
		6210	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	6211
6084	serge	6212	gen6_reset_rps_interrupts(dev);
5354	serge	6213
5060	serge	6214	if (IS_CHERRYVIEW(dev)) {
		6215	cherryview_enable_rps(dev);
		6216	} else if (IS_VALLEYVIEW(dev)) {
4104	Serge	6217	valleyview_enable_rps(dev);
5354	serge	6218	} else if (INTEL_INFO(dev)->gen >= 9) {
6084	serge	6219	gen9_enable_rc6(dev);
5354	serge	6220	gen9_enable_rps(dev);
6084	serge	6221	if (IS_SKYLAKE(dev))
		6222	__gen6_update_ring_freq(dev);
4560	Serge	6223	} else if (IS_BROADWELL(dev)) {
		6224	gen8_enable_rps(dev);
5060	serge	6225	__gen6_update_ring_freq(dev);
4104	Serge	6226	} else {
6084	serge	6227	gen6_enable_rps(dev);
5060	serge	6228	__gen6_update_ring_freq(dev);
4104	Serge	6229	}
6084	serge	6230
		6231	WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
		6232	WARN_ON(dev_priv->rps.idle_freq > dev_priv->rps.max_freq);
		6233
		6234	WARN_ON(dev_priv->rps.efficient_freq < dev_priv->rps.min_freq);
		6235	WARN_ON(dev_priv->rps.efficient_freq > dev_priv->rps.max_freq);
		6236
4560	Serge	6237	dev_priv->rps.enabled = true;
5354	serge	6238
6084	serge	6239	gen6_enable_rps_interrupts(dev);
5354	serge	6240
3482	Serge	6241	mutex_unlock(&dev_priv->rps.hw_lock);
5060	serge	6242
		6243	intel_runtime_pm_put(dev_priv);
3482	Serge	6244	}
		6245
3031	serge	6246	void intel_enable_gt_powersave(struct drm_device *dev)
		6247	{
3243	Serge	6248	struct drm_i915_private *dev_priv = dev->dev_private;
		6249
6084	serge	6250	/* Powersaving is controlled by the host when inside a VM */
		6251	if (intel_vgpu_active(dev))
		6252	return;
		6253
3031	serge	6254	if (IS_IRONLAKE_M(dev)) {
5060	serge	6255	mutex_lock(&dev->struct_mutex);
3031	serge	6256	ironlake_enable_drps(dev);
		6257	intel_init_emon(dev);
5060	serge	6258	mutex_unlock(&dev->struct_mutex);
		6259	} else if (INTEL_INFO(dev)->gen >= 6) {
3243	Serge	6260	/*
		6261	* PCU communication is slow and this doesn't need to be
		6262	* done at any specific time, so do this out of our fast path
		6263	* to make resume and init faster.
5060	serge	6264	*
		6265	* We depend on the HW RC6 power context save/restore
		6266	* mechanism when entering D3 through runtime PM suspend. So
		6267	* disable RPM until RPS/RC6 is properly setup. We can only
		6268	* get here via the driver load/system resume/runtime resume
		6269	* paths, so the _noresume version is enough (and in case of
		6270	* runtime resume it's necessary).
3243	Serge	6271	*/
5060	serge	6272	if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		6273	round_jiffies_up_relative(HZ)))
		6274	intel_runtime_pm_get_noresume(dev_priv);
3031	serge	6275	}
		6276	}
		6277
5060	serge	6278	void intel_reset_gt_powersave(struct drm_device *dev)
		6279	{
		6280	struct drm_i915_private *dev_priv = dev->dev_private;
		6281
5354	serge	6282	if (INTEL_INFO(dev)->gen < 6)
		6283	return;
		6284
		6285	gen6_suspend_rps(dev);
5060	serge	6286	dev_priv->rps.enabled = false;
		6287	}
		6288
3243	Serge	6289	static void ibx_init_clock_gating(struct drm_device *dev)
		6290	{
		6291	struct drm_i915_private *dev_priv = dev->dev_private;
		6292
		6293	/*
		6294	* On Ibex Peak and Cougar Point, we need to disable clock
		6295	* gating for the panel power sequencer or it will fail to
		6296	* start up when no ports are active.
		6297	*/
		6298	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		6299	}
		6300
4104	Serge	6301	static void g4x_disable_trickle_feed(struct drm_device *dev)
		6302	{
		6303	struct drm_i915_private *dev_priv = dev->dev_private;
6084	serge	6304	enum pipe pipe;
4104	Serge	6305
5354	serge	6306	for_each_pipe(dev_priv, pipe) {
4104	Serge	6307	I915_WRITE(DSPCNTR(pipe),
		6308	I915_READ(DSPCNTR(pipe)) \|
		6309	DISPPLANE_TRICKLE_FEED_DISABLE);
6084	serge	6310
		6311	I915_WRITE(DSPSURF(pipe), I915_READ(DSPSURF(pipe)));
		6312	POSTING_READ(DSPSURF(pipe));
4104	Serge	6313	}
		6314	}
		6315
4560	Serge	6316	static void ilk_init_lp_watermarks(struct drm_device *dev)
		6317	{
		6318	struct drm_i915_private *dev_priv = dev->dev_private;
		6319
		6320	I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
		6321	I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
		6322	I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
		6323
		6324	/*
		6325	* Don't touch WM1S_LP_EN here.
		6326	* Doing so could cause underruns.
		6327	*/
		6328	}
		6329
3031	serge	6330	static void ironlake_init_clock_gating(struct drm_device *dev)
		6331	{
		6332	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6333	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6334
4104	Serge	6335	/*
		6336	* Required for FBC
		6337	* WaFbcDisableDpfcClockGating:ilk
		6338	*/
3243	Serge	6339	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		6340	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		6341	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	6342
		6343	I915_WRITE(PCH_3DCGDIS0,
		6344	MARIUNIT_CLOCK_GATE_DISABLE \|
		6345	SVSMUNIT_CLOCK_GATE_DISABLE);
		6346	I915_WRITE(PCH_3DCGDIS1,
		6347	VFMUNIT_CLOCK_GATE_DISABLE);
		6348
		6349	/*
		6350	* According to the spec the following bits should be set in
		6351	* order to enable memory self-refresh
		6352	* The bit 22/21 of 0x42004
		6353	* The bit 5 of 0x42020
		6354	* The bit 15 of 0x45000
		6355	*/
		6356	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6357	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6358	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	6359	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	6360	I915_WRITE(DISP_ARB_CTL,
		6361	(I915_READ(DISP_ARB_CTL) \|
		6362	DISP_FBC_WM_DIS));
		6363
4560	Serge	6364	ilk_init_lp_watermarks(dev);
		6365
3031	serge	6366	/*
		6367	* Based on the document from hardware guys the following bits
		6368	* should be set unconditionally in order to enable FBC.
		6369	* The bit 22 of 0x42000
		6370	* The bit 22 of 0x42004
		6371	* The bit 7,8,9 of 0x42020.
		6372	*/
		6373	if (IS_IRONLAKE_M(dev)) {
4104	Serge	6374	/* WaFbcAsynchFlipDisableFbcQueue:ilk */
3031	serge	6375	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6376	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6377	ILK_FBCQ_DIS);
		6378	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6379	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6380	ILK_DPARB_GATE);
		6381	}
		6382
3243	Serge	6383	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		6384
3031	serge	6385	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6386	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6387	ILK_ELPIN_409_SELECT);
		6388	I915_WRITE(_3D_CHICKEN2,
		6389	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		6390	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	6391
4104	Serge	6392	/* WaDisableRenderCachePipelinedFlush:ilk */
3243	Serge	6393	I915_WRITE(CACHE_MODE_0,
		6394	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		6395
5060	serge	6396	/* WaDisable_RenderCache_OperationalFlush:ilk */
		6397	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6398
4104	Serge	6399	g4x_disable_trickle_feed(dev);
		6400
3243	Serge	6401	ibx_init_clock_gating(dev);
3031	serge	6402	}
		6403
3243	Serge	6404	static void cpt_init_clock_gating(struct drm_device *dev)
		6405	{
		6406	struct drm_i915_private *dev_priv = dev->dev_private;
		6407	int pipe;
3746	Serge	6408	uint32_t val;
3243	Serge	6409
		6410	/*
		6411	* On Ibex Peak and Cougar Point, we need to disable clock
		6412	* gating for the panel power sequencer or it will fail to
		6413	* start up when no ports are active.
		6414	*/
4280	Serge	6415	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE \|
		6416	PCH_DPLUNIT_CLOCK_GATE_DISABLE \|
		6417	PCH_CPUNIT_CLOCK_GATE_DISABLE);
3243	Serge	6418	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		6419	DPLS_EDP_PPS_FIX_DIS);
		6420	/* The below fixes the weird display corruption, a few pixels shifted
		6421	* downward, on (only) LVDS of some HP laptops with IVY.
		6422	*/
5354	serge	6423	for_each_pipe(dev_priv, pipe) {
3746	Serge	6424	val = I915_READ(TRANS_CHICKEN2(pipe));
		6425	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		6426	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4104	Serge	6427	if (dev_priv->vbt.fdi_rx_polarity_inverted)
3746	Serge	6428	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		6429	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		6430	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		6431	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		6432	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		6433	}
3243	Serge	6434	/* WADP0ClockGatingDisable */
5354	serge	6435	for_each_pipe(dev_priv, pipe) {
3243	Serge	6436	I915_WRITE(TRANS_CHICKEN1(pipe),
		6437	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		6438	}
		6439	}
		6440
3480	Serge	6441	static void gen6_check_mch_setup(struct drm_device *dev)
		6442	{
		6443	struct drm_i915_private *dev_priv = dev->dev_private;
		6444	uint32_t tmp;
		6445
		6446	tmp = I915_READ(MCH_SSKPD);
5060	serge	6447	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
		6448	DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
		6449	tmp);
3480	Serge	6450	}
		6451
3031	serge	6452	static void gen6_init_clock_gating(struct drm_device *dev)
		6453	{
		6454	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6455	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6456
3243	Serge	6457	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	6458
		6459	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6460	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6461	ILK_ELPIN_409_SELECT);
		6462
4104	Serge	6463	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3243	Serge	6464	I915_WRITE(_3D_CHICKEN,
		6465	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		6466
5060	serge	6467	/* WaDisable_RenderCache_OperationalFlush:snb */
		6468	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6469
		6470	/*
		6471	* BSpec recoomends 8x4 when MSAA is used,
		6472	* however in practice 16x4 seems fastest.
		6473	*
		6474	* Note that PS/WM thread counts depend on the WIZ hashing
		6475	* disable bit, which we don't touch here, but it's good
		6476	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6477	*/
		6478	I915_WRITE(GEN6_GT_MODE,
5354	serge	6479	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6480
4560	Serge	6481	ilk_init_lp_watermarks(dev);
3031	serge	6482
		6483	I915_WRITE(CACHE_MODE_0,
		6484	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		6485
		6486	I915_WRITE(GEN6_UCGCTL1,
		6487	I915_READ(GEN6_UCGCTL1) \|
		6488	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		6489	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6490
		6491	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		6492	* gating disable must be set. Failure to set it results in
		6493	* flickering pixels due to Z write ordering failures after
		6494	* some amount of runtime in the Mesa "fire" demo, and Unigine
		6495	* Sanctuary and Tropics, and apparently anything else with
		6496	* alpha test or pixel discard.
		6497	*
		6498	* According to the spec, bit 11 (RCCUNIT) must also be set,
		6499	* but we didn't debug actual testcases to find it out.
		6500	*
5060	serge	6501	* WaDisableRCCUnitClockGating:snb
		6502	* WaDisableRCPBUnitClockGating:snb
3031	serge	6503	*/
		6504	I915_WRITE(GEN6_UCGCTL2,
		6505	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		6506	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		6507
5060	serge	6508	/* WaStripsFansDisableFastClipPerformanceFix:snb */
		6509	I915_WRITE(_3D_CHICKEN3,
		6510	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
3031	serge	6511
		6512	/*
5060	serge	6513	* Bspec says:
		6514	* "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
		6515	* 3DSTATE_SF number of SF output attributes is more than 16."
		6516	*/
		6517	I915_WRITE(_3D_CHICKEN3,
		6518	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
		6519
		6520	/*
3031	serge	6521	* According to the spec the following bits should be
		6522	* set in order to enable memory self-refresh and fbc:
		6523	* The bit21 and bit22 of 0x42000
		6524	* The bit21 and bit22 of 0x42004
		6525	* The bit5 and bit7 of 0x42020
		6526	* The bit14 of 0x70180
		6527	* The bit14 of 0x71180
4104	Serge	6528	*
		6529	* WaFbcAsynchFlipDisableFbcQueue:snb
3031	serge	6530	*/
		6531	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6532	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6533	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		6534	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6535	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6536	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	6537	I915_WRITE(ILK_DSPCLK_GATE_D,
		6538	I915_READ(ILK_DSPCLK_GATE_D) \|
		6539	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		6540	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	6541
4104	Serge	6542	g4x_disable_trickle_feed(dev);
3031	serge	6543
3243	Serge	6544	cpt_init_clock_gating(dev);
3480	Serge	6545
		6546	gen6_check_mch_setup(dev);
3031	serge	6547	}
		6548
		6549	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		6550	{
		6551	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		6552
5060	serge	6553	/*
		6554	* WaVSThreadDispatchOverride:ivb,vlv
		6555	*
		6556	* This actually overrides the dispatch
		6557	* mode for all thread types.
		6558	*/
3031	serge	6559	reg &= ~GEN7_FF_SCHED_MASK;
		6560	reg \|= GEN7_FF_TS_SCHED_HW;
		6561	reg \|= GEN7_FF_VS_SCHED_HW;
		6562	reg \|= GEN7_FF_DS_SCHED_HW;
		6563
		6564	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		6565	}
		6566
3243	Serge	6567	static void lpt_init_clock_gating(struct drm_device *dev)
		6568	{
		6569	struct drm_i915_private *dev_priv = dev->dev_private;
		6570
		6571	/*
		6572	* TODO: this bit should only be enabled when really needed, then
		6573	* disabled when not needed anymore in order to save power.
		6574	*/
6084	serge	6575	if (HAS_PCH_LPT_LP(dev))
3243	Serge	6576	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		6577	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		6578	PCH_LP_PARTITION_LEVEL_DISABLE);
4104	Serge	6579
		6580	/* WADPOClockGatingDisable:hsw */
6084	serge	6581	I915_WRITE(TRANS_CHICKEN1(PIPE_A),
		6582	I915_READ(TRANS_CHICKEN1(PIPE_A)) \|
4104	Serge	6583	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3243	Serge	6584	}
		6585
4104	Serge	6586	static void lpt_suspend_hw(struct drm_device *dev)
		6587	{
		6588	struct drm_i915_private *dev_priv = dev->dev_private;
		6589
6084	serge	6590	if (HAS_PCH_LPT_LP(dev)) {
4104	Serge	6591	uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
		6592
		6593	val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		6594	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
		6595	}
		6596	}
		6597
5354	serge	6598	static void broadwell_init_clock_gating(struct drm_device *dev)
3031	serge	6599	{
		6600	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	6601	enum pipe pipe;
6084	serge	6602	uint32_t misccpctl;
3031	serge	6603
6084	serge	6604	ilk_init_lp_watermarks(dev);
3031	serge	6605
4560	Serge	6606	/* WaSwitchSolVfFArbitrationPriority:bdw */
		6607	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6608
		6609	/* WaPsrDPAMaskVBlankInSRD:bdw */
		6610	I915_WRITE(CHICKEN_PAR1_1,
		6611	I915_READ(CHICKEN_PAR1_1) \| DPA_MASK_VBLANK_SRD);
		6612
		6613	/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
5354	serge	6614	for_each_pipe(dev_priv, pipe) {
5060	serge	6615	I915_WRITE(CHICKEN_PIPESL_1(pipe),
		6616	I915_READ(CHICKEN_PIPESL_1(pipe)) \|
		6617	BDW_DPRS_MASK_VBLANK_SRD);
4560	Serge	6618	}
		6619
		6620	/* WaVSRefCountFullforceMissDisable:bdw */
		6621	/* WaDSRefCountFullforceMissDisable:bdw */
		6622	I915_WRITE(GEN7_FF_THREAD_MODE,
		6623	I915_READ(GEN7_FF_THREAD_MODE) &
		6624	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
5060	serge	6625
		6626	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6627	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6628
		6629	/* WaDisableSDEUnitClockGating:bdw */
		6630	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6631	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		6632
6084	serge	6633	/*
		6634	* WaProgramL3SqcReg1Default:bdw
		6635	* WaTempDisableDOPClkGating:bdw
		6636	*/
		6637	misccpctl = I915_READ(GEN7_MISCCPCTL);
		6638	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
		6639	I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT);
6660	serge	6640	/*
		6641	* Wait at least 100 clocks before re-enabling clock gating. See
		6642	* the definition of L3SQCREG1 in BSpec.
		6643	*/
		6644	POSTING_READ(GEN8_L3SQCREG1);
		6645	udelay(1);
6084	serge	6646	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
		6647
		6648	/*
		6649	* WaGttCachingOffByDefault:bdw
		6650	* GTT cache may not work with big pages, so if those
		6651	* are ever enabled GTT cache may need to be disabled.
		6652	*/
		6653	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
		6654
5354	serge	6655	lpt_init_clock_gating(dev);
4560	Serge	6656	}
		6657
		6658	static void haswell_init_clock_gating(struct drm_device *dev)
		6659	{
		6660	struct drm_i915_private *dev_priv = dev->dev_private;
		6661
		6662	ilk_init_lp_watermarks(dev);
		6663
4104	Serge	6664	/* L3 caching of data atomics doesn't work -- disable it. */
		6665	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
		6666	I915_WRITE(HSW_ROW_CHICKEN3,
		6667	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
		6668
		6669	/* This is required by WaCatErrorRejectionIssue:hsw */
3031	serge	6670	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6671	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6672	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6673
4104	Serge	6674	/* WaVSRefCountFullforceMissDisable:hsw */
5060	serge	6675	I915_WRITE(GEN7_FF_THREAD_MODE,
		6676	I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
3031	serge	6677
5060	serge	6678	/* WaDisable_RenderCache_OperationalFlush:hsw */
		6679	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6680
		6681	/* enable HiZ Raw Stall Optimization */
		6682	I915_WRITE(CACHE_MODE_0_GEN7,
		6683	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6684
4104	Serge	6685	/* WaDisable4x2SubspanOptimization:hsw */
3031	serge	6686	I915_WRITE(CACHE_MODE_1,
		6687	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6688
5060	serge	6689	/*
		6690	* BSpec recommends 8x4 when MSAA is used,
		6691	* however in practice 16x4 seems fastest.
		6692	*
		6693	* Note that PS/WM thread counts depend on the WIZ hashing
		6694	* disable bit, which we don't touch here, but it's good
		6695	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6696	*/
		6697	I915_WRITE(GEN7_GT_MODE,
5354	serge	6698	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6699
6084	serge	6700	/* WaSampleCChickenBitEnable:hsw */
		6701	I915_WRITE(HALF_SLICE_CHICKEN3,
		6702	_MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
		6703
4104	Serge	6704	/* WaSwitchSolVfFArbitrationPriority:hsw */
3746	Serge	6705	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6706
4104	Serge	6707	/* WaRsPkgCStateDisplayPMReq:hsw */
		6708	I915_WRITE(CHICKEN_PAR1_1,
		6709	I915_READ(CHICKEN_PAR1_1) \| FORCE_ARB_IDLE_PLANES);
3031	serge	6710
3243	Serge	6711	lpt_init_clock_gating(dev);
3031	serge	6712	}
		6713
		6714	static void ivybridge_init_clock_gating(struct drm_device *dev)
		6715	{
		6716	struct drm_i915_private *dev_priv = dev->dev_private;
		6717	uint32_t snpcr;
		6718
4560	Serge	6719	ilk_init_lp_watermarks(dev);
3031	serge	6720
3243	Serge	6721	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	6722
4104	Serge	6723	/* WaDisableEarlyCull:ivb */
3243	Serge	6724	I915_WRITE(_3D_CHICKEN3,
		6725	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6726
4104	Serge	6727	/* WaDisableBackToBackFlipFix:ivb */
3031	serge	6728	I915_WRITE(IVB_CHICKEN3,
		6729	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6730	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6731
4104	Serge	6732	/* WaDisablePSDDualDispatchEnable:ivb */
3243	Serge	6733	if (IS_IVB_GT1(dev))
		6734	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		6735	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		6736
5060	serge	6737	/* WaDisable_RenderCache_OperationalFlush:ivb */
		6738	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6739
4104	Serge	6740	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
3031	serge	6741	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		6742	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		6743
4104	Serge	6744	/* WaApplyL3ControlAndL3ChickenMode:ivb */
3031	serge	6745	I915_WRITE(GEN7_L3CNTLREG1,
		6746	GEN7_WA_FOR_GEN7_L3_CONTROL);
		6747	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
6084	serge	6748	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	6749	if (IS_IVB_GT1(dev))
		6750	I915_WRITE(GEN7_ROW_CHICKEN2,
		6751	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6752	else {
		6753	/* must write both registers */
		6754	I915_WRITE(GEN7_ROW_CHICKEN2,
		6755	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3243	Serge	6756	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		6757	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6758	}
3031	serge	6759
4104	Serge	6760	/* WaForceL3Serialization:ivb */
3243	Serge	6761	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6762	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6763
5060	serge	6764	/*
3031	serge	6765	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6766	* This implements the WaDisableRCZUnitClockGating:ivb workaround.
3031	serge	6767	*/
		6768	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6769	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6770
4104	Serge	6771	/* This is required by WaCatErrorRejectionIssue:ivb */
3031	serge	6772	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6773	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6774	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6775
4104	Serge	6776	g4x_disable_trickle_feed(dev);
3031	serge	6777
		6778	gen7_setup_fixed_func_scheduler(dev_priv);
		6779
5060	serge	6780	if (0) { /* causes HiZ corruption on ivb:gt1 */
		6781	/* enable HiZ Raw Stall Optimization */
		6782	I915_WRITE(CACHE_MODE_0_GEN7,
		6783	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6784	}
		6785
4104	Serge	6786	/* WaDisable4x2SubspanOptimization:ivb */
3031	serge	6787	I915_WRITE(CACHE_MODE_1,
		6788	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6789
5060	serge	6790	/*
		6791	* BSpec recommends 8x4 when MSAA is used,
		6792	* however in practice 16x4 seems fastest.
		6793	*
		6794	* Note that PS/WM thread counts depend on the WIZ hashing
		6795	* disable bit, which we don't touch here, but it's good
		6796	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6797	*/
		6798	I915_WRITE(GEN7_GT_MODE,
5354	serge	6799	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6800
3031	serge	6801	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		6802	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		6803	snpcr \|= GEN6_MBC_SNPCR_MED;
		6804	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	6805
3746	Serge	6806	if (!HAS_PCH_NOP(dev))
6084	serge	6807	cpt_init_clock_gating(dev);
3480	Serge	6808
		6809	gen6_check_mch_setup(dev);
3031	serge	6810	}
		6811
6084	serge	6812	static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
		6813	{
6935	serge	6814	u32 val;
6084	serge	6815
6935	serge	6816	/*
		6817	* On driver load, a pipe may be active and driving a DSI display.
		6818	* Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
		6819	* (and never recovering) in this case. intel_dsi_post_disable() will
		6820	* clear it when we turn off the display.
		6821	*/
		6822	val = I915_READ(DSPCLK_GATE_D);
		6823	val &= DPOUNIT_CLOCK_GATE_DISABLE;
		6824	val \|= VRHUNIT_CLOCK_GATE_DISABLE;
		6825	I915_WRITE(DSPCLK_GATE_D, val);
		6826
6084	serge	6827	/*
		6828	* Disable trickle feed and enable pnd deadline calculation
		6829	*/
		6830	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
		6831	I915_WRITE(CBR1_VLV, 0);
		6832	}
		6833
3031	serge	6834	static void valleyview_init_clock_gating(struct drm_device *dev)
		6835	{
		6836	struct drm_i915_private *dev_priv = dev->dev_private;
		6837
6084	serge	6838	vlv_init_display_clock_gating(dev_priv);
3031	serge	6839
4104	Serge	6840	/* WaDisableEarlyCull:vlv */
3243	Serge	6841	I915_WRITE(_3D_CHICKEN3,
		6842	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6843
4104	Serge	6844	/* WaDisableBackToBackFlipFix:vlv */
3031	serge	6845	I915_WRITE(IVB_CHICKEN3,
		6846	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6847	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6848
5060	serge	6849	/* WaPsdDispatchEnable:vlv */
4104	Serge	6850	/* WaDisablePSDDualDispatchEnable:vlv */
3243	Serge	6851	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	6852	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		6853	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	6854
5060	serge	6855	/* WaDisable_RenderCache_OperationalFlush:vlv */
		6856	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6857
4104	Serge	6858	/* WaForceL3Serialization:vlv */
3243	Serge	6859	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6860	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6861
4104	Serge	6862	/* WaDisableDopClockGating:vlv */
3243	Serge	6863	I915_WRITE(GEN7_ROW_CHICKEN2,
		6864	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		6865
4104	Serge	6866	/* This is required by WaCatErrorRejectionIssue:vlv */
3031	serge	6867	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6868	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6869	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6870
5060	serge	6871	gen7_setup_fixed_func_scheduler(dev_priv);
		6872
		6873	/*
3031	serge	6874	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6875	* This implements the WaDisableRCZUnitClockGating:vlv workaround.
3031	serge	6876	*/
		6877	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6878	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6879
5060	serge	6880	/* WaDisableL3Bank2xClockGate:vlv
		6881	* Disabling L3 clock gating- MMIO 940c[25] = 1
		6882	* Set bit 25, to disable L3_BANK_2x_CLK_GATING */
		6883	I915_WRITE(GEN7_UCGCTL4,
		6884	I915_READ(GEN7_UCGCTL4) \| GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
3031	serge	6885
5060	serge	6886	/*
		6887	* BSpec says this must be set, even though
		6888	* WaDisable4x2SubspanOptimization isn't listed for VLV.
		6889	*/
3031	serge	6890	I915_WRITE(CACHE_MODE_1,
		6891	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6892
		6893	/*
6084	serge	6894	* BSpec recommends 8x4 when MSAA is used,
		6895	* however in practice 16x4 seems fastest.
		6896	*
		6897	* Note that PS/WM thread counts depend on the WIZ hashing
		6898	* disable bit, which we don't touch here, but it's good
		6899	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6900	*/
		6901	I915_WRITE(GEN7_GT_MODE,
		6902	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
		6903
		6904	/*
5060	serge	6905	* WaIncreaseL3CreditsForVLVB0:vlv
		6906	* This is the hardware default actually.
		6907	*/
		6908	I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
		6909
		6910	/*
4104	Serge	6911	* WaDisableVLVClockGating_VBIIssue:vlv
3243	Serge	6912	* Disable clock gating on th GCFG unit to prevent a delay
		6913	* in the reporting of vblank events.
		6914	*/
5060	serge	6915	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
		6916	}
3746	Serge	6917
5060	serge	6918	static void cherryview_init_clock_gating(struct drm_device *dev)
		6919	{
		6920	struct drm_i915_private *dev_priv = dev->dev_private;
		6921
6084	serge	6922	vlv_init_display_clock_gating(dev_priv);
5060	serge	6923
		6924	/* WaVSRefCountFullforceMissDisable:chv */
		6925	/* WaDSRefCountFullforceMissDisable:chv */
		6926	I915_WRITE(GEN7_FF_THREAD_MODE,
		6927	I915_READ(GEN7_FF_THREAD_MODE) &
		6928	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
		6929
		6930	/* WaDisableSemaphoreAndSyncFlipWait:chv */
		6931	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6932	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6933
		6934	/* WaDisableCSUnitClockGating:chv */
		6935	I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) \|
		6936	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6937
		6938	/* WaDisableSDEUnitClockGating:chv */
		6939	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6940	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6084	serge	6941
		6942	/*
		6943	* GTT cache may not work with big pages, so if those
		6944	* are ever enabled GTT cache may need to be disabled.
		6945	*/
		6946	I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
3031	serge	6947	}
		6948
		6949	static void g4x_init_clock_gating(struct drm_device *dev)
		6950	{
		6951	struct drm_i915_private *dev_priv = dev->dev_private;
		6952	uint32_t dspclk_gate;
		6953
		6954	I915_WRITE(RENCLK_GATE_D1, 0);
		6955	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		6956	GS_UNIT_CLOCK_GATE_DISABLE \|
		6957	CL_UNIT_CLOCK_GATE_DISABLE);
		6958	I915_WRITE(RAMCLK_GATE_D, 0);
		6959	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		6960	OVRUNIT_CLOCK_GATE_DISABLE \|
		6961	OVCUNIT_CLOCK_GATE_DISABLE;
		6962	if (IS_GM45(dev))
		6963	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		6964	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	6965
		6966	/* WaDisableRenderCachePipelinedFlush */
		6967	I915_WRITE(CACHE_MODE_0,
		6968	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4104	Serge	6969
5060	serge	6970	/* WaDisable_RenderCache_OperationalFlush:g4x */
		6971	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6972
4104	Serge	6973	g4x_disable_trickle_feed(dev);
3031	serge	6974	}
		6975
		6976	static void crestline_init_clock_gating(struct drm_device *dev)
		6977	{
		6978	struct drm_i915_private *dev_priv = dev->dev_private;
		6979
		6980	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		6981	I915_WRITE(RENCLK_GATE_D2, 0);
		6982	I915_WRITE(DSPCLK_GATE_D, 0);
		6983	I915_WRITE(RAMCLK_GATE_D, 0);
		6984	I915_WRITE16(DEUC, 0);
4104	Serge	6985	I915_WRITE(MI_ARB_STATE,
		6986	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	6987
		6988	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		6989	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6990	}
		6991
		6992	static void broadwater_init_clock_gating(struct drm_device *dev)
		6993	{
		6994	struct drm_i915_private *dev_priv = dev->dev_private;
		6995
		6996	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		6997	I965_RCC_CLOCK_GATE_DISABLE \|
		6998	I965_RCPB_CLOCK_GATE_DISABLE \|
		6999	I965_ISC_CLOCK_GATE_DISABLE \|
		7000	I965_FBC_CLOCK_GATE_DISABLE);
		7001	I915_WRITE(RENCLK_GATE_D2, 0);
4104	Serge	7002	I915_WRITE(MI_ARB_STATE,
		7003	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	7004
		7005	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		7006	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	7007	}
		7008
		7009	static void gen3_init_clock_gating(struct drm_device *dev)
		7010	{
		7011	struct drm_i915_private *dev_priv = dev->dev_private;
		7012	u32 dstate = I915_READ(D_STATE);
		7013
		7014	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		7015	DSTATE_DOT_CLOCK_GATING;
		7016	I915_WRITE(D_STATE, dstate);
		7017
		7018	if (IS_PINEVIEW(dev))
		7019	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		7020
		7021	/* IIR "flip pending" means done if this bit is set */
		7022	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
5060	serge	7023
		7024	/* interrupts should cause a wake up from C3 */
		7025	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
		7026
		7027	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		7028	I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
5354	serge	7029
		7030	I915_WRITE(MI_ARB_STATE,
		7031	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	7032	}
		7033
		7034	static void i85x_init_clock_gating(struct drm_device *dev)
		7035	{
		7036	struct drm_i915_private *dev_priv = dev->dev_private;
		7037
		7038	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
5060	serge	7039
		7040	/* interrupts should cause a wake up from C3 */
		7041	I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) \|
		7042	_MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
5354	serge	7043
		7044	I915_WRITE(MEM_MODE,
		7045	_MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	7046	}
		7047
		7048	static void i830_init_clock_gating(struct drm_device *dev)
		7049	{
		7050	struct drm_i915_private *dev_priv = dev->dev_private;
		7051
		7052	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
5354	serge	7053
		7054	I915_WRITE(MEM_MODE,
		7055	_MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) \|
		7056	_MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
3031	serge	7057	}
		7058
		7059	void intel_init_clock_gating(struct drm_device *dev)
		7060	{
		7061	struct drm_i915_private *dev_priv = dev->dev_private;
		7062
6084	serge	7063	if (dev_priv->display.init_clock_gating)
		7064	dev_priv->display.init_clock_gating(dev);
3031	serge	7065	}
		7066
4104	Serge	7067	void intel_suspend_hw(struct drm_device *dev)
		7068	{
		7069	if (HAS_PCH_LPT(dev))
		7070	lpt_suspend_hw(dev);
		7071	}
		7072
5354	serge	7073	/* Set up chip specific power management-related functions */
		7074	void intel_init_pm(struct drm_device *dev)
		7075	{
		7076	struct drm_i915_private *dev_priv = dev->dev_private;
		7077
6084	serge	7078	intel_fbc_init(dev_priv);
5354	serge	7079
3031	serge	7080	/* For cxsr */
		7081	if (IS_PINEVIEW(dev))
		7082	i915_pineview_get_mem_freq(dev);
		7083	else if (IS_GEN5(dev))
		7084	i915_ironlake_get_mem_freq(dev);
		7085
		7086	/* For FIFO watermark updates */
5354	serge	7087	if (INTEL_INFO(dev)->gen >= 9) {
		7088	skl_setup_wm_latency(dev);
		7089
6084	serge	7090	if (IS_BROXTON(dev))
		7091	dev_priv->display.init_clock_gating =
		7092	bxt_init_clock_gating;
5354	serge	7093	dev_priv->display.update_wm = skl_update_wm;
		7094	dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
		7095	} else if (HAS_PCH_SPLIT(dev)) {
5060	serge	7096	ilk_setup_wm_latency(dev);
4104	Serge	7097
4560	Serge	7098	if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
		7099	dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) \|\|
		7100	(!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
		7101	dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
		7102	dev_priv->display.update_wm = ilk_update_wm;
		7103	dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
6084	serge	7104	} else {
		7105	DRM_DEBUG_KMS("Failed to read display plane latency. "
		7106	"Disable CxSR\n");
		7107	}
4560	Serge	7108
		7109	if (IS_GEN5(dev))
		7110	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		7111	else if (IS_GEN6(dev))
3031	serge	7112	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
4560	Serge	7113	else if (IS_IVYBRIDGE(dev))
3031	serge	7114	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
4560	Serge	7115	else if (IS_HASWELL(dev))
3031	serge	7116	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
4560	Serge	7117	else if (INTEL_INFO(dev)->gen == 8)
5354	serge	7118	dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
5060	serge	7119	} else if (IS_CHERRYVIEW(dev)) {
6084	serge	7120	vlv_setup_wm_latency(dev);
		7121
		7122	dev_priv->display.update_wm = vlv_update_wm;
5060	serge	7123	dev_priv->display.init_clock_gating =
		7124	cherryview_init_clock_gating;
3031	serge	7125	} else if (IS_VALLEYVIEW(dev)) {
6084	serge	7126	vlv_setup_wm_latency(dev);
		7127
		7128	dev_priv->display.update_wm = vlv_update_wm;
3031	serge	7129	dev_priv->display.init_clock_gating =
		7130	valleyview_init_clock_gating;
		7131	} else if (IS_PINEVIEW(dev)) {
		7132	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		7133	dev_priv->is_ddr3,
		7134	dev_priv->fsb_freq,
		7135	dev_priv->mem_freq)) {
		7136	DRM_INFO("failed to find known CxSR latency "
		7137	"(found ddr%s fsb freq %d, mem freq %d), "
		7138	"disabling CxSR\n",
		7139	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		7140	dev_priv->fsb_freq, dev_priv->mem_freq);
		7141	/* Disable CxSR and never update its watermark again */
5060	serge	7142	intel_set_memory_cxsr(dev_priv, false);
3031	serge	7143	dev_priv->display.update_wm = NULL;
		7144	} else
		7145	dev_priv->display.update_wm = pineview_update_wm;
		7146	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		7147	} else if (IS_G4X(dev)) {
		7148	dev_priv->display.update_wm = g4x_update_wm;
		7149	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		7150	} else if (IS_GEN4(dev)) {
		7151	dev_priv->display.update_wm = i965_update_wm;
		7152	if (IS_CRESTLINE(dev))
		7153	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		7154	else if (IS_BROADWATER(dev))
		7155	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		7156	} else if (IS_GEN3(dev)) {
		7157	dev_priv->display.update_wm = i9xx_update_wm;
		7158	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		7159	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
4560	Serge	7160	} else if (IS_GEN2(dev)) {
		7161	if (INTEL_INFO(dev)->num_pipes == 1) {
		7162	dev_priv->display.update_wm = i845_update_wm;
		7163	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		7164	} else {
		7165	dev_priv->display.update_wm = i9xx_update_wm;
6084	serge	7166	dev_priv->display.get_fifo_size = i830_get_fifo_size;
4560	Serge	7167	}
		7168
		7169	if (IS_I85X(dev) \|\| IS_I865G(dev))
6084	serge	7170	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
4560	Serge	7171	else
		7172	dev_priv->display.init_clock_gating = i830_init_clock_gating;
3031	serge	7173	} else {
4560	Serge	7174	DRM_ERROR("unexpected fall-through in intel_init_pm\n");
3031	serge	7175	}
		7176	}
		7177
5354	serge	7178	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u32 mbox, u32 val)
3243	Serge	7179	{
		7180	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	7181
3243	Serge	7182	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7183	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		7184	return -EAGAIN;
		7185	}
3031	serge	7186
3243	Serge	7187	I915_WRITE(GEN6_PCODE_DATA, *val);
5354	serge	7188	I915_WRITE(GEN6_PCODE_DATA1, 0);
3243	Serge	7189	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7190
		7191	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7192	500)) {
		7193	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		7194	return -ETIMEDOUT;
6084	serge	7195	}
3243	Serge	7196
		7197	*val = I915_READ(GEN6_PCODE_DATA);
		7198	I915_WRITE(GEN6_PCODE_DATA, 0);
		7199
		7200	return 0;
		7201	}
		7202
5354	serge	7203	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
3243	Serge	7204	{
		7205	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		7206
		7207	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7208	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		7209	return -EAGAIN;
6084	serge	7210	}
3243	Serge	7211
		7212	I915_WRITE(GEN6_PCODE_DATA, val);
		7213	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7214
		7215	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7216	500)) {
		7217	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		7218	return -ETIMEDOUT;
3031	serge	7219	}
3243	Serge	7220
		7221	I915_WRITE(GEN6_PCODE_DATA, 0);
		7222
		7223	return 0;
3031	serge	7224	}
3746	Serge	7225
5354	serge	7226	static int vlv_gpu_freq_div(unsigned int czclk_freq)
3746	Serge	7227	{
5354	serge	7228	switch (czclk_freq) {
		7229	case 200:
		7230	return 10;
		7231	case 267:
		7232	return 12;
		7233	case 320:
		7234	case 333:
		7235	return 16;
		7236	case 400:
		7237	return 20;
4104	Serge	7238	default:
		7239	return -1;
		7240	}
5354	serge	7241	}
3746	Serge	7242
5354	serge	7243	static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7244	{
6084	serge	7245	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5354	serge	7246
		7247	div = vlv_gpu_freq_div(czclk_freq);
		7248	if (div < 0)
		7249	return div;
		7250
		7251	return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
4104	Serge	7252	}
3746	Serge	7253
5060	serge	7254	static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
4104	Serge	7255	{
6084	serge	7256	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
3746	Serge	7257
5354	serge	7258	mul = vlv_gpu_freq_div(czclk_freq);
		7259	if (mul < 0)
		7260	return mul;
3746	Serge	7261
5354	serge	7262	return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
3746	Serge	7263	}
		7264
5060	serge	7265	static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7266	{
6084	serge	7267	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5060	serge	7268
5354	serge	7269	div = vlv_gpu_freq_div(czclk_freq) / 2;
		7270	if (div < 0)
		7271	return div;
5060	serge	7272
5354	serge	7273	return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
5060	serge	7274	}
		7275
		7276	static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7277	{
6084	serge	7278	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->czclk_freq, 1000);
5060	serge	7279
5354	serge	7280	mul = vlv_gpu_freq_div(czclk_freq) / 2;
		7281	if (mul < 0)
		7282	return mul;
5060	serge	7283
5354	serge	7284	/* CHV needs even values */
		7285	return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
5060	serge	7286	}
		7287
6084	serge	7288	int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
5060	serge	7289	{
6084	serge	7290	if (IS_GEN9(dev_priv->dev))
		7291	return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
		7292	GEN9_FREQ_SCALER);
		7293	else if (IS_CHERRYVIEW(dev_priv->dev))
		7294	return chv_gpu_freq(dev_priv, val);
		7295	else if (IS_VALLEYVIEW(dev_priv->dev))
		7296	return byt_gpu_freq(dev_priv, val);
		7297	else
		7298	return val * GT_FREQUENCY_MULTIPLIER;
		7299	}
5060	serge	7300
6084	serge	7301	int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7302	{
		7303	if (IS_GEN9(dev_priv->dev))
		7304	return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
		7305	GT_FREQUENCY_MULTIPLIER);
		7306	else if (IS_CHERRYVIEW(dev_priv->dev))
		7307	return chv_freq_opcode(dev_priv, val);
5060	serge	7308	else if (IS_VALLEYVIEW(dev_priv->dev))
6084	serge	7309	return byt_freq_opcode(dev_priv, val);
		7310	else
		7311	return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
		7312	}
5060	serge	7313
6084	serge	7314	struct request_boost {
		7315	struct work_struct work;
		7316	struct drm_i915_gem_request *req;
		7317	};
		7318
		7319	static void __intel_rps_boost_work(struct work_struct *work)
		7320	{
		7321	struct request_boost *boost = container_of(work, struct request_boost, work);
		7322	struct drm_i915_gem_request *req = boost->req;
		7323
		7324	if (!i915_gem_request_completed(req, true))
		7325	gen6_rps_boost(to_i915(req->ring->dev), NULL,
		7326	req->emitted_jiffies);
		7327
		7328	i915_gem_request_unreference__unlocked(req);
		7329	kfree(boost);
5060	serge	7330	}
		7331
6084	serge	7332	void intel_queue_rps_boost_for_request(struct drm_device *dev,
		7333	struct drm_i915_gem_request *req)
5060	serge	7334	{
6084	serge	7335	struct request_boost *boost;
5060	serge	7336
6084	serge	7337	if (req == NULL \|\| INTEL_INFO(dev)->gen < 6)
		7338	return;
5060	serge	7339
6084	serge	7340	if (i915_gem_request_completed(req, true))
		7341	return;
		7342
		7343	boost = kmalloc(sizeof(*boost), GFP_ATOMIC);
		7344	if (boost == NULL)
		7345	return;
		7346
		7347	i915_gem_request_reference(req);
		7348	boost->req = req;
		7349
		7350	INIT_WORK(&boost->work, __intel_rps_boost_work);
		7351	queue_work(to_i915(dev)->wq, &boost->work);
5060	serge	7352	}
		7353
4560	Serge	7354	void intel_pm_setup(struct drm_device *dev)
3746	Serge	7355	{
4104	Serge	7356	struct drm_i915_private *dev_priv = dev->dev_private;
		7357
4560	Serge	7358	mutex_init(&dev_priv->rps.hw_lock);
6084	serge	7359	spin_lock_init(&dev_priv->rps.client_lock);
4560	Serge	7360
4104	Serge	7361	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		7362	intel_gen6_powersave_work);
6084	serge	7363	INIT_LIST_HEAD(&dev_priv->rps.clients);
		7364	INIT_LIST_HEAD(&dev_priv->rps.semaphores.link);
		7365	INIT_LIST_HEAD(&dev_priv->rps.mmioflips.link);
5060	serge	7366
		7367	dev_priv->pm.suspended = false;
3746	Serge	7368	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 6935