WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	#include
		32	//#include "../../../platform/x86/intel_ips.h"
		33	#include
		34
4560	Serge	35	#include
		36
3031	serge	37	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		38
		39	void getrawmonotonic(struct timespec *ts);
		40
5060	serge	41	union ktime {
		42	s64 tv64;
		43	};
3031	serge	44
5060	serge	45	typedef union ktime ktime_t; /* Kill this */
		46
		47	#define ktime_to_ns(kt) ((kt).tv64)
		48
		49	static inline u64 ktime_get_raw_ns(void)
		50	{
		51	return 0; //ktime_to_ns(ktime_get_raw());
		52	}
4560	Serge	53	/**
		54	* RC6 is a special power stage which allows the GPU to enter an very
		55	* low-voltage mode when idle, using down to 0V while at this stage. This
		56	* stage is entered automatically when the GPU is idle when RC6 support is
		57	* enabled, and as soon as new workload arises GPU wakes up automatically as well.
		58	*
		59	* There are different RC6 modes available in Intel GPU, which differentiate
		60	* among each other with the latency required to enter and leave RC6 and
		61	* voltage consumed by the GPU in different states.
		62	*
		63	* The combination of the following flags define which states GPU is allowed
		64	* to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
		65	* RC6pp is deepest RC6. Their support by hardware varies according to the
		66	* GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
		67	* which brings the most power savings; deeper states save more power, but
		68	* require higher latency to switch to and wake up.
		69	*/
		70	#define INTEL_RC6_ENABLE (1<<0)
		71	#define INTEL_RC6p_ENABLE (1<<1)
		72	#define INTEL_RC6pp_ENABLE (1<<2)
		73
3031	serge	74	/* FBC, or Frame Buffer Compression, is a technique employed to compress the
		75	* framebuffer contents in-memory, aiming at reducing the required bandwidth
		76	* during in-memory transfers and, therefore, reduce the power packet.
		77	*
		78	* The benefits of FBC are mostly visible with solid backgrounds and
		79	* variation-less patterns.
		80	*
		81	* FBC-related functionality can be enabled by the means of the
		82	* i915.i915_enable_fbc parameter
		83	*/
		84
5354	serge	85	static void gen9_init_clock_gating(struct drm_device *dev)
		86	{
		87	struct drm_i915_private *dev_priv = dev->dev_private;
		88
		89	/*
		90	* WaDisableSDEUnitClockGating:skl
		91	* This seems to be a pre-production w/a.
		92	*/
		93	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		94	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		95
		96	/*
		97	* WaDisableDgMirrorFixInHalfSliceChicken5:skl
		98	* This is a pre-production w/a.
		99	*/
		100	I915_WRITE(GEN9_HALF_SLICE_CHICKEN5,
		101	I915_READ(GEN9_HALF_SLICE_CHICKEN5) &
		102	~GEN9_DG_MIRROR_FIX_ENABLE);
		103
		104	/* Wa4x4STCOptimizationDisable:skl */
		105	I915_WRITE(CACHE_MODE_1,
		106	_MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE));
		107	}
		108
3031	serge	109	static void i8xx_disable_fbc(struct drm_device *dev)
		110	{
		111	struct drm_i915_private *dev_priv = dev->dev_private;
		112	u32 fbc_ctl;
		113
5354	serge	114	dev_priv->fbc.enabled = false;
		115
3031	serge	116	/* Disable compression */
		117	fbc_ctl = I915_READ(FBC_CONTROL);
		118	if ((fbc_ctl & FBC_CTL_EN) == 0)
		119	return;
		120
		121	fbc_ctl &= ~FBC_CTL_EN;
		122	I915_WRITE(FBC_CONTROL, fbc_ctl);
		123
		124	/* Wait for compressing bit to clear */
		125	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
		126	DRM_DEBUG_KMS("FBC idle timed out\n");
		127	return;
		128	}
		129
		130	DRM_DEBUG_KMS("disabled FBC\n");
		131	}
		132
4560	Serge	133	static void i8xx_enable_fbc(struct drm_crtc *crtc)
3031	serge	134	{
		135	struct drm_device *dev = crtc->dev;
		136	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	137	struct drm_framebuffer *fb = crtc->primary->fb;
		138	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
3031	serge	139	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		140	int cfb_pitch;
5060	serge	141	int i;
4560	Serge	142	u32 fbc_ctl;
3031	serge	143
5354	serge	144	dev_priv->fbc.enabled = true;
		145
4104	Serge	146	cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE;
3031	serge	147	if (fb->pitches[0] < cfb_pitch)
		148	cfb_pitch = fb->pitches[0];
		149
4560	Serge	150	/* FBC_CTL wants 32B or 64B units */
		151	if (IS_GEN2(dev))
		152	cfb_pitch = (cfb_pitch / 32) - 1;
		153	else
3031	serge	154	cfb_pitch = (cfb_pitch / 64) - 1;
		155
		156	/* Clear old tags */
		157	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
		158	I915_WRITE(FBC_TAG + (i * 4), 0);
		159
4560	Serge	160	if (IS_GEN4(dev)) {
		161	u32 fbc_ctl2;
		162
3031	serge	163	/* Set it up... */
		164	fbc_ctl2 = FBC_CTL_FENCE_DBL \| FBC_CTL_IDLE_IMM \| FBC_CTL_CPU_FENCE;
5060	serge	165	fbc_ctl2 \|= FBC_CTL_PLANE(intel_crtc->plane);
3031	serge	166	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
		167	I915_WRITE(FBC_FENCE_OFF, crtc->y);
4560	Serge	168	}
3031	serge	169
		170	/* enable it... */
4560	Serge	171	fbc_ctl = I915_READ(FBC_CONTROL);
		172	fbc_ctl &= 0x3fff << FBC_CTL_INTERVAL_SHIFT;
		173	fbc_ctl \|= FBC_CTL_EN \| FBC_CTL_PERIODIC;
3031	serge	174	if (IS_I945GM(dev))
		175	fbc_ctl \|= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
		176	fbc_ctl \|= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
		177	fbc_ctl \|= obj->fence_reg;
		178	I915_WRITE(FBC_CONTROL, fbc_ctl);
		179
5060	serge	180	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n",
4104	Serge	181	cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
3031	serge	182	}
		183
		184	static bool i8xx_fbc_enabled(struct drm_device *dev)
		185	{
		186	struct drm_i915_private *dev_priv = dev->dev_private;
		187
		188	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
		189	}
		190
4560	Serge	191	static void g4x_enable_fbc(struct drm_crtc *crtc)
3031	serge	192	{
		193	struct drm_device *dev = crtc->dev;
		194	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	195	struct drm_framebuffer *fb = crtc->primary->fb;
		196	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
3031	serge	197	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		198	u32 dpfc_ctl;
		199
5354	serge	200	dev_priv->fbc.enabled = true;
		201
5060	serge	202	dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) \| DPFC_SR_EN;
		203	if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
		204	dpfc_ctl \|= DPFC_CTL_LIMIT_2X;
		205	else
		206	dpfc_ctl \|= DPFC_CTL_LIMIT_1X;
3031	serge	207	dpfc_ctl \|= DPFC_CTL_FENCE_EN \| obj->fence_reg;
		208
		209	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
		210
		211	/* enable it... */
5060	serge	212	I915_WRITE(DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
3031	serge	213
4104	Serge	214	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
3031	serge	215	}
		216
		217	static void g4x_disable_fbc(struct drm_device *dev)
		218	{
		219	struct drm_i915_private *dev_priv = dev->dev_private;
		220	u32 dpfc_ctl;
		221
5354	serge	222	dev_priv->fbc.enabled = false;
		223
3031	serge	224	/* Disable compression */
		225	dpfc_ctl = I915_READ(DPFC_CONTROL);
		226	if (dpfc_ctl & DPFC_CTL_EN) {
		227	dpfc_ctl &= ~DPFC_CTL_EN;
		228	I915_WRITE(DPFC_CONTROL, dpfc_ctl);
		229
		230	DRM_DEBUG_KMS("disabled FBC\n");
		231	}
		232	}
		233
		234	static bool g4x_fbc_enabled(struct drm_device *dev)
		235	{
		236	struct drm_i915_private *dev_priv = dev->dev_private;
		237
		238	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
		239	}
		240
		241	static void sandybridge_blit_fbc_update(struct drm_device *dev)
		242	{
		243	struct drm_i915_private *dev_priv = dev->dev_private;
		244	u32 blt_ecoskpd;
		245
		246	/* Make sure blitter notifies FBC of writes */
4560	Serge	247
		248	/* Blitter is part of Media powerwell on VLV. No impact of
		249	* his param in other platforms for now */
		250	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_MEDIA);
		251
3031	serge	252	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
		253	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY <<
		254	GEN6_BLITTER_LOCK_SHIFT;
		255	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		256	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY;
		257	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		258	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
		259	GEN6_BLITTER_LOCK_SHIFT);
		260	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		261	POSTING_READ(GEN6_BLITTER_ECOSKPD);
4560	Serge	262
		263	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_MEDIA);
3031	serge	264	}
		265
4560	Serge	266	static void ironlake_enable_fbc(struct drm_crtc *crtc)
3031	serge	267	{
		268	struct drm_device *dev = crtc->dev;
		269	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	270	struct drm_framebuffer *fb = crtc->primary->fb;
		271	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
3031	serge	272	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		273	u32 dpfc_ctl;
		274
5354	serge	275	dev_priv->fbc.enabled = true;
		276
5060	serge	277	dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane);
		278	if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
		279	dev_priv->fbc.threshold++;
		280
		281	switch (dev_priv->fbc.threshold) {
		282	case 4:
		283	case 3:
		284	dpfc_ctl \|= DPFC_CTL_LIMIT_4X;
		285	break;
		286	case 2:
		287	dpfc_ctl \|= DPFC_CTL_LIMIT_2X;
		288	break;
		289	case 1:
		290	dpfc_ctl \|= DPFC_CTL_LIMIT_1X;
		291	break;
		292	}
4560	Serge	293	dpfc_ctl \|= DPFC_CTL_FENCE_EN;
		294	if (IS_GEN5(dev))
		295	dpfc_ctl \|= obj->fence_reg;
3031	serge	296
		297	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
4104	Serge	298	I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) \| ILK_FBC_RT_VALID);
3031	serge	299	/* enable it... */
		300	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		301
		302	if (IS_GEN6(dev)) {
		303	I915_WRITE(SNB_DPFC_CTL_SA,
		304	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		305	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		306	sandybridge_blit_fbc_update(dev);
		307	}
		308
4104	Serge	309	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
3031	serge	310	}
		311
		312	static void ironlake_disable_fbc(struct drm_device *dev)
		313	{
		314	struct drm_i915_private *dev_priv = dev->dev_private;
		315	u32 dpfc_ctl;
		316
5354	serge	317	dev_priv->fbc.enabled = false;
		318
3031	serge	319	/* Disable compression */
		320	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		321	if (dpfc_ctl & DPFC_CTL_EN) {
		322	dpfc_ctl &= ~DPFC_CTL_EN;
		323	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
		324
		325	DRM_DEBUG_KMS("disabled FBC\n");
		326	}
		327	}
		328
		329	static bool ironlake_fbc_enabled(struct drm_device *dev)
		330	{
		331	struct drm_i915_private *dev_priv = dev->dev_private;
		332
		333	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
		334	}
		335
4560	Serge	336	static void gen7_enable_fbc(struct drm_crtc *crtc)
4104	Serge	337	{
		338	struct drm_device *dev = crtc->dev;
		339	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	340	struct drm_framebuffer *fb = crtc->primary->fb;
		341	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
4104	Serge	342	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5060	serge	343	u32 dpfc_ctl;
4104	Serge	344
5354	serge	345	dev_priv->fbc.enabled = true;
		346
5060	serge	347	dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane);
		348	if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
		349	dev_priv->fbc.threshold++;
4104	Serge	350
5060	serge	351	switch (dev_priv->fbc.threshold) {
		352	case 4:
		353	case 3:
		354	dpfc_ctl \|= DPFC_CTL_LIMIT_4X;
		355	break;
		356	case 2:
		357	dpfc_ctl \|= DPFC_CTL_LIMIT_2X;
		358	break;
		359	case 1:
		360	dpfc_ctl \|= DPFC_CTL_LIMIT_1X;
		361	break;
		362	}
4104	Serge	363
5060	serge	364	dpfc_ctl \|= IVB_DPFC_CTL_FENCE_EN;
		365
5354	serge	366	if (dev_priv->fbc.false_color)
		367	dpfc_ctl \|= FBC_CTL_FALSE_COLOR;
		368
5060	serge	369	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		370
4104	Serge	371	if (IS_IVYBRIDGE(dev)) {
		372	/* WaFbcAsynchFlipDisableFbcQueue:ivb */
5060	serge	373	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		374	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		375	ILK_FBCQ_DIS);
4104	Serge	376	} else {
5060	serge	377	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
		378	I915_WRITE(CHICKEN_PIPESL_1(intel_crtc->pipe),
		379	I915_READ(CHICKEN_PIPESL_1(intel_crtc->pipe)) \|
		380	HSW_FBCQ_DIS);
4104	Serge	381	}
		382
		383	I915_WRITE(SNB_DPFC_CTL_SA,
		384	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		385	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		386
		387	sandybridge_blit_fbc_update(dev);
		388
4560	Serge	389	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
4104	Serge	390	}
		391
3031	serge	392	bool intel_fbc_enabled(struct drm_device *dev)
		393	{
		394	struct drm_i915_private *dev_priv = dev->dev_private;
		395
5354	serge	396	return dev_priv->fbc.enabled;
		397	}
3031	serge	398
5354	serge	399	void bdw_fbc_sw_flush(struct drm_device *dev, u32 value)
		400	{
		401	struct drm_i915_private *dev_priv = dev->dev_private;
		402
		403	if (!IS_GEN8(dev))
		404	return;
		405
		406	if (!intel_fbc_enabled(dev))
		407	return;
		408
		409	I915_WRITE(MSG_FBC_REND_STATE, value);
3031	serge	410	}
		411
		412	static void intel_fbc_work_fn(struct work_struct *__work)
		413	{
		414	struct intel_fbc_work *work =
		415	container_of(to_delayed_work(__work),
		416	struct intel_fbc_work, work);
		417	struct drm_device *dev = work->crtc->dev;
		418	struct drm_i915_private *dev_priv = dev->dev_private;
		419
		420	mutex_lock(&dev->struct_mutex);
4104	Serge	421	if (work == dev_priv->fbc.fbc_work) {
3031	serge	422	/* Double check that we haven't switched fb without cancelling
		423	* the prior work.
		424	*/
5060	serge	425	if (work->crtc->primary->fb == work->fb) {
4560	Serge	426	dev_priv->display.enable_fbc(work->crtc);
3031	serge	427
4104	Serge	428	dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane;
5060	serge	429	dev_priv->fbc.fb_id = work->crtc->primary->fb->base.id;
4104	Serge	430	dev_priv->fbc.y = work->crtc->y;
3031	serge	431	}
		432
4104	Serge	433	dev_priv->fbc.fbc_work = NULL;
3031	serge	434	}
		435	mutex_unlock(&dev->struct_mutex);
		436
		437	kfree(work);
		438	}
		439
		440	static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
		441	{
4104	Serge	442	if (dev_priv->fbc.fbc_work == NULL)
3031	serge	443	return;
		444
		445	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
		446
		447	/* Synchronisation is provided by struct_mutex and checking of
4104	Serge	448	* dev_priv->fbc.fbc_work, so we can perform the cancellation
3031	serge	449	* entirely asynchronously.
		450	*/
4293	Serge	451	if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work))
3031	serge	452	/* tasklet was killed before being run, clean up */
4293	Serge	453	kfree(dev_priv->fbc.fbc_work);
3031	serge	454
		455	/* Mark the work as no longer wanted so that if it does
		456	* wake-up (because the work was already running and waiting
		457	* for our mutex), it will discover that is no longer
		458	* necessary to run.
		459	*/
4104	Serge	460	dev_priv->fbc.fbc_work = NULL;
3031	serge	461	}
		462
4560	Serge	463	static void intel_enable_fbc(struct drm_crtc *crtc)
3031	serge	464	{
		465	struct intel_fbc_work *work;
		466	struct drm_device *dev = crtc->dev;
		467	struct drm_i915_private *dev_priv = dev->dev_private;
		468
3482	Serge	469	if (!dev_priv->display.enable_fbc)
3031	serge	470	return;
3482	Serge	471
3031	serge	472	intel_cancel_fbc_work(dev_priv);
		473
4560	Serge	474	work = kzalloc(sizeof(*work), GFP_KERNEL);
3031	serge	475	if (work == NULL) {
4104	Serge	476	DRM_ERROR("Failed to allocate FBC work structure\n");
4560	Serge	477	dev_priv->display.enable_fbc(crtc);
3031	serge	478	return;
		479	}
		480
		481	work->crtc = crtc;
5060	serge	482	work->fb = crtc->primary->fb;
3031	serge	483	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
		484
4104	Serge	485	dev_priv->fbc.fbc_work = work;
3031	serge	486
		487	/* Delay the actual enabling to let pageflipping cease and the
		488	* display to settle before starting the compression. Note that
		489	* this delay also serves a second purpose: it allows for a
		490	* vblank to pass after disabling the FBC before we attempt
		491	* to modify the control registers.
		492	*
		493	* A more complicated solution would involve tracking vblanks
		494	* following the termination of the page-flipping sequence
		495	* and indeed performing the enable as a co-routine and not
		496	* waiting synchronously upon the vblank.
4104	Serge	497	*
		498	* WaFbcWaitForVBlankBeforeEnable:ilk,snb
3031	serge	499	*/
		500	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
		501	}
		502
		503	void intel_disable_fbc(struct drm_device *dev)
		504	{
		505	struct drm_i915_private *dev_priv = dev->dev_private;
		506
3482	Serge	507	intel_cancel_fbc_work(dev_priv);
3031	serge	508
3482	Serge	509	if (!dev_priv->display.disable_fbc)
		510	return;
3031	serge	511
3482	Serge	512	dev_priv->display.disable_fbc(dev);
4104	Serge	513	dev_priv->fbc.plane = -1;
3031	serge	514	}
		515
4104	Serge	516	static bool set_no_fbc_reason(struct drm_i915_private *dev_priv,
		517	enum no_fbc_reason reason)
		518	{
		519	if (dev_priv->fbc.no_fbc_reason == reason)
		520	return false;
		521
		522	dev_priv->fbc.no_fbc_reason = reason;
		523	return true;
		524	}
		525
3031	serge	526	/**
		527	* intel_update_fbc - enable/disable FBC as needed
		528	* @dev: the drm_device
		529	*
		530	* Set up the framebuffer compression hardware at mode set time. We
		531	* enable it if possible:
		532	* - plane A only (on pre-965)
		533	* - no pixel mulitply/line duplication
		534	* - no alpha buffer discard
		535	* - no dual wide
4104	Serge	536	* - framebuffer <= max_hdisplay in width, max_vdisplay in height
3031	serge	537	*
		538	* We can't assume that any compression will take place (worst case),
		539	* so the compressed buffer has to be the same size as the uncompressed
		540	* one. It also must reside (along with the line length buffer) in
		541	* stolen memory.
		542	*
		543	* We need to enable/disable FBC on a global basis.
		544	*/
		545	void intel_update_fbc(struct drm_device *dev)
		546	{
		547	struct drm_i915_private *dev_priv = dev->dev_private;
		548	struct drm_crtc crtc = NULL, tmp_crtc;
		549	struct intel_crtc *intel_crtc;
		550	struct drm_framebuffer *fb;
		551	struct drm_i915_gem_object *obj;
4560	Serge	552	const struct drm_display_mode *adjusted_mode;
		553	unsigned int max_width, max_height;
3031	serge	554
4560	Serge	555	if (!HAS_FBC(dev)) {
4104	Serge	556	set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED);
3031	serge	557	return;
4104	Serge	558	}
3031	serge	559
5060	serge	560	if (!i915.powersave) {
4104	Serge	561	if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
		562	DRM_DEBUG_KMS("fbc disabled per module param\n");
3031	serge	563	return;
4104	Serge	564	}
3031	serge	565
		566	/*
		567	* If FBC is already on, we just have to verify that we can
		568	* keep it that way...
		569	* Need to disable if:
		570	* - more than one pipe is active
		571	* - changing FBC params (stride, fence, mode)
		572	* - new fb is too large to fit in compressed buffer
		573	* - going to an unsupported config (interlace, pixel multiply, etc.)
		574	*/
5060	serge	575	for_each_crtc(dev, tmp_crtc) {
3243	Serge	576	if (intel_crtc_active(tmp_crtc) &&
4560	Serge	577	to_intel_crtc(tmp_crtc)->primary_enabled) {
3031	serge	578	if (crtc) {
4104	Serge	579	if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES))
3031	serge	580	DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
		581	goto out_disable;
		582	}
		583	crtc = tmp_crtc;
		584	}
		585	}
		586
5060	serge	587	if (!crtc \|\| crtc->primary->fb == NULL) {
4104	Serge	588	if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT))
3031	serge	589	DRM_DEBUG_KMS("no output, disabling\n");
		590	goto out_disable;
		591	}
		592
		593	intel_crtc = to_intel_crtc(crtc);
5060	serge	594	fb = crtc->primary->fb;
		595	obj = intel_fb_obj(fb);
4560	Serge	596	adjusted_mode = &intel_crtc->config.adjusted_mode;
3031	serge	597
5060	serge	598	if (i915.enable_fbc < 0) {
4104	Serge	599	if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT))
		600	DRM_DEBUG_KMS("disabled per chip default\n");
		601	goto out_disable;
3031	serge	602	}
5060	serge	603	if (!i915.enable_fbc) {
4104	Serge	604	if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
3031	serge	605	DRM_DEBUG_KMS("fbc disabled per module param\n");
		606	goto out_disable;
		607	}
4560	Serge	608	if ((adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) \|\|
		609	(adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)) {
4104	Serge	610	if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
3031	serge	611	DRM_DEBUG_KMS("mode incompatible with compression, "
		612	"disabling\n");
		613	goto out_disable;
		614	}
4104	Serge	615
5060	serge	616	if (INTEL_INFO(dev)->gen >= 8 \|\| IS_HASWELL(dev)) {
4560	Serge	617	max_width = 4096;
5060	serge	618	max_height = 4096;
		619	} else if (IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
		620	max_width = 4096;
4560	Serge	621	max_height = 2048;
4104	Serge	622	} else {
4560	Serge	623	max_width = 2048;
		624	max_height = 1536;
4104	Serge	625	}
4560	Serge	626	if (intel_crtc->config.pipe_src_w > max_width \|\|
		627	intel_crtc->config.pipe_src_h > max_height) {
4104	Serge	628	if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE))
3031	serge	629	DRM_DEBUG_KMS("mode too large for compression, disabling\n");
		630	goto out_disable;
		631	}
5060	serge	632	if ((INTEL_INFO(dev)->gen < 4 \|\| HAS_DDI(dev)) &&
4560	Serge	633	intel_crtc->plane != PLANE_A) {
4104	Serge	634	if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE))
4560	Serge	635	DRM_DEBUG_KMS("plane not A, disabling compression\n");
3031	serge	636	goto out_disable;
		637	}
		638
		639	/* The use of a CPU fence is mandatory in order to detect writes
		640	* by the CPU to the scanout and trigger updates to the FBC.
		641	*/
		642	if (obj->tiling_mode != I915_TILING_X \|\|
		643	obj->fence_reg == I915_FENCE_REG_NONE) {
4104	Serge	644	if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED))
3031	serge	645	DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
		646	goto out_disable;
		647	}
5354	serge	648	if (INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) &&
		649	to_intel_plane(crtc->primary)->rotation != BIT(DRM_ROTATE_0)) {
		650	if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
		651	DRM_DEBUG_KMS("Rotation unsupported, disabling\n");
		652	goto out_disable;
		653	}
3031	serge	654
		655	/* If the kernel debugger is active, always disable compression */
		656	if (in_dbg_master())
		657	goto out_disable;
		658
5060	serge	659	if (i915_gem_stolen_setup_compression(dev, obj->base.size,
		660	drm_format_plane_cpp(fb->pixel_format, 0))) {
4104	Serge	661	if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL))
3480	Serge	662	DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
		663	goto out_disable;
		664	}
		665
3031	serge	666	/* If the scanout has not changed, don't modify the FBC settings.
		667	* Note that we make the fundamental assumption that the fb->obj
		668	* cannot be unpinned (and have its GTT offset and fence revoked)
		669	* without first being decoupled from the scanout and FBC disabled.
		670	*/
4104	Serge	671	if (dev_priv->fbc.plane == intel_crtc->plane &&
		672	dev_priv->fbc.fb_id == fb->base.id &&
		673	dev_priv->fbc.y == crtc->y)
3031	serge	674	return;
		675
		676	if (intel_fbc_enabled(dev)) {
		677	/* We update FBC along two paths, after changing fb/crtc
		678	* configuration (modeswitching) and after page-flipping
		679	* finishes. For the latter, we know that not only did
		680	* we disable the FBC at the start of the page-flip
		681	* sequence, but also more than one vblank has passed.
		682	*
		683	* For the former case of modeswitching, it is possible
		684	* to switch between two FBC valid configurations
		685	* instantaneously so we do need to disable the FBC
		686	* before we can modify its control registers. We also
		687	* have to wait for the next vblank for that to take
		688	* effect. However, since we delay enabling FBC we can
		689	* assume that a vblank has passed since disabling and
		690	* that we can safely alter the registers in the deferred
		691	* callback.
		692	*
		693	* In the scenario that we go from a valid to invalid
		694	* and then back to valid FBC configuration we have
		695	* no strict enforcement that a vblank occurred since
		696	* disabling the FBC. However, along all current pipe
		697	* disabling paths we do need to wait for a vblank at
		698	* some point. And we wait before enabling FBC anyway.
		699	*/
		700	DRM_DEBUG_KMS("disabling active FBC for update\n");
		701	intel_disable_fbc(dev);
		702	}
		703
4560	Serge	704	intel_enable_fbc(crtc);
4104	Serge	705	dev_priv->fbc.no_fbc_reason = FBC_OK;
3031	serge	706	return;
		707
		708	out_disable:
		709	/* Multiple disables should be harmless */
		710	if (intel_fbc_enabled(dev)) {
		711	DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
		712	intel_disable_fbc(dev);
		713	}
3480	Serge	714	i915_gem_stolen_cleanup_compression(dev);
3031	serge	715	}
		716
		717	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		718	{
5060	serge	719	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	720	u32 tmp;
		721
		722	tmp = I915_READ(CLKCFG);
		723
		724	switch (tmp & CLKCFG_FSB_MASK) {
		725	case CLKCFG_FSB_533:
		726	dev_priv->fsb_freq = 533; /* 1334 /
		727	break;
		728	case CLKCFG_FSB_800:
		729	dev_priv->fsb_freq = 800; /* 2004 /
		730	break;
		731	case CLKCFG_FSB_667:
		732	dev_priv->fsb_freq = 667; /* 1674 /
		733	break;
		734	case CLKCFG_FSB_400:
		735	dev_priv->fsb_freq = 400; /* 1004 /
		736	break;
		737	}
		738
		739	switch (tmp & CLKCFG_MEM_MASK) {
		740	case CLKCFG_MEM_533:
		741	dev_priv->mem_freq = 533;
		742	break;
		743	case CLKCFG_MEM_667:
		744	dev_priv->mem_freq = 667;
		745	break;
		746	case CLKCFG_MEM_800:
		747	dev_priv->mem_freq = 800;
		748	break;
		749	}
		750
		751	/* detect pineview DDR3 setting */
		752	tmp = I915_READ(CSHRDDR3CTL);
		753	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		754	}
		755
		756	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		757	{
5060	serge	758	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	759	u16 ddrpll, csipll;
		760
		761	ddrpll = I915_READ16(DDRMPLL1);
		762	csipll = I915_READ16(CSIPLL0);
		763
		764	switch (ddrpll & 0xff) {
		765	case 0xc:
		766	dev_priv->mem_freq = 800;
		767	break;
		768	case 0x10:
		769	dev_priv->mem_freq = 1066;
		770	break;
		771	case 0x14:
		772	dev_priv->mem_freq = 1333;
		773	break;
		774	case 0x18:
		775	dev_priv->mem_freq = 1600;
		776	break;
		777	default:
		778	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		779	ddrpll & 0xff);
		780	dev_priv->mem_freq = 0;
		781	break;
		782	}
		783
		784	dev_priv->ips.r_t = dev_priv->mem_freq;
		785
		786	switch (csipll & 0x3ff) {
		787	case 0x00c:
		788	dev_priv->fsb_freq = 3200;
		789	break;
		790	case 0x00e:
		791	dev_priv->fsb_freq = 3733;
		792	break;
		793	case 0x010:
		794	dev_priv->fsb_freq = 4266;
		795	break;
		796	case 0x012:
		797	dev_priv->fsb_freq = 4800;
		798	break;
		799	case 0x014:
		800	dev_priv->fsb_freq = 5333;
		801	break;
		802	case 0x016:
		803	dev_priv->fsb_freq = 5866;
		804	break;
		805	case 0x018:
		806	dev_priv->fsb_freq = 6400;
		807	break;
		808	default:
		809	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		810	csipll & 0x3ff);
		811	dev_priv->fsb_freq = 0;
		812	break;
		813	}
		814
		815	if (dev_priv->fsb_freq == 3200) {
		816	dev_priv->ips.c_m = 0;
		817	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		818	dev_priv->ips.c_m = 1;
		819	} else {
		820	dev_priv->ips.c_m = 2;
		821	}
		822	}
		823
		824	static const struct cxsr_latency cxsr_latency_table[] = {
		825	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		826	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		827	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		828	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		829	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		830
		831	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		832	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		833	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		834	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		835	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		836
		837	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		838	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		839	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		840	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		841	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		842
		843	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		844	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		845	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		846	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		847	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		848
		849	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		850	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		851	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		852	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		853	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		854
		855	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		856	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		857	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		858	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		859	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		860	};
		861
		862	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		863	int is_ddr3,
		864	int fsb,
		865	int mem)
		866	{
		867	const struct cxsr_latency *latency;
		868	int i;
		869
		870	if (fsb == 0 \|\| mem == 0)
		871	return NULL;
		872
		873	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		874	latency = &cxsr_latency_table[i];
		875	if (is_desktop == latency->is_desktop &&
		876	is_ddr3 == latency->is_ddr3 &&
		877	fsb == latency->fsb_freq && mem == latency->mem_freq)
		878	return latency;
		879	}
		880
		881	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		882
		883	return NULL;
		884	}
		885
5060	serge	886	void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
3031	serge	887	{
5060	serge	888	struct drm_device *dev = dev_priv->dev;
		889	u32 val;
3031	serge	890
5060	serge	891	if (IS_VALLEYVIEW(dev)) {
		892	I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
		893	} else if (IS_G4X(dev) \|\| IS_CRESTLINE(dev)) {
		894	I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
		895	} else if (IS_PINEVIEW(dev)) {
		896	val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
		897	val \|= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
		898	I915_WRITE(DSPFW3, val);
		899	} else if (IS_I945G(dev) \|\| IS_I945GM(dev)) {
		900	val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
		901	_MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
		902	I915_WRITE(FW_BLC_SELF, val);
		903	} else if (IS_I915GM(dev)) {
		904	val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
		905	_MASKED_BIT_DISABLE(INSTPM_SELF_EN);
		906	I915_WRITE(INSTPM, val);
		907	} else {
		908	return;
		909	}
		910
		911	DRM_DEBUG_KMS("memory self-refresh is %s\n",
		912	enable ? "enabled" : "disabled");
3031	serge	913	}
		914
		915	/*
		916	* Latency for FIFO fetches is dependent on several factors:
		917	* - memory configuration (speed, channels)
		918	* - chipset
		919	* - current MCH state
		920	* It can be fairly high in some situations, so here we assume a fairly
		921	* pessimal value. It's a tradeoff between extra memory fetches (if we
		922	* set this value too high, the FIFO will fetch frequently to stay full)
		923	* and power consumption (set it too low to save power and we might see
		924	* FIFO underruns and display "flicker").
		925	*
		926	* A value of 5us seems to be a good balance; safe for very low end
		927	* platforms but not overly aggressive on lower latency configs.
		928	*/
5354	serge	929	static const int pessimal_latency_ns = 5000;
3031	serge	930
		931	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		932	{
		933	struct drm_i915_private *dev_priv = dev->dev_private;
		934	uint32_t dsparb = I915_READ(DSPARB);
		935	int size;
		936
		937	size = dsparb & 0x7f;
		938	if (plane)
		939	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		940
		941	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		942	plane ? "B" : "A", size);
		943
		944	return size;
		945	}
		946
4560	Serge	947	static int i830_get_fifo_size(struct drm_device *dev, int plane)
3031	serge	948	{
		949	struct drm_i915_private *dev_priv = dev->dev_private;
		950	uint32_t dsparb = I915_READ(DSPARB);
		951	int size;
		952
		953	size = dsparb & 0x1ff;
		954	if (plane)
		955	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		956	size >>= 1; /* Convert to cachelines */
		957
		958	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		959	plane ? "B" : "A", size);
		960
		961	return size;
		962	}
		963
		964	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		965	{
		966	struct drm_i915_private *dev_priv = dev->dev_private;
		967	uint32_t dsparb = I915_READ(DSPARB);
		968	int size;
		969
		970	size = dsparb & 0x7f;
		971	size >>= 2; /* Convert to cachelines */
		972
		973	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		974	plane ? "B" : "A",
		975	size);
		976
		977	return size;
		978	}
		979
		980	/* Pineview has different values for various configs */
		981	static const struct intel_watermark_params pineview_display_wm = {
5060	serge	982	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		983	.max_wm = PINEVIEW_MAX_WM,
		984	.default_wm = PINEVIEW_DFT_WM,
		985	.guard_size = PINEVIEW_GUARD_WM,
		986	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	987	};
		988	static const struct intel_watermark_params pineview_display_hplloff_wm = {
5060	serge	989	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		990	.max_wm = PINEVIEW_MAX_WM,
		991	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
		992	.guard_size = PINEVIEW_GUARD_WM,
		993	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	994	};
		995	static const struct intel_watermark_params pineview_cursor_wm = {
5060	serge	996	.fifo_size = PINEVIEW_CURSOR_FIFO,
		997	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		998	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		999	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		1000	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	1001	};
		1002	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
5060	serge	1003	.fifo_size = PINEVIEW_CURSOR_FIFO,
		1004	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		1005	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		1006	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		1007	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	1008	};
		1009	static const struct intel_watermark_params g4x_wm_info = {
5060	serge	1010	.fifo_size = G4X_FIFO_SIZE,
		1011	.max_wm = G4X_MAX_WM,
		1012	.default_wm = G4X_MAX_WM,
		1013	.guard_size = 2,
		1014	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	1015	};
		1016	static const struct intel_watermark_params g4x_cursor_wm_info = {
5060	serge	1017	.fifo_size = I965_CURSOR_FIFO,
		1018	.max_wm = I965_CURSOR_MAX_WM,
		1019	.default_wm = I965_CURSOR_DFT_WM,
		1020	.guard_size = 2,
		1021	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	1022	};
		1023	static const struct intel_watermark_params valleyview_wm_info = {
5060	serge	1024	.fifo_size = VALLEYVIEW_FIFO_SIZE,
		1025	.max_wm = VALLEYVIEW_MAX_WM,
		1026	.default_wm = VALLEYVIEW_MAX_WM,
		1027	.guard_size = 2,
		1028	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	1029	};
		1030	static const struct intel_watermark_params valleyview_cursor_wm_info = {
5060	serge	1031	.fifo_size = I965_CURSOR_FIFO,
		1032	.max_wm = VALLEYVIEW_CURSOR_MAX_WM,
		1033	.default_wm = I965_CURSOR_DFT_WM,
		1034	.guard_size = 2,
		1035	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	1036	};
		1037	static const struct intel_watermark_params i965_cursor_wm_info = {
5060	serge	1038	.fifo_size = I965_CURSOR_FIFO,
		1039	.max_wm = I965_CURSOR_MAX_WM,
		1040	.default_wm = I965_CURSOR_DFT_WM,
		1041	.guard_size = 2,
		1042	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	1043	};
		1044	static const struct intel_watermark_params i945_wm_info = {
5060	serge	1045	.fifo_size = I945_FIFO_SIZE,
		1046	.max_wm = I915_MAX_WM,
		1047	.default_wm = 1,
		1048	.guard_size = 2,
		1049	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	1050	};
		1051	static const struct intel_watermark_params i915_wm_info = {
5060	serge	1052	.fifo_size = I915_FIFO_SIZE,
		1053	.max_wm = I915_MAX_WM,
		1054	.default_wm = 1,
		1055	.guard_size = 2,
		1056	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	1057	};
5354	serge	1058	static const struct intel_watermark_params i830_a_wm_info = {
5060	serge	1059	.fifo_size = I855GM_FIFO_SIZE,
		1060	.max_wm = I915_MAX_WM,
		1061	.default_wm = 1,
		1062	.guard_size = 2,
		1063	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	1064	};
5354	serge	1065	static const struct intel_watermark_params i830_bc_wm_info = {
		1066	.fifo_size = I855GM_FIFO_SIZE,
		1067	.max_wm = I915_MAX_WM/2,
		1068	.default_wm = 1,
		1069	.guard_size = 2,
		1070	.cacheline_size = I830_FIFO_LINE_SIZE,
		1071	};
4560	Serge	1072	static const struct intel_watermark_params i845_wm_info = {
5060	serge	1073	.fifo_size = I830_FIFO_SIZE,
		1074	.max_wm = I915_MAX_WM,
		1075	.default_wm = 1,
		1076	.guard_size = 2,
		1077	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	1078	};
		1079
		1080	/**
		1081	* intel_calculate_wm - calculate watermark level
		1082	* @clock_in_khz: pixel clock
		1083	* @wm: chip FIFO params
		1084	* @pixel_size: display pixel size
		1085	* @latency_ns: memory latency for the platform
		1086	*
		1087	* Calculate the watermark level (the level at which the display plane will
		1088	* start fetching from memory again). Each chip has a different display
		1089	* FIFO size and allocation, so the caller needs to figure that out and pass
		1090	* in the correct intel_watermark_params structure.
		1091	*
		1092	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		1093	* on the pixel size. When it reaches the watermark level, it'll start
		1094	* fetching FIFO line sized based chunks from memory until the FIFO fills
		1095	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		1096	* will occur, and a display engine hang could result.
		1097	*/
		1098	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		1099	const struct intel_watermark_params *wm,
		1100	int fifo_size,
		1101	int pixel_size,
		1102	unsigned long latency_ns)
		1103	{
		1104	long entries_required, wm_size;
		1105
		1106	/*
		1107	* Note: we need to make sure we don't overflow for various clock &
		1108	* latency values.
		1109	* clocks go from a few thousand to several hundred thousand.
		1110	* latency is usually a few thousand
		1111	*/
		1112	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		1113	1000;
		1114	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		1115
		1116	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		1117
		1118	wm_size = fifo_size - (entries_required + wm->guard_size);
		1119
		1120	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		1121
		1122	/* Don't promote wm_size to unsigned... */
		1123	if (wm_size > (long)wm->max_wm)
		1124	wm_size = wm->max_wm;
		1125	if (wm_size <= 0)
		1126	wm_size = wm->default_wm;
5354	serge	1127
		1128	/*
		1129	* Bspec seems to indicate that the value shouldn't be lower than
		1130	* 'burst size + 1'. Certainly 830 is quite unhappy with low values.
		1131	* Lets go for 8 which is the burst size since certain platforms
		1132	* already use a hardcoded 8 (which is what the spec says should be
		1133	* done).
		1134	*/
		1135	if (wm_size <= 8)
		1136	wm_size = 8;
		1137
3031	serge	1138	return wm_size;
		1139	}
		1140
		1141	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		1142	{
		1143	struct drm_crtc crtc, enabled = NULL;
		1144
5060	serge	1145	for_each_crtc(dev, crtc) {
3243	Serge	1146	if (intel_crtc_active(crtc)) {
3031	serge	1147	if (enabled)
		1148	return NULL;
		1149	enabled = crtc;
		1150	}
		1151	}
		1152
		1153	return enabled;
		1154	}
		1155
4560	Serge	1156	static void pineview_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1157	{
4560	Serge	1158	struct drm_device *dev = unused_crtc->dev;
3031	serge	1159	struct drm_i915_private *dev_priv = dev->dev_private;
		1160	struct drm_crtc *crtc;
		1161	const struct cxsr_latency *latency;
		1162	u32 reg;
		1163	unsigned long wm;
		1164
		1165	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		1166	dev_priv->fsb_freq, dev_priv->mem_freq);
		1167	if (!latency) {
		1168	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
5060	serge	1169	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1170	return;
		1171	}
		1172
		1173	crtc = single_enabled_crtc(dev);
		1174	if (crtc) {
4560	Serge	1175	const struct drm_display_mode *adjusted_mode;
5060	serge	1176	int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
4560	Serge	1177	int clock;
3031	serge	1178
4560	Serge	1179	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1180	clock = adjusted_mode->crtc_clock;
		1181
3031	serge	1182	/* Display SR */
		1183	wm = intel_calculate_wm(clock, &pineview_display_wm,
		1184	pineview_display_wm.fifo_size,
		1185	pixel_size, latency->display_sr);
		1186	reg = I915_READ(DSPFW1);
		1187	reg &= ~DSPFW_SR_MASK;
		1188	reg \|= wm << DSPFW_SR_SHIFT;
		1189	I915_WRITE(DSPFW1, reg);
		1190	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		1191
		1192	/* cursor SR */
		1193	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		1194	pineview_display_wm.fifo_size,
		1195	pixel_size, latency->cursor_sr);
		1196	reg = I915_READ(DSPFW3);
		1197	reg &= ~DSPFW_CURSOR_SR_MASK;
		1198	reg \|= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
		1199	I915_WRITE(DSPFW3, reg);
		1200
		1201	/* Display HPLL off SR */
		1202	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		1203	pineview_display_hplloff_wm.fifo_size,
		1204	pixel_size, latency->display_hpll_disable);
		1205	reg = I915_READ(DSPFW3);
		1206	reg &= ~DSPFW_HPLL_SR_MASK;
		1207	reg \|= wm & DSPFW_HPLL_SR_MASK;
		1208	I915_WRITE(DSPFW3, reg);
		1209
		1210	/* cursor HPLL off SR */
		1211	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		1212	pineview_display_hplloff_wm.fifo_size,
		1213	pixel_size, latency->cursor_hpll_disable);
		1214	reg = I915_READ(DSPFW3);
		1215	reg &= ~DSPFW_HPLL_CURSOR_MASK;
		1216	reg \|= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
		1217	I915_WRITE(DSPFW3, reg);
		1218	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		1219
5060	serge	1220	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1221	} else {
5060	serge	1222	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1223	}
		1224	}
		1225
		1226	static bool g4x_compute_wm0(struct drm_device *dev,
		1227	int plane,
		1228	const struct intel_watermark_params *display,
		1229	int display_latency_ns,
		1230	const struct intel_watermark_params *cursor,
		1231	int cursor_latency_ns,
		1232	int *plane_wm,
		1233	int *cursor_wm)
		1234	{
		1235	struct drm_crtc *crtc;
4560	Serge	1236	const struct drm_display_mode *adjusted_mode;
3031	serge	1237	int htotal, hdisplay, clock, pixel_size;
		1238	int line_time_us, line_count;
		1239	int entries, tlb_miss;
		1240
		1241	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1242	if (!intel_crtc_active(crtc)) {
3031	serge	1243	*cursor_wm = cursor->guard_size;
		1244	*plane_wm = display->guard_size;
		1245	return false;
		1246	}
		1247
4560	Serge	1248	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1249	clock = adjusted_mode->crtc_clock;
		1250	htotal = adjusted_mode->crtc_htotal;
		1251	hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
5060	serge	1252	pixel_size = crtc->primary->fb->bits_per_pixel / 8;
3031	serge	1253
		1254	/* Use the small buffer method to calculate plane watermark */
		1255	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1256	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		1257	if (tlb_miss > 0)
		1258	entries += tlb_miss;
		1259	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1260	*plane_wm = entries + display->guard_size;
		1261	if (*plane_wm > (int)display->max_wm)
		1262	*plane_wm = display->max_wm;
		1263
		1264	/* Use the large buffer method to calculate cursor watermark */
5060	serge	1265	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1266	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
5060	serge	1267	entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size;
3031	serge	1268	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		1269	if (tlb_miss > 0)
		1270	entries += tlb_miss;
		1271	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1272	*cursor_wm = entries + cursor->guard_size;
		1273	if (*cursor_wm > (int)cursor->max_wm)
		1274	*cursor_wm = (int)cursor->max_wm;
		1275
		1276	return true;
		1277	}
		1278
		1279	/*
		1280	* Check the wm result.
		1281	*
		1282	* If any calculated watermark values is larger than the maximum value that
		1283	* can be programmed into the associated watermark register, that watermark
		1284	* must be disabled.
		1285	*/
		1286	static bool g4x_check_srwm(struct drm_device *dev,
		1287	int display_wm, int cursor_wm,
		1288	const struct intel_watermark_params *display,
		1289	const struct intel_watermark_params *cursor)
		1290	{
		1291	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		1292	display_wm, cursor_wm);
		1293
		1294	if (display_wm > display->max_wm) {
		1295	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		1296	display_wm, display->max_wm);
		1297	return false;
		1298	}
		1299
		1300	if (cursor_wm > cursor->max_wm) {
		1301	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		1302	cursor_wm, cursor->max_wm);
		1303	return false;
		1304	}
		1305
		1306	if (!(display_wm \|\| cursor_wm)) {
		1307	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		1308	return false;
		1309	}
		1310
		1311	return true;
		1312	}
		1313
		1314	static bool g4x_compute_srwm(struct drm_device *dev,
		1315	int plane,
		1316	int latency_ns,
		1317	const struct intel_watermark_params *display,
		1318	const struct intel_watermark_params *cursor,
		1319	int display_wm, int cursor_wm)
		1320	{
		1321	struct drm_crtc *crtc;
4560	Serge	1322	const struct drm_display_mode *adjusted_mode;
3031	serge	1323	int hdisplay, htotal, pixel_size, clock;
		1324	unsigned long line_time_us;
		1325	int line_count, line_size;
		1326	int small, large;
		1327	int entries;
		1328
		1329	if (!latency_ns) {
		1330	display_wm = cursor_wm = 0;
		1331	return false;
		1332	}
		1333
		1334	crtc = intel_get_crtc_for_plane(dev, plane);
4560	Serge	1335	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1336	clock = adjusted_mode->crtc_clock;
		1337	htotal = adjusted_mode->crtc_htotal;
		1338	hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
5060	serge	1339	pixel_size = crtc->primary->fb->bits_per_pixel / 8;
3031	serge	1340
5060	serge	1341	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1342	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1343	line_size = hdisplay * pixel_size;
		1344
		1345	/* Use the minimum of the small and large buffer method for primary */
		1346	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1347	large = line_count * line_size;
		1348
		1349	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1350	*display_wm = entries + display->guard_size;
		1351
		1352	/* calculate the self-refresh watermark for display cursor */
5060	serge	1353	entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width;
3031	serge	1354	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1355	*cursor_wm = entries + cursor->guard_size;
		1356
		1357	return g4x_check_srwm(dev,
		1358	display_wm, cursor_wm,
		1359	display, cursor);
		1360	}
		1361
5354	serge	1362	static bool vlv_compute_drain_latency(struct drm_crtc *crtc,
		1363	int pixel_size,
		1364	int *prec_mult,
		1365	int *drain_latency)
3031	serge	1366	{
5354	serge	1367	struct drm_device *dev = crtc->dev;
3031	serge	1368	int entries;
5354	serge	1369	int clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
3031	serge	1370
5354	serge	1371	if (WARN(clock == 0, "Pixel clock is zero!\n"))
3031	serge	1372	return false;
		1373
5354	serge	1374	if (WARN(pixel_size == 0, "Pixel size is zero!\n"))
		1375	return false;
3031	serge	1376
5354	serge	1377	entries = DIV_ROUND_UP(clock, 1000) * pixel_size;
		1378	if (IS_CHERRYVIEW(dev))
		1379	*prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_32 :
		1380	DRAIN_LATENCY_PRECISION_16;
		1381	else
		1382	*prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_64 :
		1383	DRAIN_LATENCY_PRECISION_32;
		1384	drain_latency = (64 (prec_mult) 4) / entries;
3031	serge	1385
5354	serge	1386	if (*drain_latency > DRAIN_LATENCY_MASK)
		1387	*drain_latency = DRAIN_LATENCY_MASK;
3031	serge	1388
		1389	return true;
		1390	}
		1391
		1392	/*
		1393	* Update drain latency registers of memory arbiter
		1394	*
		1395	* Valleyview SoC has a new memory arbiter and needs drain latency registers
		1396	* to be programmed. Each plane has a drain latency multiplier and a drain
		1397	* latency value.
		1398	*/
		1399
5354	serge	1400	static void vlv_update_drain_latency(struct drm_crtc *crtc)
3031	serge	1401	{
5354	serge	1402	struct drm_device *dev = crtc->dev;
3031	serge	1403	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	1404	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1405	int pixel_size;
		1406	int drain_latency;
		1407	enum pipe pipe = intel_crtc->pipe;
		1408	int plane_prec, prec_mult, plane_dl;
		1409	const int high_precision = IS_CHERRYVIEW(dev) ?
		1410	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_64;
3031	serge	1411
5354	serge	1412	plane_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_PLANE_PRECISION_HIGH \|
		1413	DRAIN_LATENCY_MASK \| DDL_CURSOR_PRECISION_HIGH \|
		1414	(DRAIN_LATENCY_MASK << DDL_CURSOR_SHIFT));
3031	serge	1415
5354	serge	1416	if (!intel_crtc_active(crtc)) {
		1417	I915_WRITE(VLV_DDL(pipe), plane_dl);
		1418	return;
3031	serge	1419	}
		1420
5354	serge	1421	/* Primary plane Drain Latency */
		1422	pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */
		1423	if (vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
		1424	plane_prec = (prec_mult == high_precision) ?
		1425	DDL_PLANE_PRECISION_HIGH :
		1426	DDL_PLANE_PRECISION_LOW;
		1427	plane_dl \|= plane_prec \| drain_latency;
		1428	}
3031	serge	1429
5354	serge	1430	/* Cursor Drain Latency
		1431	* BPP is always 4 for cursor
		1432	*/
		1433	pixel_size = 4;
		1434
		1435	/* Program cursor DL only if it is enabled */
		1436	if (intel_crtc->cursor_base &&
		1437	vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
		1438	plane_prec = (prec_mult == high_precision) ?
		1439	DDL_CURSOR_PRECISION_HIGH :
		1440	DDL_CURSOR_PRECISION_LOW;
		1441	plane_dl \|= plane_prec \| (drain_latency << DDL_CURSOR_SHIFT);
3031	serge	1442	}
5354	serge	1443
		1444	I915_WRITE(VLV_DDL(pipe), plane_dl);
3031	serge	1445	}
		1446
		1447	#define single_plane_enabled(mask) is_power_of_2(mask)
		1448
4560	Serge	1449	static void valleyview_update_wm(struct drm_crtc *crtc)
3031	serge	1450	{
4560	Serge	1451	struct drm_device *dev = crtc->dev;
3031	serge	1452	static const int sr_latency_ns = 12000;
		1453	struct drm_i915_private *dev_priv = dev->dev_private;
		1454	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1455	int plane_sr, cursor_sr;
3243	Serge	1456	int ignore_plane_sr, ignore_cursor_sr;
3031	serge	1457	unsigned int enabled = 0;
5060	serge	1458	bool cxsr_enabled;
3031	serge	1459
5354	serge	1460	vlv_update_drain_latency(crtc);
3031	serge	1461
3746	Serge	1462	if (g4x_compute_wm0(dev, PIPE_A,
5354	serge	1463	&valleyview_wm_info, pessimal_latency_ns,
		1464	&valleyview_cursor_wm_info, pessimal_latency_ns,
3031	serge	1465	&planea_wm, &cursora_wm))
3746	Serge	1466	enabled \|= 1 << PIPE_A;
3031	serge	1467
3746	Serge	1468	if (g4x_compute_wm0(dev, PIPE_B,
5354	serge	1469	&valleyview_wm_info, pessimal_latency_ns,
		1470	&valleyview_cursor_wm_info, pessimal_latency_ns,
3031	serge	1471	&planeb_wm, &cursorb_wm))
3746	Serge	1472	enabled \|= 1 << PIPE_B;
3031	serge	1473
		1474	if (single_plane_enabled(enabled) &&
		1475	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1476	sr_latency_ns,
		1477	&valleyview_wm_info,
		1478	&valleyview_cursor_wm_info,
3243	Serge	1479	&plane_sr, &ignore_cursor_sr) &&
		1480	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1481	2*sr_latency_ns,
		1482	&valleyview_wm_info,
		1483	&valleyview_cursor_wm_info,
		1484	&ignore_plane_sr, &cursor_sr)) {
5060	serge	1485	cxsr_enabled = true;
3243	Serge	1486	} else {
5060	serge	1487	cxsr_enabled = false;
		1488	intel_set_memory_cxsr(dev_priv, false);
3243	Serge	1489	plane_sr = cursor_sr = 0;
		1490	}
3031	serge	1491
5354	serge	1492	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
		1493	"B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
3031	serge	1494	planea_wm, cursora_wm,
		1495	planeb_wm, cursorb_wm,
		1496	plane_sr, cursor_sr);
		1497
		1498	I915_WRITE(DSPFW1,
		1499	(plane_sr << DSPFW_SR_SHIFT) \|
		1500	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1501	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
5354	serge	1502	(planea_wm << DSPFW_PLANEA_SHIFT));
3031	serge	1503	I915_WRITE(DSPFW2,
3243	Serge	1504	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1505	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1506	I915_WRITE(DSPFW3,
3243	Serge	1507	(I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) \|
		1508	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
5060	serge	1509
		1510	if (cxsr_enabled)
		1511	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1512	}
		1513
5354	serge	1514	static void cherryview_update_wm(struct drm_crtc *crtc)
		1515	{
		1516	struct drm_device *dev = crtc->dev;
		1517	static const int sr_latency_ns = 12000;
		1518	struct drm_i915_private *dev_priv = dev->dev_private;
		1519	int planea_wm, planeb_wm, planec_wm;
		1520	int cursora_wm, cursorb_wm, cursorc_wm;
		1521	int plane_sr, cursor_sr;
		1522	int ignore_plane_sr, ignore_cursor_sr;
		1523	unsigned int enabled = 0;
		1524	bool cxsr_enabled;
		1525
		1526	vlv_update_drain_latency(crtc);
		1527
		1528	if (g4x_compute_wm0(dev, PIPE_A,
		1529	&valleyview_wm_info, pessimal_latency_ns,
		1530	&valleyview_cursor_wm_info, pessimal_latency_ns,
		1531	&planea_wm, &cursora_wm))
		1532	enabled \|= 1 << PIPE_A;
		1533
		1534	if (g4x_compute_wm0(dev, PIPE_B,
		1535	&valleyview_wm_info, pessimal_latency_ns,
		1536	&valleyview_cursor_wm_info, pessimal_latency_ns,
		1537	&planeb_wm, &cursorb_wm))
		1538	enabled \|= 1 << PIPE_B;
		1539
		1540	if (g4x_compute_wm0(dev, PIPE_C,
		1541	&valleyview_wm_info, pessimal_latency_ns,
		1542	&valleyview_cursor_wm_info, pessimal_latency_ns,
		1543	&planec_wm, &cursorc_wm))
		1544	enabled \|= 1 << PIPE_C;
		1545
		1546	if (single_plane_enabled(enabled) &&
		1547	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1548	sr_latency_ns,
		1549	&valleyview_wm_info,
		1550	&valleyview_cursor_wm_info,
		1551	&plane_sr, &ignore_cursor_sr) &&
		1552	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1553	2*sr_latency_ns,
		1554	&valleyview_wm_info,
		1555	&valleyview_cursor_wm_info,
		1556	&ignore_plane_sr, &cursor_sr)) {
		1557	cxsr_enabled = true;
		1558	} else {
		1559	cxsr_enabled = false;
		1560	intel_set_memory_cxsr(dev_priv, false);
		1561	plane_sr = cursor_sr = 0;
		1562	}
		1563
		1564	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
		1565	"B: plane=%d, cursor=%d, C: plane=%d, cursor=%d, "
		1566	"SR: plane=%d, cursor=%d\n",
		1567	planea_wm, cursora_wm,
		1568	planeb_wm, cursorb_wm,
		1569	planec_wm, cursorc_wm,
		1570	plane_sr, cursor_sr);
		1571
		1572	I915_WRITE(DSPFW1,
		1573	(plane_sr << DSPFW_SR_SHIFT) \|
		1574	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1575	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1576	(planea_wm << DSPFW_PLANEA_SHIFT));
		1577	I915_WRITE(DSPFW2,
		1578	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
		1579	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1580	I915_WRITE(DSPFW3,
		1581	(I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) \|
		1582	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1583	I915_WRITE(DSPFW9_CHV,
		1584	(I915_READ(DSPFW9_CHV) & ~(DSPFW_PLANEC_MASK \|
		1585	DSPFW_CURSORC_MASK)) \|
		1586	(planec_wm << DSPFW_PLANEC_SHIFT) \|
		1587	(cursorc_wm << DSPFW_CURSORC_SHIFT));
		1588
		1589	if (cxsr_enabled)
		1590	intel_set_memory_cxsr(dev_priv, true);
		1591	}
		1592
		1593	static void valleyview_update_sprite_wm(struct drm_plane *plane,
		1594	struct drm_crtc *crtc,
		1595	uint32_t sprite_width,
		1596	uint32_t sprite_height,
		1597	int pixel_size,
		1598	bool enabled, bool scaled)
		1599	{
		1600	struct drm_device *dev = crtc->dev;
		1601	struct drm_i915_private *dev_priv = dev->dev_private;
		1602	int pipe = to_intel_plane(plane)->pipe;
		1603	int sprite = to_intel_plane(plane)->plane;
		1604	int drain_latency;
		1605	int plane_prec;
		1606	int sprite_dl;
		1607	int prec_mult;
		1608	const int high_precision = IS_CHERRYVIEW(dev) ?
		1609	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_64;
		1610
		1611	sprite_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_SPRITE_PRECISION_HIGH(sprite) \|
		1612	(DRAIN_LATENCY_MASK << DDL_SPRITE_SHIFT(sprite)));
		1613
		1614	if (enabled && vlv_compute_drain_latency(crtc, pixel_size, &prec_mult,
		1615	&drain_latency)) {
		1616	plane_prec = (prec_mult == high_precision) ?
		1617	DDL_SPRITE_PRECISION_HIGH(sprite) :
		1618	DDL_SPRITE_PRECISION_LOW(sprite);
		1619	sprite_dl \|= plane_prec \|
		1620	(drain_latency << DDL_SPRITE_SHIFT(sprite));
		1621	}
		1622
		1623	I915_WRITE(VLV_DDL(pipe), sprite_dl);
		1624	}
		1625
4560	Serge	1626	static void g4x_update_wm(struct drm_crtc *crtc)
3031	serge	1627	{
4560	Serge	1628	struct drm_device *dev = crtc->dev;
3031	serge	1629	static const int sr_latency_ns = 12000;
		1630	struct drm_i915_private *dev_priv = dev->dev_private;
		1631	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1632	int plane_sr, cursor_sr;
		1633	unsigned int enabled = 0;
5060	serge	1634	bool cxsr_enabled;
3031	serge	1635
3746	Serge	1636	if (g4x_compute_wm0(dev, PIPE_A,
5354	serge	1637	&g4x_wm_info, pessimal_latency_ns,
		1638	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1639	&planea_wm, &cursora_wm))
3746	Serge	1640	enabled \|= 1 << PIPE_A;
3031	serge	1641
3746	Serge	1642	if (g4x_compute_wm0(dev, PIPE_B,
5354	serge	1643	&g4x_wm_info, pessimal_latency_ns,
		1644	&g4x_cursor_wm_info, pessimal_latency_ns,
3031	serge	1645	&planeb_wm, &cursorb_wm))
3746	Serge	1646	enabled \|= 1 << PIPE_B;
3031	serge	1647
		1648	if (single_plane_enabled(enabled) &&
		1649	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1650	sr_latency_ns,
		1651	&g4x_wm_info,
		1652	&g4x_cursor_wm_info,
3243	Serge	1653	&plane_sr, &cursor_sr)) {
5060	serge	1654	cxsr_enabled = true;
3243	Serge	1655	} else {
5060	serge	1656	cxsr_enabled = false;
		1657	intel_set_memory_cxsr(dev_priv, false);
3243	Serge	1658	plane_sr = cursor_sr = 0;
		1659	}
3031	serge	1660
5354	serge	1661	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
		1662	"B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
3031	serge	1663	planea_wm, cursora_wm,
		1664	planeb_wm, cursorb_wm,
		1665	plane_sr, cursor_sr);
		1666
		1667	I915_WRITE(DSPFW1,
		1668	(plane_sr << DSPFW_SR_SHIFT) \|
		1669	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1670	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
5354	serge	1671	(planea_wm << DSPFW_PLANEA_SHIFT));
3031	serge	1672	I915_WRITE(DSPFW2,
3243	Serge	1673	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1674	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1675	/* HPLL off in SR has some issues on G4x... disable it */
		1676	I915_WRITE(DSPFW3,
3243	Serge	1677	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
3031	serge	1678	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
5060	serge	1679
		1680	if (cxsr_enabled)
		1681	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1682	}
		1683
4560	Serge	1684	static void i965_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1685	{
4560	Serge	1686	struct drm_device *dev = unused_crtc->dev;
3031	serge	1687	struct drm_i915_private *dev_priv = dev->dev_private;
		1688	struct drm_crtc *crtc;
		1689	int srwm = 1;
		1690	int cursor_sr = 16;
5060	serge	1691	bool cxsr_enabled;
3031	serge	1692
		1693	/* Calc sr entries for one plane configs */
		1694	crtc = single_enabled_crtc(dev);
		1695	if (crtc) {
		1696	/* self-refresh has much higher latency */
		1697	static const int sr_latency_ns = 12000;
4560	Serge	1698	const struct drm_display_mode *adjusted_mode =
		1699	&to_intel_crtc(crtc)->config.adjusted_mode;
		1700	int clock = adjusted_mode->crtc_clock;
		1701	int htotal = adjusted_mode->crtc_htotal;
		1702	int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
5060	serge	1703	int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
3031	serge	1704	unsigned long line_time_us;
		1705	int entries;
		1706
5060	serge	1707	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1708
		1709	/* Use ns/us then divide to preserve precision */
		1710	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1711	pixel_size * hdisplay;
		1712	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1713	srwm = I965_FIFO_SIZE - entries;
		1714	if (srwm < 0)
		1715	srwm = 1;
		1716	srwm &= 0x1ff;
		1717	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1718	entries, srwm);
		1719
		1720	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
5060	serge	1721	pixel_size * to_intel_crtc(crtc)->cursor_width;
3031	serge	1722	entries = DIV_ROUND_UP(entries,
		1723	i965_cursor_wm_info.cacheline_size);
		1724	cursor_sr = i965_cursor_wm_info.fifo_size -
		1725	(entries + i965_cursor_wm_info.guard_size);
		1726
		1727	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1728	cursor_sr = i965_cursor_wm_info.max_wm;
		1729
		1730	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1731	"cursor %d\n", srwm, cursor_sr);
		1732
5060	serge	1733	cxsr_enabled = true;
3031	serge	1734	} else {
5060	serge	1735	cxsr_enabled = false;
3031	serge	1736	/* Turn off self refresh if both pipes are enabled */
5060	serge	1737	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1738	}
		1739
		1740	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1741	srwm);
		1742
		1743	/* 965 has limitations... */
		1744	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) \|
5354	serge	1745	(8 << DSPFW_CURSORB_SHIFT) \|
		1746	(8 << DSPFW_PLANEB_SHIFT) \|
		1747	(8 << DSPFW_PLANEA_SHIFT));
		1748	I915_WRITE(DSPFW2, (8 << DSPFW_CURSORA_SHIFT) \|
		1749	(8 << DSPFW_PLANEC_SHIFT_OLD));
3031	serge	1750	/* update cursor SR watermark */
		1751	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
5060	serge	1752
		1753	if (cxsr_enabled)
		1754	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1755	}
		1756
4560	Serge	1757	static void i9xx_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1758	{
4560	Serge	1759	struct drm_device *dev = unused_crtc->dev;
3031	serge	1760	struct drm_i915_private *dev_priv = dev->dev_private;
		1761	const struct intel_watermark_params *wm_info;
		1762	uint32_t fwater_lo;
		1763	uint32_t fwater_hi;
		1764	int cwm, srwm = 1;
		1765	int fifo_size;
		1766	int planea_wm, planeb_wm;
		1767	struct drm_crtc crtc, enabled = NULL;
		1768
		1769	if (IS_I945GM(dev))
		1770	wm_info = &i945_wm_info;
		1771	else if (!IS_GEN2(dev))
		1772	wm_info = &i915_wm_info;
		1773	else
5354	serge	1774	wm_info = &i830_a_wm_info;
3031	serge	1775
		1776	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1777	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1778	if (intel_crtc_active(crtc)) {
4560	Serge	1779	const struct drm_display_mode *adjusted_mode;
5060	serge	1780	int cpp = crtc->primary->fb->bits_per_pixel / 8;
3243	Serge	1781	if (IS_GEN2(dev))
		1782	cpp = 4;
		1783
4560	Serge	1784	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1785	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1786	wm_info, fifo_size, cpp,
5354	serge	1787	pessimal_latency_ns);
3031	serge	1788	enabled = crtc;
5354	serge	1789	} else {
3031	serge	1790	planea_wm = fifo_size - wm_info->guard_size;
5354	serge	1791	if (planea_wm > (long)wm_info->max_wm)
		1792	planea_wm = wm_info->max_wm;
		1793	}
3031	serge	1794
5354	serge	1795	if (IS_GEN2(dev))
		1796	wm_info = &i830_bc_wm_info;
		1797
3031	serge	1798	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1799	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1800	if (intel_crtc_active(crtc)) {
4560	Serge	1801	const struct drm_display_mode *adjusted_mode;
5060	serge	1802	int cpp = crtc->primary->fb->bits_per_pixel / 8;
3243	Serge	1803	if (IS_GEN2(dev))
		1804	cpp = 4;
		1805
4560	Serge	1806	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1807	planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1808	wm_info, fifo_size, cpp,
5354	serge	1809	pessimal_latency_ns);
3031	serge	1810	if (enabled == NULL)
		1811	enabled = crtc;
		1812	else
		1813	enabled = NULL;
5354	serge	1814	} else {
3031	serge	1815	planeb_wm = fifo_size - wm_info->guard_size;
5354	serge	1816	if (planeb_wm > (long)wm_info->max_wm)
		1817	planeb_wm = wm_info->max_wm;
		1818	}
3031	serge	1819
		1820	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1821
5060	serge	1822	if (IS_I915GM(dev) && enabled) {
		1823	struct drm_i915_gem_object *obj;
		1824
		1825	obj = intel_fb_obj(enabled->primary->fb);
		1826
		1827	/* self-refresh seems busted with untiled */
		1828	if (obj->tiling_mode == I915_TILING_NONE)
		1829	enabled = NULL;
		1830	}
		1831
3031	serge	1832	/*
		1833	* Overlay gets an aggressive default since video jitter is bad.
		1834	*/
		1835	cwm = 2;
		1836
		1837	/* Play safe and disable self-refresh before adjusting watermarks. */
5060	serge	1838	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1839
		1840	/* Calc sr entries for one plane configs */
		1841	if (HAS_FW_BLC(dev) && enabled) {
		1842	/* self-refresh has much higher latency */
		1843	static const int sr_latency_ns = 6000;
4560	Serge	1844	const struct drm_display_mode *adjusted_mode =
		1845	&to_intel_crtc(enabled)->config.adjusted_mode;
		1846	int clock = adjusted_mode->crtc_clock;
		1847	int htotal = adjusted_mode->crtc_htotal;
		1848	int hdisplay = to_intel_crtc(enabled)->config.pipe_src_w;
5060	serge	1849	int pixel_size = enabled->primary->fb->bits_per_pixel / 8;
3031	serge	1850	unsigned long line_time_us;
		1851	int entries;
		1852
5060	serge	1853	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1854
		1855	/* Use ns/us then divide to preserve precision */
		1856	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1857	pixel_size * hdisplay;
		1858	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1859	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1860	srwm = wm_info->fifo_size - entries;
		1861	if (srwm < 0)
		1862	srwm = 1;
		1863
		1864	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1865	I915_WRITE(FW_BLC_SELF,
		1866	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1867	else if (IS_I915GM(dev))
		1868	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1869	}
		1870
		1871	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1872	planea_wm, planeb_wm, cwm, srwm);
		1873
		1874	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1875	fwater_hi = (cwm & 0x1f);
		1876
		1877	/* Set request length to 8 cachelines per fetch */
		1878	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1879	fwater_hi = fwater_hi \| (1 << 8);
		1880
		1881	I915_WRITE(FW_BLC, fwater_lo);
		1882	I915_WRITE(FW_BLC2, fwater_hi);
		1883
5060	serge	1884	if (enabled)
		1885	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1886	}
		1887
4560	Serge	1888	static void i845_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1889	{
4560	Serge	1890	struct drm_device *dev = unused_crtc->dev;
3031	serge	1891	struct drm_i915_private *dev_priv = dev->dev_private;
		1892	struct drm_crtc *crtc;
4560	Serge	1893	const struct drm_display_mode *adjusted_mode;
3031	serge	1894	uint32_t fwater_lo;
		1895	int planea_wm;
		1896
		1897	crtc = single_enabled_crtc(dev);
		1898	if (crtc == NULL)
		1899	return;
		1900
4560	Serge	1901	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1902	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
		1903	&i845_wm_info,
3031	serge	1904	dev_priv->display.get_fifo_size(dev, 0),
5354	serge	1905	4, pessimal_latency_ns);
3031	serge	1906	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1907	fwater_lo \|= (3<<8) \| planea_wm;
		1908
		1909	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1910
		1911	I915_WRITE(FW_BLC, fwater_lo);
		1912	}
		1913
4104	Serge	1914	static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
		1915	struct drm_crtc *crtc)
3031	serge	1916	{
4104	Serge	1917	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1918	uint32_t pixel_rate;
		1919
4560	Serge	1920	pixel_rate = intel_crtc->config.adjusted_mode.crtc_clock;
4104	Serge	1921
		1922	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
		1923	* adjust the pixel_rate here. */
		1924
		1925	if (intel_crtc->config.pch_pfit.enabled) {
		1926	uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
		1927	uint32_t pfit_size = intel_crtc->config.pch_pfit.size;
		1928
4560	Serge	1929	pipe_w = intel_crtc->config.pipe_src_w;
		1930	pipe_h = intel_crtc->config.pipe_src_h;
4104	Serge	1931	pfit_w = (pfit_size >> 16) & 0xFFFF;
		1932	pfit_h = pfit_size & 0xFFFF;
		1933	if (pipe_w < pfit_w)
		1934	pipe_w = pfit_w;
		1935	if (pipe_h < pfit_h)
		1936	pipe_h = pfit_h;
		1937
		1938	pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
		1939	pfit_w * pfit_h);
		1940	}
		1941
		1942	return pixel_rate;
		1943	}
		1944
		1945	/* latency must be in 0.1us units. */
		1946	static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		1947	uint32_t latency)
		1948	{
		1949	uint64_t ret;
		1950
		1951	if (WARN(latency == 0, "Latency value missing\n"))
		1952	return UINT_MAX;
		1953
		1954	ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
		1955	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
		1956
		1957	return ret;
		1958	}
		1959
		1960	/* latency must be in 0.1us units. */
		1961	static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		1962	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
		1963	uint32_t latency)
		1964	{
		1965	uint32_t ret;
		1966
		1967	if (WARN(latency == 0, "Latency value missing\n"))
		1968	return UINT_MAX;
		1969
		1970	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		1971	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		1972	ret = DIV_ROUND_UP(ret, 64) + 2;
		1973	return ret;
		1974	}
		1975
		1976	static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
		1977	uint8_t bytes_per_pixel)
		1978	{
		1979	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
		1980	}
		1981
5354	serge	1982	struct skl_pipe_wm_parameters {
		1983	bool active;
		1984	uint32_t pipe_htotal;
		1985	uint32_t pixel_rate; /* in KHz */
		1986	struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
		1987	struct intel_plane_wm_parameters cursor;
		1988	};
		1989
4560	Serge	1990	struct ilk_pipe_wm_parameters {
4104	Serge	1991	bool active;
		1992	uint32_t pipe_htotal;
		1993	uint32_t pixel_rate;
		1994	struct intel_plane_wm_parameters pri;
		1995	struct intel_plane_wm_parameters spr;
		1996	struct intel_plane_wm_parameters cur;
		1997	};
		1998
4560	Serge	1999	struct ilk_wm_maximums {
4104	Serge	2000	uint16_t pri;
		2001	uint16_t spr;
		2002	uint16_t cur;
		2003	uint16_t fbc;
		2004	};
		2005
		2006	/* used in computing the new watermarks state */
		2007	struct intel_wm_config {
		2008	unsigned int num_pipes_active;
		2009	bool sprites_enabled;
		2010	bool sprites_scaled;
		2011	};
		2012
		2013	/*
		2014	* For both WM_PIPE and WM_LP.
		2015	* mem_value must be in 0.1us units.
		2016	*/
4560	Serge	2017	static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	2018	uint32_t mem_value,
		2019	bool is_lp)
		2020	{
		2021	uint32_t method1, method2;
		2022
		2023	if (!params->active \|\| !params->pri.enabled)
		2024	return 0;
		2025
		2026	method1 = ilk_wm_method1(params->pixel_rate,
		2027	params->pri.bytes_per_pixel,
		2028	mem_value);
		2029
		2030	if (!is_lp)
		2031	return method1;
		2032
		2033	method2 = ilk_wm_method2(params->pixel_rate,
		2034	params->pipe_htotal,
		2035	params->pri.horiz_pixels,
		2036	params->pri.bytes_per_pixel,
		2037	mem_value);
		2038
		2039	return min(method1, method2);
		2040	}
		2041
		2042	/*
		2043	* For both WM_PIPE and WM_LP.
		2044	* mem_value must be in 0.1us units.
		2045	*/
4560	Serge	2046	static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	2047	uint32_t mem_value)
		2048	{
		2049	uint32_t method1, method2;
		2050
		2051	if (!params->active \|\| !params->spr.enabled)
		2052	return 0;
		2053
		2054	method1 = ilk_wm_method1(params->pixel_rate,
		2055	params->spr.bytes_per_pixel,
		2056	mem_value);
		2057	method2 = ilk_wm_method2(params->pixel_rate,
		2058	params->pipe_htotal,
		2059	params->spr.horiz_pixels,
		2060	params->spr.bytes_per_pixel,
		2061	mem_value);
		2062	return min(method1, method2);
		2063	}
		2064
		2065	/*
		2066	* For both WM_PIPE and WM_LP.
		2067	* mem_value must be in 0.1us units.
		2068	*/
4560	Serge	2069	static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	2070	uint32_t mem_value)
		2071	{
		2072	if (!params->active \|\| !params->cur.enabled)
		2073	return 0;
		2074
		2075	return ilk_wm_method2(params->pixel_rate,
		2076	params->pipe_htotal,
		2077	params->cur.horiz_pixels,
		2078	params->cur.bytes_per_pixel,
		2079	mem_value);
		2080	}
		2081
		2082	/* Only for WM_LP. */
4560	Serge	2083	static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	2084	uint32_t pri_val)
		2085	{
		2086	if (!params->active \|\| !params->pri.enabled)
		2087	return 0;
		2088
		2089	return ilk_wm_fbc(pri_val,
		2090	params->pri.horiz_pixels,
		2091	params->pri.bytes_per_pixel);
		2092	}
		2093
		2094	static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
		2095	{
4560	Serge	2096	if (INTEL_INFO(dev)->gen >= 8)
		2097	return 3072;
		2098	else if (INTEL_INFO(dev)->gen >= 7)
4104	Serge	2099	return 768;
		2100	else
		2101	return 512;
		2102	}
		2103
5060	serge	2104	static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
		2105	int level, bool is_sprite)
		2106	{
		2107	if (INTEL_INFO(dev)->gen >= 8)
		2108	/* BDW primary/sprite plane watermarks */
		2109	return level == 0 ? 255 : 2047;
		2110	else if (INTEL_INFO(dev)->gen >= 7)
		2111	/* IVB/HSW primary/sprite plane watermarks */
		2112	return level == 0 ? 127 : 1023;
		2113	else if (!is_sprite)
		2114	/* ILK/SNB primary plane watermarks */
		2115	return level == 0 ? 127 : 511;
		2116	else
		2117	/* ILK/SNB sprite plane watermarks */
		2118	return level == 0 ? 63 : 255;
		2119	}
		2120
		2121	static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
		2122	int level)
		2123	{
		2124	if (INTEL_INFO(dev)->gen >= 7)
		2125	return level == 0 ? 63 : 255;
		2126	else
		2127	return level == 0 ? 31 : 63;
		2128	}
		2129
		2130	static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
		2131	{
		2132	if (INTEL_INFO(dev)->gen >= 8)
		2133	return 31;
		2134	else
		2135	return 15;
		2136	}
		2137
4104	Serge	2138	/* Calculate the maximum primary/sprite plane watermark */
		2139	static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
		2140	int level,
		2141	const struct intel_wm_config *config,
		2142	enum intel_ddb_partitioning ddb_partitioning,
		2143	bool is_sprite)
		2144	{
		2145	unsigned int fifo_size = ilk_display_fifo_size(dev);
		2146
		2147	/* if sprites aren't enabled, sprites get nothing */
		2148	if (is_sprite && !config->sprites_enabled)
		2149	return 0;
		2150
		2151	/* HSW allows LP1+ watermarks even with multiple pipes */
		2152	if (level == 0 \|\| config->num_pipes_active > 1) {
		2153	fifo_size /= INTEL_INFO(dev)->num_pipes;
		2154
		2155	/*
		2156	* For some reason the non self refresh
		2157	* FIFO size is only half of the self
		2158	* refresh FIFO size on ILK/SNB.
		2159	*/
		2160	if (INTEL_INFO(dev)->gen <= 6)
		2161	fifo_size /= 2;
		2162	}
		2163
		2164	if (config->sprites_enabled) {
		2165	/* level 0 is always calculated with 1:1 split */
		2166	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
		2167	if (is_sprite)
		2168	fifo_size *= 5;
		2169	fifo_size /= 6;
		2170	} else {
		2171	fifo_size /= 2;
		2172	}
		2173	}
		2174
		2175	/* clamp to max that the registers can hold */
5060	serge	2176	return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
4104	Serge	2177	}
		2178
		2179	/* Calculate the maximum cursor plane watermark */
		2180	static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
		2181	int level,
		2182	const struct intel_wm_config *config)
		2183	{
		2184	/* HSW LP1+ watermarks w/ multiple pipes */
		2185	if (level > 0 && config->num_pipes_active > 1)
		2186	return 64;
		2187
		2188	/* otherwise just report max that registers can hold */
5060	serge	2189	return ilk_cursor_wm_reg_max(dev, level);
4539	Serge	2190	}
4104	Serge	2191
5060	serge	2192	static void ilk_compute_wm_maximums(const struct drm_device *dev,
4104	Serge	2193	int level,
		2194	const struct intel_wm_config *config,
		2195	enum intel_ddb_partitioning ddb_partitioning,
4560	Serge	2196	struct ilk_wm_maximums *max)
4104	Serge	2197	{
		2198	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
		2199	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
		2200	max->cur = ilk_cursor_wm_max(dev, level, config);
5060	serge	2201	max->fbc = ilk_fbc_wm_reg_max(dev);
4539	Serge	2202	}
4104	Serge	2203
5060	serge	2204	static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
		2205	int level,
		2206	struct ilk_wm_maximums *max)
		2207	{
		2208	max->pri = ilk_plane_wm_reg_max(dev, level, false);
		2209	max->spr = ilk_plane_wm_reg_max(dev, level, true);
		2210	max->cur = ilk_cursor_wm_reg_max(dev, level);
		2211	max->fbc = ilk_fbc_wm_reg_max(dev);
		2212	}
		2213
4560	Serge	2214	static bool ilk_validate_wm_level(int level,
		2215	const struct ilk_wm_maximums *max,
4104	Serge	2216	struct intel_wm_level *result)
		2217	{
		2218	bool ret;
		2219
		2220	/* already determined to be invalid? */
		2221	if (!result->enable)
		2222	return false;
		2223
		2224	result->enable = result->pri_val <= max->pri &&
		2225	result->spr_val <= max->spr &&
		2226	result->cur_val <= max->cur;
		2227
		2228	ret = result->enable;
		2229
		2230	/*
		2231	* HACK until we can pre-compute everything,
		2232	* and thus fail gracefully if LP0 watermarks
		2233	* are exceeded...
		2234	*/
		2235	if (level == 0 && !result->enable) {
		2236	if (result->pri_val > max->pri)
		2237	DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
		2238	level, result->pri_val, max->pri);
		2239	if (result->spr_val > max->spr)
		2240	DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
		2241	level, result->spr_val, max->spr);
		2242	if (result->cur_val > max->cur)
		2243	DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
		2244	level, result->cur_val, max->cur);
		2245
		2246	result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
		2247	result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
		2248	result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
		2249	result->enable = true;
		2250	}
		2251
		2252	return ret;
		2253	}
		2254
5060	serge	2255	static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
4104	Serge	2256	int level,
4560	Serge	2257	const struct ilk_pipe_wm_parameters *p,
4104	Serge	2258	struct intel_wm_level *result)
		2259	{
		2260	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
		2261	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
		2262	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
		2263
		2264	/* WM1+ latency values stored in 0.5us units */
		2265	if (level > 0) {
		2266	pri_latency *= 5;
		2267	spr_latency *= 5;
		2268	cur_latency *= 5;
		2269	}
		2270
		2271	result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
		2272	result->spr_val = ilk_compute_spr_wm(p, spr_latency);
		2273	result->cur_val = ilk_compute_cur_wm(p, cur_latency);
		2274	result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
		2275	result->enable = true;
		2276	}
		2277
		2278	static uint32_t
		2279	hsw_compute_linetime_wm(struct drm_device dev, struct drm_crtc crtc)
		2280	{
3031	serge	2281	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2282	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2283	struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
		2284	u32 linetime, ips_linetime;
3031	serge	2285
4104	Serge	2286	if (!intel_crtc_active(crtc))
		2287	return 0;
3031	serge	2288
		2289	/* The WM are computed with base on how long it takes to fill a single
		2290	* row at the given clock rate, multiplied by 8.
		2291	* */
4560	Serge	2292	linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
		2293	mode->crtc_clock);
		2294	ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
4104	Serge	2295	intel_ddi_get_cdclk_freq(dev_priv));
3031	serge	2296
4104	Serge	2297	return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) \|
		2298	PIPE_WM_LINETIME_TIME(linetime);
		2299	}
		2300
5354	serge	2301	static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
4104	Serge	2302	{
		2303	struct drm_i915_private *dev_priv = dev->dev_private;
		2304
5354	serge	2305	if (IS_GEN9(dev)) {
		2306	uint32_t val;
		2307	int ret, i;
		2308	int level, max_level = ilk_wm_max_level(dev);
		2309
		2310	/* read the first set of memory latencies[0:3] */
		2311	val = 0; /* data0 to be programmed to 0 for first set */
		2312	mutex_lock(&dev_priv->rps.hw_lock);
		2313	ret = sandybridge_pcode_read(dev_priv,
		2314	GEN9_PCODE_READ_MEM_LATENCY,
		2315	&val);
		2316	mutex_unlock(&dev_priv->rps.hw_lock);
		2317
		2318	if (ret) {
		2319	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2320	return;
		2321	}
		2322
		2323	wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2324	wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2325	GEN9_MEM_LATENCY_LEVEL_MASK;
		2326	wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2327	GEN9_MEM_LATENCY_LEVEL_MASK;
		2328	wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2329	GEN9_MEM_LATENCY_LEVEL_MASK;
		2330
		2331	/* read the second set of memory latencies[4:7] */
		2332	val = 1; /* data0 to be programmed to 1 for second set */
		2333	mutex_lock(&dev_priv->rps.hw_lock);
		2334	ret = sandybridge_pcode_read(dev_priv,
		2335	GEN9_PCODE_READ_MEM_LATENCY,
		2336	&val);
		2337	mutex_unlock(&dev_priv->rps.hw_lock);
		2338	if (ret) {
		2339	DRM_ERROR("SKL Mailbox read error = %d\n", ret);
		2340	return;
		2341	}
		2342
		2343	wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
		2344	wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
		2345	GEN9_MEM_LATENCY_LEVEL_MASK;
		2346	wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
		2347	GEN9_MEM_LATENCY_LEVEL_MASK;
		2348	wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
		2349	GEN9_MEM_LATENCY_LEVEL_MASK;
		2350
		2351	/*
		2352	* punit doesn't take into account the read latency so we need
		2353	* to add 2us to the various latency levels we retrieve from
		2354	* the punit.
		2355	* - W0 is a bit special in that it's the only level that
		2356	* can't be disabled if we want to have display working, so
		2357	* we always add 2us there.
		2358	* - For levels >=1, punit returns 0us latency when they are
		2359	* disabled, so we respect that and don't add 2us then
		2360	*
		2361	* Additionally, if a level n (n > 1) has a 0us latency, all
		2362	* levels m (m >= n) need to be disabled. We make sure to
		2363	* sanitize the values out of the punit to satisfy this
		2364	* requirement.
		2365	*/
		2366	wm[0] += 2;
		2367	for (level = 1; level <= max_level; level++)
		2368	if (wm[level] != 0)
		2369	wm[level] += 2;
		2370	else {
		2371	for (i = level + 1; i <= max_level; i++)
		2372	wm[i] = 0;
		2373
		2374	break;
		2375	}
		2376	} else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2377	uint64_t sskpd = I915_READ64(MCH_SSKPD);
		2378
		2379	wm[0] = (sskpd >> 56) & 0xFF;
		2380	if (wm[0] == 0)
		2381	wm[0] = sskpd & 0xF;
		2382	wm[1] = (sskpd >> 4) & 0xFF;
		2383	wm[2] = (sskpd >> 12) & 0xFF;
		2384	wm[3] = (sskpd >> 20) & 0x1FF;
		2385	wm[4] = (sskpd >> 32) & 0x1FF;
		2386	} else if (INTEL_INFO(dev)->gen >= 6) {
		2387	uint32_t sskpd = I915_READ(MCH_SSKPD);
		2388
		2389	wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		2390	wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		2391	wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		2392	wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
		2393	} else if (INTEL_INFO(dev)->gen >= 5) {
		2394	uint32_t mltr = I915_READ(MLTR_ILK);
		2395
		2396	/* ILK primary LP0 latency is 700 ns */
		2397	wm[0] = 7;
		2398	wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		2399	wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
		2400	}
		2401	}
		2402
		2403	static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2404	{
		2405	/* ILK sprite LP0 latency is 1300 ns */
		2406	if (INTEL_INFO(dev)->gen == 5)
		2407	wm[0] = 13;
		2408	}
		2409
		2410	static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2411	{
		2412	/* ILK cursor LP0 latency is 1300 ns */
		2413	if (INTEL_INFO(dev)->gen == 5)
		2414	wm[0] = 13;
		2415
		2416	/* WaDoubleCursorLP3Latency:ivb */
		2417	if (IS_IVYBRIDGE(dev))
		2418	wm[3] *= 2;
		2419	}
		2420
5060	serge	2421	int ilk_wm_max_level(const struct drm_device *dev)
4560	Serge	2422	{
		2423	/* how many WM levels are we expecting */
5354	serge	2424	if (IS_GEN9(dev))
		2425	return 7;
		2426	else if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
4560	Serge	2427	return 4;
		2428	else if (INTEL_INFO(dev)->gen >= 6)
		2429	return 3;
		2430	else
		2431	return 2;
		2432	}
		2433
4104	Serge	2434	static void intel_print_wm_latency(struct drm_device *dev,
		2435	const char *name,
5354	serge	2436	const uint16_t wm[8])
4104	Serge	2437	{
4560	Serge	2438	int level, max_level = ilk_wm_max_level(dev);
4104	Serge	2439
		2440	for (level = 0; level <= max_level; level++) {
		2441	unsigned int latency = wm[level];
		2442
		2443	if (latency == 0) {
		2444	DRM_ERROR("%s WM%d latency not provided\n",
		2445	name, level);
		2446	continue;
		2447	}
		2448
5354	serge	2449	/*
		2450	* - latencies are in us on gen9.
		2451	* - before then, WM1+ latency values are in 0.5us units
		2452	*/
		2453	if (IS_GEN9(dev))
		2454	latency *= 10;
		2455	else if (level > 0)
4104	Serge	2456	latency *= 5;
		2457
		2458	DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
		2459	name, level, wm[level],
		2460	latency / 10, latency % 10);
		2461	}
		2462	}
		2463
5060	serge	2464	static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
		2465	uint16_t wm[5], uint16_t min)
4104	Serge	2466	{
5060	serge	2467	int level, max_level = ilk_wm_max_level(dev_priv->dev);
		2468
		2469	if (wm[0] >= min)
		2470	return false;
		2471
		2472	wm[0] = max(wm[0], min);
		2473	for (level = 1; level <= max_level; level++)
		2474	wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
		2475
		2476	return true;
		2477	}
		2478
		2479	static void snb_wm_latency_quirk(struct drm_device *dev)
		2480	{
4104	Serge	2481	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2482	bool changed;
4104	Serge	2483
5060	serge	2484	/*
		2485	* The BIOS provided WM memory latency values are often
		2486	* inadequate for high resolution displays. Adjust them.
		2487	*/
		2488	changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) \|
		2489	ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) \|
		2490	ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
		2491
		2492	if (!changed)
		2493	return;
		2494
		2495	DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
		2496	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2497	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2498	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
		2499	}
		2500
		2501	static void ilk_setup_wm_latency(struct drm_device *dev)
		2502	{
		2503	struct drm_i915_private *dev_priv = dev->dev_private;
		2504
4104	Serge	2505	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
		2506
		2507	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
		2508	sizeof(dev_priv->wm.pri_latency));
		2509	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
		2510	sizeof(dev_priv->wm.pri_latency));
		2511
		2512	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
		2513	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
		2514
		2515	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2516	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2517	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
5060	serge	2518
		2519	if (IS_GEN6(dev))
		2520	snb_wm_latency_quirk(dev);
4104	Serge	2521	}
		2522
5354	serge	2523	static void skl_setup_wm_latency(struct drm_device *dev)
		2524	{
		2525	struct drm_i915_private *dev_priv = dev->dev_private;
		2526
		2527	intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
		2528	intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
		2529	}
		2530
4560	Serge	2531	static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
5060	serge	2532	struct ilk_pipe_wm_parameters *p)
4104	Serge	2533	{
4560	Serge	2534	struct drm_device *dev = crtc->dev;
		2535	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2536	enum pipe pipe = intel_crtc->pipe;
4104	Serge	2537	struct drm_plane *plane;
		2538
5060	serge	2539	if (!intel_crtc_active(crtc))
		2540	return;
		2541
		2542	p->active = true;
4560	Serge	2543	p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal;
4104	Serge	2544	p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
5060	serge	2545	p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8;
4104	Serge	2546	p->cur.bytes_per_pixel = 4;
4560	Serge	2547	p->pri.horiz_pixels = intel_crtc->config.pipe_src_w;
5060	serge	2548	p->cur.horiz_pixels = intel_crtc->cursor_width;
4104	Serge	2549	/* TODO: for now, assume primary and cursor planes are always enabled. */
		2550	p->pri.enabled = true;
		2551	p->cur.enabled = true;
		2552
5060	serge	2553	drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
4104	Serge	2554	struct intel_plane *intel_plane = to_intel_plane(plane);
		2555
5060	serge	2556	if (intel_plane->pipe == pipe) {
4104	Serge	2557	p->spr = intel_plane->wm;
5060	serge	2558	break;
		2559	}
		2560	}
		2561	}
4104	Serge	2562
5060	serge	2563	static void ilk_compute_wm_config(struct drm_device *dev,
		2564	struct intel_wm_config *config)
		2565	{
		2566	struct intel_crtc *intel_crtc;
		2567
		2568	/* Compute the currently _active_ config */
		2569	for_each_intel_crtc(dev, intel_crtc) {
		2570	const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
		2571
		2572	if (!wm->pipe_enabled)
		2573	continue;
		2574
		2575	config->sprites_enabled \|= wm->sprites_enabled;
		2576	config->sprites_scaled \|= wm->sprites_scaled;
		2577	config->num_pipes_active++;
4104	Serge	2578	}
4560	Serge	2579	}
4104	Serge	2580
4560	Serge	2581	/* Compute new watermarks for the pipe */
		2582	static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
		2583	const struct ilk_pipe_wm_parameters *params,
		2584	struct intel_pipe_wm *pipe_wm)
		2585	{
		2586	struct drm_device *dev = crtc->dev;
5060	serge	2587	const struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2588	int level, max_level = ilk_wm_max_level(dev);
		2589	/* LP0 watermark maximums depend on this pipe alone */
		2590	struct intel_wm_config config = {
		2591	.num_pipes_active = 1,
		2592	.sprites_enabled = params->spr.enabled,
		2593	.sprites_scaled = params->spr.scaled,
		2594	};
		2595	struct ilk_wm_maximums max;
4104	Serge	2596
5060	serge	2597	pipe_wm->pipe_enabled = params->active;
		2598	pipe_wm->sprites_enabled = params->spr.enabled;
		2599	pipe_wm->sprites_scaled = params->spr.scaled;
4560	Serge	2600
		2601	/* ILK/SNB: LP2+ watermarks only w/o sprites */
		2602	if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
		2603	max_level = 1;
		2604
		2605	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
		2606	if (params->spr.scaled)
		2607	max_level = 0;
		2608
5060	serge	2609	ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]);
4560	Serge	2610
		2611	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2612	pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
		2613
5060	serge	2614	/* LP0 watermarks always use 1/2 DDB partitioning */
		2615	ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
		2616
4560	Serge	2617	/* At least LP0 must be valid */
5060	serge	2618	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
		2619	return false;
		2620
		2621	ilk_compute_wm_reg_maximums(dev, 1, &max);
		2622
		2623	for (level = 1; level <= max_level; level++) {
		2624	struct intel_wm_level wm = {};
		2625
		2626	ilk_compute_wm_level(dev_priv, level, params, &wm);
		2627
		2628	/*
		2629	* Disable any watermark level that exceeds the
		2630	* register maximums since such watermarks are
		2631	* always invalid.
		2632	*/
		2633	if (!ilk_validate_wm_level(level, &max, &wm))
		2634	break;
		2635
		2636	pipe_wm->wm[level] = wm;
		2637	}
		2638
		2639	return true;
4104	Serge	2640	}
		2641
4560	Serge	2642	/*
		2643	* Merge the watermarks from all active pipes for a specific level.
		2644	*/
		2645	static void ilk_merge_wm_level(struct drm_device *dev,
		2646	int level,
		2647	struct intel_wm_level *ret_wm)
4104	Serge	2648	{
4560	Serge	2649	const struct intel_crtc *intel_crtc;
4104	Serge	2650
5060	serge	2651	ret_wm->enable = true;
4104	Serge	2652
5060	serge	2653	for_each_intel_crtc(dev, intel_crtc) {
		2654	const struct intel_pipe_wm *active = &intel_crtc->wm.active;
		2655	const struct intel_wm_level *wm = &active->wm[level];
		2656
		2657	if (!active->pipe_enabled)
		2658	continue;
		2659
		2660	/*
		2661	* The watermark values may have been used in the past,
		2662	* so we must maintain them in the registers for some
		2663	* time even if the level is now disabled.
		2664	*/
4560	Serge	2665	if (!wm->enable)
5060	serge	2666	ret_wm->enable = false;
4104	Serge	2667
4560	Serge	2668	ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
		2669	ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
		2670	ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
		2671	ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
		2672	}
		2673	}
		2674
		2675	/*
		2676	* Merge all low power watermarks for all active pipes.
		2677	*/
		2678	static void ilk_wm_merge(struct drm_device *dev,
		2679	const struct intel_wm_config *config,
		2680	const struct ilk_wm_maximums *max,
		2681	struct intel_pipe_wm *merged)
		2682	{
		2683	int level, max_level = ilk_wm_max_level(dev);
5060	serge	2684	int last_enabled_level = max_level;
4560	Serge	2685
		2686	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
		2687	if ((INTEL_INFO(dev)->gen <= 6 \|\| IS_IVYBRIDGE(dev)) &&
		2688	config->num_pipes_active > 1)
		2689	return;
		2690
		2691	/* ILK: FBC WM must be disabled always */
		2692	merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
		2693
		2694	/* merge each WM1+ level */
4104	Serge	2695	for (level = 1; level <= max_level; level++) {
4560	Serge	2696	struct intel_wm_level *wm = &merged->wm[level];
		2697
		2698	ilk_merge_wm_level(dev, level, wm);
		2699
5060	serge	2700	if (level > last_enabled_level)
		2701	wm->enable = false;
		2702	else if (!ilk_validate_wm_level(level, max, wm))
		2703	/* make sure all following levels get disabled */
		2704	last_enabled_level = level - 1;
4560	Serge	2705
		2706	/*
		2707	* The spec says it is preferred to disable
		2708	* FBC WMs instead of disabling a WM level.
		2709	*/
		2710	if (wm->fbc_val > max->fbc) {
5060	serge	2711	if (wm->enable)
4560	Serge	2712	merged->fbc_wm_enabled = false;
		2713	wm->fbc_val = 0;
4104	Serge	2714	}
		2715	}
		2716
4560	Serge	2717	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
		2718	/*
		2719	* FIXME this is racy. FBC might get enabled later.
		2720	* What we should check here is whether FBC can be
		2721	* enabled sometime later.
		2722	*/
		2723	if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
		2724	for (level = 2; level <= max_level; level++) {
		2725	struct intel_wm_level *wm = &merged->wm[level];
		2726
		2727	wm->enable = false;
		2728	}
		2729	}
		2730	}
		2731
		2732	static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
		2733	{
		2734	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
		2735	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
		2736	}
		2737
		2738	/* The value we need to program into the WM_LPx latency field */
		2739	static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
		2740	{
		2741	struct drm_i915_private *dev_priv = dev->dev_private;
		2742
		2743	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2744	return 2 * level;
		2745	else
		2746	return dev_priv->wm.pri_latency[level];
		2747	}
		2748
		2749	static void ilk_compute_wm_results(struct drm_device *dev,
		2750	const struct intel_pipe_wm *merged,
		2751	enum intel_ddb_partitioning partitioning,
		2752	struct ilk_wm_values *results)
		2753	{
		2754	struct intel_crtc *intel_crtc;
		2755	int level, wm_lp;
		2756
		2757	results->enable_fbc_wm = merged->fbc_wm_enabled;
		2758	results->partitioning = partitioning;
		2759
		2760	/* LP1+ register values */
4104	Serge	2761	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2762	const struct intel_wm_level *r;
		2763
4560	Serge	2764	level = ilk_wm_lp_to_level(wm_lp, merged);
		2765
		2766	r = &merged->wm[level];
4104	Serge	2767
5060	serge	2768	/*
		2769	* Maintain the watermark values even if the level is
		2770	* disabled. Doing otherwise could cause underruns.
		2771	*/
		2772	results->wm_lp[wm_lp - 1] =
4560	Serge	2773	(ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) \|
		2774	(r->pri_val << WM1_LP_SR_SHIFT) \|
		2775	r->cur_val;
		2776
5060	serge	2777	if (r->enable)
		2778	results->wm_lp[wm_lp - 1] \|= WM1_LP_SR_EN;
		2779
4560	Serge	2780	if (INTEL_INFO(dev)->gen >= 8)
		2781	results->wm_lp[wm_lp - 1] \|=
		2782	r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
		2783	else
		2784	results->wm_lp[wm_lp - 1] \|=
		2785	r->fbc_val << WM1_LP_FBC_SHIFT;
		2786
5060	serge	2787	/*
		2788	* Always set WM1S_LP_EN when spr_val != 0, even if the
		2789	* level is disabled. Doing otherwise could cause underruns.
		2790	*/
4560	Serge	2791	if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
		2792	WARN_ON(wm_lp != 1);
		2793	results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN \| r->spr_val;
		2794	} else
4104	Serge	2795	results->wm_lp_spr[wm_lp - 1] = r->spr_val;
		2796	}
		2797
4560	Serge	2798	/* LP0 register values */
5060	serge	2799	for_each_intel_crtc(dev, intel_crtc) {
4560	Serge	2800	enum pipe pipe = intel_crtc->pipe;
		2801	const struct intel_wm_level *r =
		2802	&intel_crtc->wm.active.wm[0];
4104	Serge	2803
4560	Serge	2804	if (WARN_ON(!r->enable))
		2805	continue;
		2806
		2807	results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
		2808
		2809	results->wm_pipe[pipe] =
		2810	(r->pri_val << WM0_PIPE_PLANE_SHIFT) \|
		2811	(r->spr_val << WM0_PIPE_SPRITE_SHIFT) \|
		2812	r->cur_val;
4104	Serge	2813	}
		2814	}
		2815
		2816	/* Find the result with the highest level enabled. Check for enable_fbc_wm in
		2817	* case both are at the same level. Prefer r1 in case they're the same. */
4560	Serge	2818	static struct intel_pipe_wm ilk_find_best_result(struct drm_device dev,
		2819	struct intel_pipe_wm *r1,
		2820	struct intel_pipe_wm *r2)
4104	Serge	2821	{
4560	Serge	2822	int level, max_level = ilk_wm_max_level(dev);
		2823	int level1 = 0, level2 = 0;
4104	Serge	2824
4560	Serge	2825	for (level = 1; level <= max_level; level++) {
		2826	if (r1->wm[level].enable)
		2827	level1 = level;
		2828	if (r2->wm[level].enable)
		2829	level2 = level;
4104	Serge	2830	}
		2831
4560	Serge	2832	if (level1 == level2) {
		2833	if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
4104	Serge	2834	return r2;
		2835	else
		2836	return r1;
4560	Serge	2837	} else if (level1 > level2) {
4104	Serge	2838	return r1;
		2839	} else {
		2840	return r2;
		2841	}
		2842	}
		2843
4560	Serge	2844	/* dirty bits used to track which watermarks need changes */
		2845	#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
		2846	#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
		2847	#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
		2848	#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) \| WM_DIRTY_LP(2) \| WM_DIRTY_LP(3))
		2849	#define WM_DIRTY_FBC (1 << 24)
		2850	#define WM_DIRTY_DDB (1 << 25)
		2851
5354	serge	2852	static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
4560	Serge	2853	const struct ilk_wm_values *old,
		2854	const struct ilk_wm_values *new)
		2855	{
		2856	unsigned int dirty = 0;
		2857	enum pipe pipe;
		2858	int wm_lp;
		2859
5354	serge	2860	for_each_pipe(dev_priv, pipe) {
4560	Serge	2861	if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
		2862	dirty \|= WM_DIRTY_LINETIME(pipe);
		2863	/* Must disable LP1+ watermarks too */
		2864	dirty \|= WM_DIRTY_LP_ALL;
		2865	}
		2866
		2867	if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
		2868	dirty \|= WM_DIRTY_PIPE(pipe);
		2869	/* Must disable LP1+ watermarks too */
		2870	dirty \|= WM_DIRTY_LP_ALL;
		2871	}
		2872	}
		2873
		2874	if (old->enable_fbc_wm != new->enable_fbc_wm) {
		2875	dirty \|= WM_DIRTY_FBC;
		2876	/* Must disable LP1+ watermarks too */
		2877	dirty \|= WM_DIRTY_LP_ALL;
		2878	}
		2879
		2880	if (old->partitioning != new->partitioning) {
		2881	dirty \|= WM_DIRTY_DDB;
		2882	/* Must disable LP1+ watermarks too */
		2883	dirty \|= WM_DIRTY_LP_ALL;
		2884	}
		2885
		2886	/* LP1+ watermarks already deemed dirty, no need to continue */
		2887	if (dirty & WM_DIRTY_LP_ALL)
		2888	return dirty;
		2889
		2890	/* Find the lowest numbered LP1+ watermark in need of an update... */
		2891	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2892	if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] \|\|
		2893	old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
		2894	break;
		2895	}
		2896
		2897	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
		2898	for (; wm_lp <= 3; wm_lp++)
		2899	dirty \|= WM_DIRTY_LP(wm_lp);
		2900
		2901	return dirty;
		2902	}
		2903
		2904	static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
		2905	unsigned int dirty)
		2906	{
		2907	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2908	bool changed = false;
		2909
		2910	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
		2911	previous->wm_lp[2] &= ~WM1_LP_SR_EN;
		2912	I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
		2913	changed = true;
		2914	}
		2915	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
		2916	previous->wm_lp[1] &= ~WM1_LP_SR_EN;
		2917	I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
		2918	changed = true;
		2919	}
		2920	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
		2921	previous->wm_lp[0] &= ~WM1_LP_SR_EN;
		2922	I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
		2923	changed = true;
		2924	}
		2925
		2926	/*
		2927	* Don't touch WM1S_LP_EN here.
		2928	* Doing so could cause underruns.
		2929	*/
		2930
		2931	return changed;
		2932	}
		2933
4104	Serge	2934	/*
		2935	* The spec says we shouldn't write when we don't need, because every write
		2936	* causes WMs to be re-evaluated, expending some power.
3031	serge	2937	*/
4560	Serge	2938	static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
		2939	struct ilk_wm_values *results)
4104	Serge	2940	{
4560	Serge	2941	struct drm_device *dev = dev_priv->dev;
		2942	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2943	unsigned int dirty;
4104	Serge	2944	uint32_t val;
3031	serge	2945
5354	serge	2946	dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
4560	Serge	2947	if (!dirty)
4104	Serge	2948	return;
		2949
4560	Serge	2950	_ilk_disable_lp_wm(dev_priv, dirty);
4104	Serge	2951
4560	Serge	2952	if (dirty & WM_DIRTY_PIPE(PIPE_A))
4104	Serge	2953	I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
4560	Serge	2954	if (dirty & WM_DIRTY_PIPE(PIPE_B))
4104	Serge	2955	I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
4560	Serge	2956	if (dirty & WM_DIRTY_PIPE(PIPE_C))
4104	Serge	2957	I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
		2958
4560	Serge	2959	if (dirty & WM_DIRTY_LINETIME(PIPE_A))
4104	Serge	2960	I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
4560	Serge	2961	if (dirty & WM_DIRTY_LINETIME(PIPE_B))
4104	Serge	2962	I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
4560	Serge	2963	if (dirty & WM_DIRTY_LINETIME(PIPE_C))
4104	Serge	2964	I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
		2965
4560	Serge	2966	if (dirty & WM_DIRTY_DDB) {
		2967	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2968	val = I915_READ(WM_MISC);
4560	Serge	2969	if (results->partitioning == INTEL_DDB_PART_1_2)
4104	Serge	2970	val &= ~WM_MISC_DATA_PARTITION_5_6;
		2971	else
		2972	val \|= WM_MISC_DATA_PARTITION_5_6;
		2973	I915_WRITE(WM_MISC, val);
4560	Serge	2974	} else {
		2975	val = I915_READ(DISP_ARB_CTL2);
		2976	if (results->partitioning == INTEL_DDB_PART_1_2)
		2977	val &= ~DISP_DATA_PARTITION_5_6;
		2978	else
		2979	val \|= DISP_DATA_PARTITION_5_6;
		2980	I915_WRITE(DISP_ARB_CTL2, val);
		2981	}
4104	Serge	2982	}
		2983
4560	Serge	2984	if (dirty & WM_DIRTY_FBC) {
4104	Serge	2985	val = I915_READ(DISP_ARB_CTL);
		2986	if (results->enable_fbc_wm)
		2987	val &= ~DISP_FBC_WM_DIS;
		2988	else
		2989	val \|= DISP_FBC_WM_DIS;
		2990	I915_WRITE(DISP_ARB_CTL, val);
		2991	}
		2992
4560	Serge	2993	if (dirty & WM_DIRTY_LP(1) &&
		2994	previous->wm_lp_spr[0] != results->wm_lp_spr[0])
4104	Serge	2995	I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
4560	Serge	2996
		2997	if (INTEL_INFO(dev)->gen >= 7) {
		2998	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
4104	Serge	2999	I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
4560	Serge	3000	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
4104	Serge	3001	I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
4560	Serge	3002	}
4104	Serge	3003
4560	Serge	3004	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
4104	Serge	3005	I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
4560	Serge	3006	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
4104	Serge	3007	I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
4560	Serge	3008	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
4104	Serge	3009	I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
4560	Serge	3010
		3011	dev_priv->wm.hw = *results;
3031	serge	3012	}
		3013
4560	Serge	3014	static bool ilk_disable_lp_wm(struct drm_device *dev)
4104	Serge	3015	{
		3016	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	3017
		3018	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
		3019	}
		3020
5354	serge	3021	/*
		3022	* On gen9, we need to allocate Display Data Buffer (DDB) portions to the
		3023	* different active planes.
		3024	*/
		3025
		3026	#define SKL_DDB_SIZE 896 /* in blocks */
		3027
		3028	static void
		3029	skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
		3030	struct drm_crtc *for_crtc,
		3031	const struct intel_wm_config *config,
		3032	const struct skl_pipe_wm_parameters *params,
		3033	struct skl_ddb_entry alloc / out */)
		3034	{
		3035	struct drm_crtc *crtc;
		3036	unsigned int pipe_size, ddb_size;
		3037	int nth_active_pipe;
		3038
		3039	if (!params->active) {
		3040	alloc->start = 0;
		3041	alloc->end = 0;
		3042	return;
		3043	}
		3044
		3045	ddb_size = SKL_DDB_SIZE;
		3046
		3047	ddb_size -= 4; /* 4 blocks for bypass path allocation */
		3048
		3049	nth_active_pipe = 0;
		3050	for_each_crtc(dev, crtc) {
		3051	if (!intel_crtc_active(crtc))
		3052	continue;
		3053
		3054	if (crtc == for_crtc)
		3055	break;
		3056
		3057	nth_active_pipe++;
		3058	}
		3059
		3060	pipe_size = ddb_size / config->num_pipes_active;
		3061	alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
		3062	alloc->end = alloc->start + pipe_size;
		3063	}
		3064
		3065	static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
		3066	{
		3067	if (config->num_pipes_active == 1)
		3068	return 32;
		3069
		3070	return 8;
		3071	}
		3072
		3073	static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
		3074	{
		3075	entry->start = reg & 0x3ff;
		3076	entry->end = (reg >> 16) & 0x3ff;
		3077	if (entry->end)
		3078	entry->end += 1;
		3079	}
		3080
		3081	void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
		3082	struct skl_ddb_allocation ddb / out */)
		3083	{
		3084	struct drm_device *dev = dev_priv->dev;
		3085	enum pipe pipe;
		3086	int plane;
		3087	u32 val;
		3088
		3089	for_each_pipe(dev_priv, pipe) {
		3090	for_each_plane(pipe, plane) {
		3091	val = I915_READ(PLANE_BUF_CFG(pipe, plane));
		3092	skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
		3093	val);
		3094	}
		3095
		3096	val = I915_READ(CUR_BUF_CFG(pipe));
		3097	skl_ddb_entry_init_from_hw(&ddb->cursor[pipe], val);
		3098	}
		3099	}
		3100
		3101	static unsigned int
		3102	skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p)
		3103	{
		3104	return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
		3105	}
		3106
		3107	/*
		3108	* We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
		3109	* a 8192x4096@32bpp framebuffer:
		3110	* 3 * 4096 * 8192 * 4 < 2^32
		3111	*/
		3112	static unsigned int
		3113	skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
		3114	const struct skl_pipe_wm_parameters *params)
		3115	{
		3116	unsigned int total_data_rate = 0;
		3117	int plane;
		3118
		3119	for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
		3120	const struct intel_plane_wm_parameters *p;
		3121
		3122	p = ¶ms->plane[plane];
		3123	if (!p->enabled)
		3124	continue;
		3125
		3126	total_data_rate += skl_plane_relative_data_rate(p);
		3127	}
		3128
		3129	return total_data_rate;
		3130	}
		3131
		3132	static void
		3133	skl_allocate_pipe_ddb(struct drm_crtc *crtc,
		3134	const struct intel_wm_config *config,
		3135	const struct skl_pipe_wm_parameters *params,
		3136	struct skl_ddb_allocation ddb / out */)
		3137	{
		3138	struct drm_device *dev = crtc->dev;
		3139	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3140	enum pipe pipe = intel_crtc->pipe;
		3141	struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
		3142	uint16_t alloc_size, start, cursor_blocks;
		3143	unsigned int total_data_rate;
		3144	int plane;
		3145
		3146	skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
		3147	alloc_size = skl_ddb_entry_size(alloc);
		3148	if (alloc_size == 0) {
		3149	memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
		3150	memset(&ddb->cursor[pipe], 0, sizeof(ddb->cursor[pipe]));
		3151	return;
		3152	}
		3153
		3154	cursor_blocks = skl_cursor_allocation(config);
		3155	ddb->cursor[pipe].start = alloc->end - cursor_blocks;
		3156	ddb->cursor[pipe].end = alloc->end;
		3157
		3158	alloc_size -= cursor_blocks;
		3159	alloc->end -= cursor_blocks;
		3160
		3161	/*
		3162	* Each active plane get a portion of the remaining space, in
		3163	* proportion to the amount of data they need to fetch from memory.
		3164	*
		3165	* FIXME: we may not allocate every single block here.
		3166	*/
		3167	total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
		3168
		3169	start = alloc->start;
		3170	for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
		3171	const struct intel_plane_wm_parameters *p;
		3172	unsigned int data_rate;
		3173	uint16_t plane_blocks;
		3174
		3175	p = ¶ms->plane[plane];
		3176	if (!p->enabled)
		3177	continue;
		3178
		3179	data_rate = skl_plane_relative_data_rate(p);
		3180
		3181	/*
		3182	* promote the expression to 64 bits to avoid overflowing, the
		3183	* result is < available as data_rate / total_data_rate < 1
		3184	*/
		3185	plane_blocks = div_u64((uint64_t)alloc_size * data_rate,
		3186	total_data_rate);
		3187
		3188	ddb->plane[pipe][plane].start = start;
		3189	ddb->plane[pipe][plane].end = start + plane_blocks;
		3190
		3191	start += plane_blocks;
		3192	}
		3193
		3194	}
		3195
		3196	static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_config *config)
		3197	{
		3198	/* TODO: Take into account the scalers once we support them */
		3199	return config->adjusted_mode.crtc_clock;
		3200	}
		3201
		3202	/*
		3203	* The max latency should be 257 (max the punit can code is 255 and we add 2us
		3204	* for the read latency) and bytes_per_pixel should always be <= 8, so that
		3205	* should allow pixel_rate up to ~2 GHz which seems sufficient since max
		3206	* 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
		3207	*/
		3208	static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		3209	uint32_t latency)
		3210	{
		3211	uint32_t wm_intermediate_val, ret;
		3212
		3213	if (latency == 0)
		3214	return UINT_MAX;
		3215
		3216	wm_intermediate_val = latency * pixel_rate * bytes_per_pixel;
		3217	ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
		3218
		3219	return ret;
		3220	}
		3221
		3222	static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		3223	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
		3224	uint32_t latency)
		3225	{
		3226	uint32_t ret, plane_bytes_per_line, wm_intermediate_val;
		3227
		3228	if (latency == 0)
		3229	return UINT_MAX;
		3230
		3231	plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
		3232	wm_intermediate_val = latency * pixel_rate;
		3233	ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
		3234	plane_bytes_per_line;
		3235
		3236	return ret;
		3237	}
		3238
		3239	static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
		3240	const struct intel_crtc *intel_crtc)
		3241	{
		3242	struct drm_device *dev = intel_crtc->base.dev;
		3243	struct drm_i915_private *dev_priv = dev->dev_private;
		3244	const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3245	enum pipe pipe = intel_crtc->pipe;
		3246
		3247	if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
		3248	sizeof(new_ddb->plane[pipe])))
		3249	return true;
		3250
		3251	if (memcmp(&new_ddb->cursor[pipe], &cur_ddb->cursor[pipe],
		3252	sizeof(new_ddb->cursor[pipe])))
		3253	return true;
		3254
		3255	return false;
		3256	}
		3257
		3258	static void skl_compute_wm_global_parameters(struct drm_device *dev,
		3259	struct intel_wm_config *config)
		3260	{
		3261	struct drm_crtc *crtc;
		3262	struct drm_plane *plane;
		3263
		3264	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
		3265	config->num_pipes_active += intel_crtc_active(crtc);
		3266
		3267	/* FIXME: I don't think we need those two global parameters on SKL */
		3268	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		3269	struct intel_plane *intel_plane = to_intel_plane(plane);
		3270
		3271	config->sprites_enabled \|= intel_plane->wm.enabled;
		3272	config->sprites_scaled \|= intel_plane->wm.scaled;
		3273	}
		3274	}
		3275
		3276	static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
		3277	struct skl_pipe_wm_parameters *p)
		3278	{
		3279	struct drm_device *dev = crtc->dev;
		3280	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3281	enum pipe pipe = intel_crtc->pipe;
		3282	struct drm_plane *plane;
		3283	int i = 1; /* Index for sprite planes start */
		3284
		3285	p->active = intel_crtc_active(crtc);
		3286	if (p->active) {
		3287	p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal;
		3288	p->pixel_rate = skl_pipe_pixel_rate(&intel_crtc->config);
		3289
		3290	/*
		3291	* For now, assume primary and cursor planes are always enabled.
		3292	*/
		3293	p->plane[0].enabled = true;
		3294	p->plane[0].bytes_per_pixel =
		3295	crtc->primary->fb->bits_per_pixel / 8;
		3296	p->plane[0].horiz_pixels = intel_crtc->config.pipe_src_w;
		3297	p->plane[0].vert_pixels = intel_crtc->config.pipe_src_h;
		3298
		3299	p->cursor.enabled = true;
		3300	p->cursor.bytes_per_pixel = 4;
		3301	p->cursor.horiz_pixels = intel_crtc->cursor_width ?
		3302	intel_crtc->cursor_width : 64;
		3303	}
		3304
		3305	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		3306	struct intel_plane *intel_plane = to_intel_plane(plane);
		3307
		3308	if (intel_plane->pipe == pipe)
		3309	p->plane[i++] = intel_plane->wm;
		3310	}
		3311	}
		3312
		3313	static bool skl_compute_plane_wm(struct skl_pipe_wm_parameters *p,
		3314	struct intel_plane_wm_parameters *p_params,
		3315	uint16_t ddb_allocation,
		3316	uint32_t mem_value,
		3317	uint16_t out_blocks, / out */
		3318	uint8_t out_lines / out */)
		3319	{
		3320	uint32_t method1, method2, plane_bytes_per_line, res_blocks, res_lines;
		3321	uint32_t result_bytes;
		3322
		3323	if (mem_value == 0 \|\| !p->active \|\| !p_params->enabled)
		3324	return false;
		3325
		3326	method1 = skl_wm_method1(p->pixel_rate,
		3327	p_params->bytes_per_pixel,
		3328	mem_value);
		3329	method2 = skl_wm_method2(p->pixel_rate,
		3330	p->pipe_htotal,
		3331	p_params->horiz_pixels,
		3332	p_params->bytes_per_pixel,
		3333	mem_value);
		3334
		3335	plane_bytes_per_line = p_params->horiz_pixels *
		3336	p_params->bytes_per_pixel;
		3337
		3338	/* For now xtile and linear */
		3339	if (((ddb_allocation * 512) / plane_bytes_per_line) >= 1)
		3340	result_bytes = min(method1, method2);
		3341	else
		3342	result_bytes = method1;
		3343
		3344	res_blocks = DIV_ROUND_UP(result_bytes, 512) + 1;
		3345	res_lines = DIV_ROUND_UP(result_bytes, plane_bytes_per_line);
		3346
		3347	if (res_blocks > ddb_allocation \|\| res_lines > 31)
		3348	return false;
		3349
		3350	*out_blocks = res_blocks;
		3351	*out_lines = res_lines;
		3352
		3353	return true;
		3354	}
		3355
		3356	static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
		3357	struct skl_ddb_allocation *ddb,
		3358	struct skl_pipe_wm_parameters *p,
		3359	enum pipe pipe,
		3360	int level,
		3361	int num_planes,
		3362	struct skl_wm_level *result)
		3363	{
		3364	uint16_t latency = dev_priv->wm.skl_latency[level];
		3365	uint16_t ddb_blocks;
		3366	int i;
		3367
		3368	for (i = 0; i < num_planes; i++) {
		3369	ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
		3370
		3371	result->plane_en[i] = skl_compute_plane_wm(p, &p->plane[i],
		3372	ddb_blocks,
		3373	latency,
		3374	&result->plane_res_b[i],
		3375	&result->plane_res_l[i]);
		3376	}
		3377
		3378	ddb_blocks = skl_ddb_entry_size(&ddb->cursor[pipe]);
		3379	result->cursor_en = skl_compute_plane_wm(p, &p->cursor, ddb_blocks,
		3380	latency, &result->cursor_res_b,
		3381	&result->cursor_res_l);
		3382	}
		3383
		3384	static uint32_t
		3385	skl_compute_linetime_wm(struct drm_crtc crtc, struct skl_pipe_wm_parameters p)
		3386	{
		3387	if (!intel_crtc_active(crtc))
		3388	return 0;
		3389
		3390	return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
		3391
		3392	}
		3393
		3394	static void skl_compute_transition_wm(struct drm_crtc *crtc,
		3395	struct skl_pipe_wm_parameters *params,
		3396	struct skl_wm_level trans_wm / out */)
		3397	{
		3398	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3399	int i;
		3400
		3401	if (!params->active)
		3402	return;
		3403
		3404	/* Until we know more, just disable transition WMs */
		3405	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3406	trans_wm->plane_en[i] = false;
		3407	trans_wm->cursor_en = false;
		3408	}
		3409
		3410	static void skl_compute_pipe_wm(struct drm_crtc *crtc,
		3411	struct skl_ddb_allocation *ddb,
		3412	struct skl_pipe_wm_parameters *params,
		3413	struct skl_pipe_wm *pipe_wm)
		3414	{
		3415	struct drm_device *dev = crtc->dev;
		3416	const struct drm_i915_private *dev_priv = dev->dev_private;
		3417	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3418	int level, max_level = ilk_wm_max_level(dev);
		3419
		3420	for (level = 0; level <= max_level; level++) {
		3421	skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
		3422	level, intel_num_planes(intel_crtc),
		3423	&pipe_wm->wm[level]);
		3424	}
		3425	pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
		3426
		3427	skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
		3428	}
		3429
		3430	static void skl_compute_wm_results(struct drm_device *dev,
		3431	struct skl_pipe_wm_parameters *p,
		3432	struct skl_pipe_wm *p_wm,
		3433	struct skl_wm_values *r,
		3434	struct intel_crtc *intel_crtc)
		3435	{
		3436	int level, max_level = ilk_wm_max_level(dev);
		3437	enum pipe pipe = intel_crtc->pipe;
		3438	uint32_t temp;
		3439	int i;
		3440
		3441	for (level = 0; level <= max_level; level++) {
		3442	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3443	temp = 0;
		3444
		3445	temp \|= p_wm->wm[level].plane_res_l[i] <<
		3446	PLANE_WM_LINES_SHIFT;
		3447	temp \|= p_wm->wm[level].plane_res_b[i];
		3448	if (p_wm->wm[level].plane_en[i])
		3449	temp \|= PLANE_WM_EN;
		3450
		3451	r->plane[pipe][i][level] = temp;
		3452	}
		3453
		3454	temp = 0;
		3455
		3456	temp \|= p_wm->wm[level].cursor_res_l << PLANE_WM_LINES_SHIFT;
		3457	temp \|= p_wm->wm[level].cursor_res_b;
		3458
		3459	if (p_wm->wm[level].cursor_en)
		3460	temp \|= PLANE_WM_EN;
		3461
		3462	r->cursor[pipe][level] = temp;
		3463
		3464	}
		3465
		3466	/* transition WMs */
		3467	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3468	temp = 0;
		3469	temp \|= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
		3470	temp \|= p_wm->trans_wm.plane_res_b[i];
		3471	if (p_wm->trans_wm.plane_en[i])
		3472	temp \|= PLANE_WM_EN;
		3473
		3474	r->plane_trans[pipe][i] = temp;
		3475	}
		3476
		3477	temp = 0;
		3478	temp \|= p_wm->trans_wm.cursor_res_l << PLANE_WM_LINES_SHIFT;
		3479	temp \|= p_wm->trans_wm.cursor_res_b;
		3480	if (p_wm->trans_wm.cursor_en)
		3481	temp \|= PLANE_WM_EN;
		3482
		3483	r->cursor_trans[pipe] = temp;
		3484
		3485	r->wm_linetime[pipe] = p_wm->linetime;
		3486	}
		3487
		3488	static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
		3489	const struct skl_ddb_entry *entry)
		3490	{
		3491	if (entry->end)
		3492	I915_WRITE(reg, (entry->end - 1) << 16 \| entry->start);
		3493	else
		3494	I915_WRITE(reg, 0);
		3495	}
		3496
		3497	static void skl_write_wm_values(struct drm_i915_private *dev_priv,
		3498	const struct skl_wm_values *new)
		3499	{
		3500	struct drm_device *dev = dev_priv->dev;
		3501	struct intel_crtc *crtc;
		3502
		3503	list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
		3504	int i, level, max_level = ilk_wm_max_level(dev);
		3505	enum pipe pipe = crtc->pipe;
		3506
		3507	if (!new->dirty[pipe])
		3508	continue;
		3509
		3510	I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
		3511
		3512	for (level = 0; level <= max_level; level++) {
		3513	for (i = 0; i < intel_num_planes(crtc); i++)
		3514	I915_WRITE(PLANE_WM(pipe, i, level),
		3515	new->plane[pipe][i][level]);
		3516	I915_WRITE(CUR_WM(pipe, level),
		3517	new->cursor[pipe][level]);
		3518	}
		3519	for (i = 0; i < intel_num_planes(crtc); i++)
		3520	I915_WRITE(PLANE_WM_TRANS(pipe, i),
		3521	new->plane_trans[pipe][i]);
		3522	I915_WRITE(CUR_WM_TRANS(pipe), new->cursor_trans[pipe]);
		3523
		3524	for (i = 0; i < intel_num_planes(crtc); i++)
		3525	skl_ddb_entry_write(dev_priv,
		3526	PLANE_BUF_CFG(pipe, i),
		3527	&new->ddb.plane[pipe][i]);
		3528
		3529	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
		3530	&new->ddb.cursor[pipe]);
		3531	}
		3532	}
		3533
		3534	/*
		3535	* When setting up a new DDB allocation arrangement, we need to correctly
		3536	* sequence the times at which the new allocations for the pipes are taken into
		3537	* account or we'll have pipes fetching from space previously allocated to
		3538	* another pipe.
		3539	*
		3540	* Roughly the sequence looks like:
		3541	* 1. re-allocate the pipe(s) with the allocation being reduced and not
		3542	* overlapping with a previous light-up pipe (another way to put it is:
		3543	* pipes with their new allocation strickly included into their old ones).
		3544	* 2. re-allocate the other pipes that get their allocation reduced
		3545	* 3. allocate the pipes having their allocation increased
		3546	*
		3547	* Steps 1. and 2. are here to take care of the following case:
		3548	* - Initially DDB looks like this:
		3549	* \| B \| C \|
		3550	* - enable pipe A.
		3551	* - pipe B has a reduced DDB allocation that overlaps with the old pipe C
		3552	* allocation
		3553	* \| A \| B \| C \|
		3554	*
		3555	* We need to sequence the re-allocation: C, B, A (and not B, C, A).
		3556	*/
		3557
		3558	static void
		3559	skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
		3560	{
		3561	struct drm_device *dev = dev_priv->dev;
		3562	int plane;
		3563
		3564	DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
		3565
		3566	for_each_plane(pipe, plane) {
		3567	I915_WRITE(PLANE_SURF(pipe, plane),
		3568	I915_READ(PLANE_SURF(pipe, plane)));
		3569	}
		3570	I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
		3571	}
		3572
		3573	static bool
		3574	skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
		3575	const struct skl_ddb_allocation *new,
		3576	enum pipe pipe)
		3577	{
		3578	uint16_t old_size, new_size;
		3579
		3580	old_size = skl_ddb_entry_size(&old->pipe[pipe]);
		3581	new_size = skl_ddb_entry_size(&new->pipe[pipe]);
		3582
		3583	return old_size != new_size &&
		3584	new->pipe[pipe].start >= old->pipe[pipe].start &&
		3585	new->pipe[pipe].end <= old->pipe[pipe].end;
		3586	}
		3587
		3588	static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
		3589	struct skl_wm_values *new_values)
		3590	{
		3591	struct drm_device *dev = dev_priv->dev;
		3592	struct skl_ddb_allocation cur_ddb, new_ddb;
		3593	bool reallocated[I915_MAX_PIPES] = {false, false, false};
		3594	struct intel_crtc *crtc;
		3595	enum pipe pipe;
		3596
		3597	new_ddb = &new_values->ddb;
		3598	cur_ddb = &dev_priv->wm.skl_hw.ddb;
		3599
		3600	/*
		3601	* First pass: flush the pipes with the new allocation contained into
		3602	* the old space.
		3603	*
		3604	* We'll wait for the vblank on those pipes to ensure we can safely
		3605	* re-allocate the freed space without this pipe fetching from it.
		3606	*/
		3607	for_each_intel_crtc(dev, crtc) {
		3608	if (!crtc->active)
		3609	continue;
		3610
		3611	pipe = crtc->pipe;
		3612
		3613	if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
		3614	continue;
		3615
		3616	skl_wm_flush_pipe(dev_priv, pipe, 1);
		3617	intel_wait_for_vblank(dev, pipe);
		3618
		3619	reallocated[pipe] = true;
		3620	}
		3621
		3622
		3623	/*
		3624	* Second pass: flush the pipes that are having their allocation
		3625	* reduced, but overlapping with a previous allocation.
		3626	*
		3627	* Here as well we need to wait for the vblank to make sure the freed
		3628	* space is not used anymore.
		3629	*/
		3630	for_each_intel_crtc(dev, crtc) {
		3631	if (!crtc->active)
		3632	continue;
		3633
		3634	pipe = crtc->pipe;
		3635
		3636	if (reallocated[pipe])
		3637	continue;
		3638
		3639	if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
		3640	skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
		3641	skl_wm_flush_pipe(dev_priv, pipe, 2);
		3642	intel_wait_for_vblank(dev, pipe);
		3643	}
		3644
		3645	reallocated[pipe] = true;
		3646	}
		3647
		3648	/*
		3649	* Third pass: flush the pipes that got more space allocated.
		3650	*
		3651	* We don't need to actively wait for the update here, next vblank
		3652	* will just get more DDB space with the correct WM values.
		3653	*/
		3654	for_each_intel_crtc(dev, crtc) {
		3655	if (!crtc->active)
		3656	continue;
		3657
		3658	pipe = crtc->pipe;
		3659
		3660	/*
		3661	* At this point, only the pipes more space than before are
		3662	* left to re-allocate.
		3663	*/
		3664	if (reallocated[pipe])
		3665	continue;
		3666
		3667	skl_wm_flush_pipe(dev_priv, pipe, 3);
		3668	}
		3669	}
		3670
		3671	static bool skl_update_pipe_wm(struct drm_crtc *crtc,
		3672	struct skl_pipe_wm_parameters *params,
		3673	struct intel_wm_config *config,
		3674	struct skl_ddb_allocation ddb, / out */
		3675	struct skl_pipe_wm pipe_wm / out */)
		3676	{
		3677	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3678
		3679	skl_compute_wm_pipe_parameters(crtc, params);
		3680	skl_allocate_pipe_ddb(crtc, config, params, ddb);
		3681	skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
		3682
		3683	if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
		3684	return false;
		3685
		3686	intel_crtc->wm.skl_active = *pipe_wm;
		3687	return true;
		3688	}
		3689
		3690	static void skl_update_other_pipe_wm(struct drm_device *dev,
		3691	struct drm_crtc *crtc,
		3692	struct intel_wm_config *config,
		3693	struct skl_wm_values *r)
		3694	{
		3695	struct intel_crtc *intel_crtc;
		3696	struct intel_crtc *this_crtc = to_intel_crtc(crtc);
		3697
		3698	/*
		3699	* If the WM update hasn't changed the allocation for this_crtc (the
		3700	* crtc we are currently computing the new WM values for), other
		3701	* enabled crtcs will keep the same allocation and we don't need to
		3702	* recompute anything for them.
		3703	*/
		3704	if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
		3705	return;
		3706
		3707	/*
		3708	* Otherwise, because of this_crtc being freshly enabled/disabled, the
		3709	* other active pipes need new DDB allocation and WM values.
		3710	*/
		3711	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
		3712	base.head) {
		3713	struct skl_pipe_wm_parameters params = {};
		3714	struct skl_pipe_wm pipe_wm = {};
		3715	bool wm_changed;
		3716
		3717	if (this_crtc->pipe == intel_crtc->pipe)
		3718	continue;
		3719
		3720	if (!intel_crtc->active)
		3721	continue;
		3722
		3723	wm_changed = skl_update_pipe_wm(&intel_crtc->base,
		3724	¶ms, config,
		3725	&r->ddb, &pipe_wm);
		3726
		3727	/*
		3728	* If we end up re-computing the other pipe WM values, it's
		3729	* because it was really needed, so we expect the WM values to
		3730	* be different.
		3731	*/
		3732	WARN_ON(!wm_changed);
		3733
		3734	skl_compute_wm_results(dev, ¶ms, &pipe_wm, r, intel_crtc);
		3735	r->dirty[intel_crtc->pipe] = true;
		3736	}
		3737	}
		3738
		3739	static void skl_update_wm(struct drm_crtc *crtc)
		3740	{
		3741	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3742	struct drm_device *dev = crtc->dev;
		3743	struct drm_i915_private *dev_priv = dev->dev_private;
		3744	struct skl_pipe_wm_parameters params = {};
		3745	struct skl_wm_values *results = &dev_priv->wm.skl_results;
		3746	struct skl_pipe_wm pipe_wm = {};
		3747	struct intel_wm_config config = {};
		3748
		3749	memset(results, 0, sizeof(*results));
		3750
		3751	skl_compute_wm_global_parameters(dev, &config);
		3752
		3753	if (!skl_update_pipe_wm(crtc, ¶ms, &config,
		3754	&results->ddb, &pipe_wm))
		3755	return;
		3756
		3757	skl_compute_wm_results(dev, ¶ms, &pipe_wm, results, intel_crtc);
		3758	results->dirty[intel_crtc->pipe] = true;
		3759
		3760	skl_update_other_pipe_wm(dev, crtc, &config, results);
		3761	skl_write_wm_values(dev_priv, results);
		3762	skl_flush_wm_values(dev_priv, results);
		3763
		3764	/* store the new configuration */
		3765	dev_priv->wm.skl_hw = *results;
		3766	}
		3767
		3768	static void
		3769	skl_update_sprite_wm(struct drm_plane plane, struct drm_crtc crtc,
		3770	uint32_t sprite_width, uint32_t sprite_height,
		3771	int pixel_size, bool enabled, bool scaled)
		3772	{
		3773	struct intel_plane *intel_plane = to_intel_plane(plane);
		3774
		3775	intel_plane->wm.enabled = enabled;
		3776	intel_plane->wm.scaled = scaled;
		3777	intel_plane->wm.horiz_pixels = sprite_width;
		3778	intel_plane->wm.vert_pixels = sprite_height;
		3779	intel_plane->wm.bytes_per_pixel = pixel_size;
		3780
		3781	skl_update_wm(crtc);
		3782	}
		3783
4560	Serge	3784	static void ilk_update_wm(struct drm_crtc *crtc)
		3785	{
		3786	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3787	struct drm_device *dev = crtc->dev;
		3788	struct drm_i915_private *dev_priv = dev->dev_private;
		3789	struct ilk_wm_maximums max;
		3790	struct ilk_pipe_wm_parameters params = {};
		3791	struct ilk_wm_values results = {};
4104	Serge	3792	enum intel_ddb_partitioning partitioning;
4560	Serge	3793	struct intel_pipe_wm pipe_wm = {};
		3794	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
		3795	struct intel_wm_config config = {};
4104	Serge	3796
5060	serge	3797	ilk_compute_wm_parameters(crtc, ¶ms);
4104	Serge	3798
4560	Serge	3799	intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm);
		3800
		3801	if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
		3802	return;
		3803
		3804	intel_crtc->wm.active = pipe_wm;
		3805
5060	serge	3806	ilk_compute_wm_config(dev, &config);
		3807
4560	Serge	3808	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
		3809	ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
		3810
		3811	/* 5/6 split only in single pipe config on IVB+ */
		3812	if (INTEL_INFO(dev)->gen >= 7 &&
		3813	config.num_pipes_active == 1 && config.sprites_enabled) {
		3814	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
		3815	ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
		3816
		3817	best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
4104	Serge	3818	} else {
4560	Serge	3819	best_lp_wm = &lp_wm_1_2;
4104	Serge	3820	}
		3821
4560	Serge	3822	partitioning = (best_lp_wm == &lp_wm_1_2) ?
4104	Serge	3823	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
		3824
4560	Serge	3825	ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
		3826
		3827	ilk_write_wm_values(dev_priv, &results);
4104	Serge	3828	}
		3829
5060	serge	3830	static void
		3831	ilk_update_sprite_wm(struct drm_plane *plane,
5354	serge	3832	struct drm_crtc *crtc,
5060	serge	3833	uint32_t sprite_width, uint32_t sprite_height,
		3834	int pixel_size, bool enabled, bool scaled)
4104	Serge	3835	{
4560	Serge	3836	struct drm_device *dev = plane->dev;
5354	serge	3837	struct intel_plane *intel_plane = to_intel_plane(plane);
4104	Serge	3838
		3839	intel_plane->wm.enabled = enabled;
		3840	intel_plane->wm.scaled = scaled;
		3841	intel_plane->wm.horiz_pixels = sprite_width;
5060	serge	3842	intel_plane->wm.vert_pixels = sprite_width;
5354	serge	3843	intel_plane->wm.bytes_per_pixel = pixel_size;
4104	Serge	3844
4560	Serge	3845	/*
		3846	* IVB workaround: must disable low power watermarks for at least
		3847	* one frame before enabling scaling. LP watermarks can be re-enabled
		3848	* when scaling is disabled.
		3849	*
		3850	* WaCxSRDisabledForSpriteScaling:ivb
		3851	*/
		3852	if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
		3853	intel_wait_for_vblank(dev, intel_plane->pipe);
		3854
		3855	ilk_update_wm(crtc);
4104	Serge	3856	}
		3857
5354	serge	3858	static void skl_pipe_wm_active_state(uint32_t val,
		3859	struct skl_pipe_wm *active,
		3860	bool is_transwm,
		3861	bool is_cursor,
		3862	int i,
		3863	int level)
		3864	{
		3865	bool is_enabled = (val & PLANE_WM_EN) != 0;
		3866
		3867	if (!is_transwm) {
		3868	if (!is_cursor) {
		3869	active->wm[level].plane_en[i] = is_enabled;
		3870	active->wm[level].plane_res_b[i] =
		3871	val & PLANE_WM_BLOCKS_MASK;
		3872	active->wm[level].plane_res_l[i] =
		3873	(val >> PLANE_WM_LINES_SHIFT) &
		3874	PLANE_WM_LINES_MASK;
		3875	} else {
		3876	active->wm[level].cursor_en = is_enabled;
		3877	active->wm[level].cursor_res_b =
		3878	val & PLANE_WM_BLOCKS_MASK;
		3879	active->wm[level].cursor_res_l =
		3880	(val >> PLANE_WM_LINES_SHIFT) &
		3881	PLANE_WM_LINES_MASK;
		3882	}
		3883	} else {
		3884	if (!is_cursor) {
		3885	active->trans_wm.plane_en[i] = is_enabled;
		3886	active->trans_wm.plane_res_b[i] =
		3887	val & PLANE_WM_BLOCKS_MASK;
		3888	active->trans_wm.plane_res_l[i] =
		3889	(val >> PLANE_WM_LINES_SHIFT) &
		3890	PLANE_WM_LINES_MASK;
		3891	} else {
		3892	active->trans_wm.cursor_en = is_enabled;
		3893	active->trans_wm.cursor_res_b =
		3894	val & PLANE_WM_BLOCKS_MASK;
		3895	active->trans_wm.cursor_res_l =
		3896	(val >> PLANE_WM_LINES_SHIFT) &
		3897	PLANE_WM_LINES_MASK;
		3898	}
		3899	}
		3900	}
		3901
		3902	static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
		3903	{
		3904	struct drm_device *dev = crtc->dev;
		3905	struct drm_i915_private *dev_priv = dev->dev_private;
		3906	struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
		3907	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3908	struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
		3909	enum pipe pipe = intel_crtc->pipe;
		3910	int level, i, max_level;
		3911	uint32_t temp;
		3912
		3913	max_level = ilk_wm_max_level(dev);
		3914
		3915	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
		3916
		3917	for (level = 0; level <= max_level; level++) {
		3918	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3919	hw->plane[pipe][i][level] =
		3920	I915_READ(PLANE_WM(pipe, i, level));
		3921	hw->cursor[pipe][level] = I915_READ(CUR_WM(pipe, level));
		3922	}
		3923
		3924	for (i = 0; i < intel_num_planes(intel_crtc); i++)
		3925	hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
		3926	hw->cursor_trans[pipe] = I915_READ(CUR_WM_TRANS(pipe));
		3927
		3928	if (!intel_crtc_active(crtc))
		3929	return;
		3930
		3931	hw->dirty[pipe] = true;
		3932
		3933	active->linetime = hw->wm_linetime[pipe];
		3934
		3935	for (level = 0; level <= max_level; level++) {
		3936	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3937	temp = hw->plane[pipe][i][level];
		3938	skl_pipe_wm_active_state(temp, active, false,
		3939	false, i, level);
		3940	}
		3941	temp = hw->cursor[pipe][level];
		3942	skl_pipe_wm_active_state(temp, active, false, true, i, level);
		3943	}
		3944
		3945	for (i = 0; i < intel_num_planes(intel_crtc); i++) {
		3946	temp = hw->plane_trans[pipe][i];
		3947	skl_pipe_wm_active_state(temp, active, true, false, i, 0);
		3948	}
		3949
		3950	temp = hw->cursor_trans[pipe];
		3951	skl_pipe_wm_active_state(temp, active, true, true, i, 0);
		3952	}
		3953
		3954	void skl_wm_get_hw_state(struct drm_device *dev)
		3955	{
		3956	struct drm_i915_private *dev_priv = dev->dev_private;
		3957	struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
		3958	struct drm_crtc *crtc;
		3959
		3960	skl_ddb_get_hw_state(dev_priv, ddb);
		3961	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
		3962	skl_pipe_wm_get_hw_state(crtc);
		3963	}
		3964
4560	Serge	3965	static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3031	serge	3966	{
4560	Serge	3967	struct drm_device *dev = crtc->dev;
		3968	struct drm_i915_private *dev_priv = dev->dev_private;
		3969	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		3970	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		3971	struct intel_pipe_wm *active = &intel_crtc->wm.active;
		3972	enum pipe pipe = intel_crtc->pipe;
		3973	static const unsigned int wm0_pipe_reg[] = {
		3974	[PIPE_A] = WM0_PIPEA_ILK,
		3975	[PIPE_B] = WM0_PIPEB_ILK,
		3976	[PIPE_C] = WM0_PIPEC_IVB,
		3977	};
3031	serge	3978
4560	Serge	3979	hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
		3980	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		3981	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3031	serge	3982
5060	serge	3983	active->pipe_enabled = intel_crtc_active(crtc);
		3984
		3985	if (active->pipe_enabled) {
4560	Serge	3986	u32 tmp = hw->wm_pipe[pipe];
3031	serge	3987
4560	Serge	3988	/*
		3989	* For active pipes LP0 watermark is marked as
		3990	* enabled, and LP1+ watermaks as disabled since
		3991	* we can't really reverse compute them in case
		3992	* multiple pipes are active.
		3993	*/
		3994	active->wm[0].enable = true;
		3995	active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
		3996	active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
		3997	active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
		3998	active->linetime = hw->wm_linetime[pipe];
		3999	} else {
		4000	int level, max_level = ilk_wm_max_level(dev);
3031	serge	4001
4560	Serge	4002	/*
		4003	* For inactive pipes, all watermark levels
		4004	* should be marked as enabled but zeroed,
		4005	* which is what we'd compute them to.
		4006	*/
		4007	for (level = 0; level <= max_level; level++)
		4008	active->wm[level].enable = true;
3031	serge	4009	}
		4010	}
		4011
4560	Serge	4012	void ilk_wm_get_hw_state(struct drm_device *dev)
3031	serge	4013	{
		4014	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4015	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		4016	struct drm_crtc *crtc;
3031	serge	4017
5060	serge	4018	for_each_crtc(dev, crtc)
4560	Serge	4019	ilk_pipe_wm_get_hw_state(crtc);
4104	Serge	4020
4560	Serge	4021	hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
		4022	hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
		4023	hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3031	serge	4024
4560	Serge	4025	hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
5060	serge	4026	if (INTEL_INFO(dev)->gen >= 7) {
5354	serge	4027	hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		4028	hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
5060	serge	4029	}
3031	serge	4030
4560	Serge	4031	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4032	hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
		4033	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
		4034	else if (IS_IVYBRIDGE(dev))
		4035	hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
		4036	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3031	serge	4037
4560	Serge	4038	hw->enable_fbc_wm =
		4039	!(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3031	serge	4040	}
		4041
		4042	/**
		4043	* intel_update_watermarks - update FIFO watermark values based on current modes
		4044	*
		4045	* Calculate watermark values for the various WM regs based on current mode
		4046	* and plane configuration.
		4047	*
		4048	* There are several cases to deal with here:
		4049	* - normal (i.e. non-self-refresh)
		4050	* - self-refresh (SR) mode
		4051	* - lines are large relative to FIFO size (buffer can hold up to 2)
		4052	* - lines are small relative to FIFO size (buffer can hold more than 2
		4053	* lines), so need to account for TLB latency
		4054	*
		4055	* The normal calculation is:
		4056	* watermark = dotclock * bytes per pixel * latency
		4057	* where latency is platform & configuration dependent (we assume pessimal
		4058	* values here).
		4059	*
		4060	* The SR calculation is:
		4061	* watermark = (trunc(latency/line time)+1) * surface width *
		4062	* bytes per pixel
		4063	* where
		4064	* line time = htotal / dotclock
		4065	* surface width = hdisplay for normal plane and 64 for cursor
		4066	* and latency is assumed to be high, as above.
		4067	*
		4068	* The final value programmed to the register should always be rounded up,
		4069	* and include an extra 2 entries to account for clock crossings.
		4070	*
		4071	* We don't use the sprite, so we can ignore that. And on Crestline we have
		4072	* to set the non-SR watermarks to 8.
		4073	*/
4560	Serge	4074	void intel_update_watermarks(struct drm_crtc *crtc)
3031	serge	4075	{
4560	Serge	4076	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3031	serge	4077
		4078	if (dev_priv->display.update_wm)
4560	Serge	4079	dev_priv->display.update_wm(crtc);
3031	serge	4080	}
		4081
4104	Serge	4082	void intel_update_sprite_watermarks(struct drm_plane *plane,
		4083	struct drm_crtc *crtc,
5060	serge	4084	uint32_t sprite_width,
		4085	uint32_t sprite_height,
		4086	int pixel_size,
4104	Serge	4087	bool enabled, bool scaled)
3031	serge	4088	{
4104	Serge	4089	struct drm_i915_private *dev_priv = plane->dev->dev_private;
3031	serge	4090
		4091	if (dev_priv->display.update_sprite_wm)
5060	serge	4092	dev_priv->display.update_sprite_wm(plane, crtc,
		4093	sprite_width, sprite_height,
4104	Serge	4094	pixel_size, enabled, scaled);
3031	serge	4095	}
		4096
		4097	static struct drm_i915_gem_object *
		4098	intel_alloc_context_page(struct drm_device *dev)
		4099	{
		4100	struct drm_i915_gem_object *ctx;
		4101	int ret;
		4102
		4103	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		4104
		4105	ctx = i915_gem_alloc_object(dev, 4096);
		4106	if (!ctx) {
		4107	DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
		4108	return NULL;
		4109	}
		4110
5060	serge	4111	ret = i915_gem_obj_ggtt_pin(ctx, 4096, 0);
3031	serge	4112	if (ret) {
		4113	DRM_ERROR("failed to pin power context: %d\n", ret);
		4114	goto err_unref;
		4115	}
		4116
		4117	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
		4118	if (ret) {
		4119	DRM_ERROR("failed to set-domain on power context: %d\n", ret);
		4120	goto err_unpin;
		4121	}
		4122
		4123	return ctx;
		4124
		4125	err_unpin:
5060	serge	4126	i915_gem_object_ggtt_unpin(ctx);
3031	serge	4127	err_unref:
		4128	drm_gem_object_unreference(&ctx->base);
		4129	return NULL;
		4130	}
		4131
		4132	/**
		4133	* Lock protecting IPS related data structures
		4134	*/
		4135	DEFINE_SPINLOCK(mchdev_lock);
		4136
		4137	/* Global for IPS driver to get at the current i915 device. Protected by
		4138	* mchdev_lock. */
		4139	static struct drm_i915_private *i915_mch_dev;
		4140
		4141	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		4142	{
		4143	struct drm_i915_private *dev_priv = dev->dev_private;
		4144	u16 rgvswctl;
		4145
		4146	assert_spin_locked(&mchdev_lock);
		4147
		4148	rgvswctl = I915_READ16(MEMSWCTL);
		4149	if (rgvswctl & MEMCTL_CMD_STS) {
		4150	DRM_DEBUG("gpu busy, RCS change rejected\n");
		4151	return false; /* still busy with another command */
		4152	}
		4153
		4154	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		4155	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		4156	I915_WRITE16(MEMSWCTL, rgvswctl);
		4157	POSTING_READ16(MEMSWCTL);
		4158
		4159	rgvswctl \|= MEMCTL_CMD_STS;
		4160	I915_WRITE16(MEMSWCTL, rgvswctl);
		4161
		4162	return true;
		4163	}
		4164
		4165	static void ironlake_enable_drps(struct drm_device *dev)
		4166	{
		4167	struct drm_i915_private *dev_priv = dev->dev_private;
		4168	u32 rgvmodectl = I915_READ(MEMMODECTL);
		4169	u8 fmax, fmin, fstart, vstart;
		4170
		4171	spin_lock_irq(&mchdev_lock);
		4172
		4173	/* Enable temp reporting */
		4174	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		4175	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		4176
		4177	/* 100ms RC evaluation intervals */
		4178	I915_WRITE(RCUPEI, 100000);
		4179	I915_WRITE(RCDNEI, 100000);
		4180
		4181	/* Set max/min thresholds to 90ms and 80ms respectively */
		4182	I915_WRITE(RCBMAXAVG, 90000);
		4183	I915_WRITE(RCBMINAVG, 80000);
		4184
		4185	I915_WRITE(MEMIHYST, 1);
		4186
		4187	/* Set up min, max, and cur for interrupt handling */
		4188	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		4189	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		4190	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		4191	MEMMODE_FSTART_SHIFT;
		4192
		4193	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
		4194	PXVFREQ_PX_SHIFT;
		4195
		4196	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		4197	dev_priv->ips.fstart = fstart;
		4198
		4199	dev_priv->ips.max_delay = fstart;
		4200	dev_priv->ips.min_delay = fmin;
		4201	dev_priv->ips.cur_delay = fstart;
		4202
		4203	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		4204	fmax, fmin, fstart);
		4205
		4206	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		4207
		4208	/*
		4209	* Interrupts will be enabled in ironlake_irq_postinstall
		4210	*/
		4211
		4212	I915_WRITE(VIDSTART, vstart);
		4213	POSTING_READ(VIDSTART);
		4214
		4215	rgvmodectl \|= MEMMODE_SWMODE_EN;
		4216	I915_WRITE(MEMMODECTL, rgvmodectl);
		4217
		4218	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		4219	DRM_ERROR("stuck trying to change perf mode\n");
		4220	mdelay(1);
		4221
		4222	ironlake_set_drps(dev, fstart);
		4223
		4224	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
		4225	I915_READ(0x112e0);
5060	serge	4226	dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
3031	serge	4227	dev_priv->ips.last_count2 = I915_READ(0x112f4);
5060	serge	4228	dev_priv->ips.last_time2 = ktime_get_raw_ns();
3031	serge	4229
		4230	spin_unlock_irq(&mchdev_lock);
		4231	}
		4232
		4233	static void ironlake_disable_drps(struct drm_device *dev)
		4234	{
		4235	struct drm_i915_private *dev_priv = dev->dev_private;
		4236	u16 rgvswctl;
		4237
		4238	spin_lock_irq(&mchdev_lock);
		4239
		4240	rgvswctl = I915_READ16(MEMSWCTL);
		4241
		4242	/* Ack interrupts, disable EFC interrupt */
		4243	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		4244	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		4245	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		4246	I915_WRITE(DEIIR, DE_PCU_EVENT);
		4247	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		4248
		4249	/* Go back to the starting frequency */
		4250	ironlake_set_drps(dev, dev_priv->ips.fstart);
		4251	mdelay(1);
		4252	rgvswctl \|= MEMCTL_CMD_STS;
		4253	I915_WRITE(MEMSWCTL, rgvswctl);
		4254	mdelay(1);
		4255
		4256	spin_unlock_irq(&mchdev_lock);
		4257	}
		4258
		4259	/* There's a funny hw issue where the hw returns all 0 when reading from
		4260	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		4261	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		4262	* all limits and the gpu stuck at whatever frequency it is at atm).
		4263	*/
4560	Serge	4264	static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3031	serge	4265	{
		4266	u32 limits;
		4267
		4268	/* Only set the down limit when we've reached the lowest level to avoid
		4269	* getting more interrupts, otherwise leave this clear. This prevents a
		4270	* race in the hw when coming out of rc6: There's a tiny window where
		4271	* the hw runs at the minimal clock before selecting the desired
		4272	* frequency, if the down threshold expires in that window we will not
		4273	* receive a down interrupt. */
5060	serge	4274	limits = dev_priv->rps.max_freq_softlimit << 24;
		4275	if (val <= dev_priv->rps.min_freq_softlimit)
		4276	limits \|= dev_priv->rps.min_freq_softlimit << 16;
3031	serge	4277
		4278	return limits;
		4279	}
		4280
4560	Serge	4281	static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
		4282	{
		4283	int new_power;
		4284
		4285	new_power = dev_priv->rps.power;
		4286	switch (dev_priv->rps.power) {
		4287	case LOW_POWER:
5060	serge	4288	if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
4560	Serge	4289	new_power = BETWEEN;
		4290	break;
		4291
		4292	case BETWEEN:
5060	serge	4293	if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
4560	Serge	4294	new_power = LOW_POWER;
5060	serge	4295	else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
4560	Serge	4296	new_power = HIGH_POWER;
		4297	break;
		4298
		4299	case HIGH_POWER:
5060	serge	4300	if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
4560	Serge	4301	new_power = BETWEEN;
		4302	break;
		4303	}
		4304	/* Max/min bins are special */
5060	serge	4305	if (val == dev_priv->rps.min_freq_softlimit)
4560	Serge	4306	new_power = LOW_POWER;
5060	serge	4307	if (val == dev_priv->rps.max_freq_softlimit)
4560	Serge	4308	new_power = HIGH_POWER;
		4309	if (new_power == dev_priv->rps.power)
		4310	return;
		4311
		4312	/* Note the units here are not exactly 1us, but 1280ns. */
		4313	switch (new_power) {
		4314	case LOW_POWER:
		4315	/* Upclock if more than 95% busy over 16ms */
		4316	I915_WRITE(GEN6_RP_UP_EI, 12500);
		4317	I915_WRITE(GEN6_RP_UP_THRESHOLD, 11800);
		4318
		4319	/* Downclock if less than 85% busy over 32ms */
		4320	I915_WRITE(GEN6_RP_DOWN_EI, 25000);
		4321	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 21250);
		4322
		4323	I915_WRITE(GEN6_RP_CONTROL,
		4324	GEN6_RP_MEDIA_TURBO \|
		4325	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4326	GEN6_RP_MEDIA_IS_GFX \|
		4327	GEN6_RP_ENABLE \|
		4328	GEN6_RP_UP_BUSY_AVG \|
		4329	GEN6_RP_DOWN_IDLE_AVG);
		4330	break;
		4331
		4332	case BETWEEN:
		4333	/* Upclock if more than 90% busy over 13ms */
		4334	I915_WRITE(GEN6_RP_UP_EI, 10250);
		4335	I915_WRITE(GEN6_RP_UP_THRESHOLD, 9225);
		4336
		4337	/* Downclock if less than 75% busy over 32ms */
		4338	I915_WRITE(GEN6_RP_DOWN_EI, 25000);
		4339	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 18750);
		4340
		4341	I915_WRITE(GEN6_RP_CONTROL,
		4342	GEN6_RP_MEDIA_TURBO \|
		4343	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4344	GEN6_RP_MEDIA_IS_GFX \|
		4345	GEN6_RP_ENABLE \|
		4346	GEN6_RP_UP_BUSY_AVG \|
		4347	GEN6_RP_DOWN_IDLE_AVG);
		4348	break;
		4349
		4350	case HIGH_POWER:
		4351	/* Upclock if more than 85% busy over 10ms */
		4352	I915_WRITE(GEN6_RP_UP_EI, 8000);
		4353	I915_WRITE(GEN6_RP_UP_THRESHOLD, 6800);
		4354
		4355	/* Downclock if less than 60% busy over 32ms */
		4356	I915_WRITE(GEN6_RP_DOWN_EI, 25000);
		4357	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 15000);
		4358
		4359	I915_WRITE(GEN6_RP_CONTROL,
		4360	GEN6_RP_MEDIA_TURBO \|
		4361	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4362	GEN6_RP_MEDIA_IS_GFX \|
		4363	GEN6_RP_ENABLE \|
		4364	GEN6_RP_UP_BUSY_AVG \|
		4365	GEN6_RP_DOWN_IDLE_AVG);
		4366	break;
		4367	}
		4368
		4369	dev_priv->rps.power = new_power;
		4370	dev_priv->rps.last_adj = 0;
		4371	}
		4372
5060	serge	4373	static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
		4374	{
		4375	u32 mask = 0;
		4376
		4377	if (val > dev_priv->rps.min_freq_softlimit)
		4378	mask \|= GEN6_PM_RP_DOWN_THRESHOLD \| GEN6_PM_RP_DOWN_TIMEOUT;
		4379	if (val < dev_priv->rps.max_freq_softlimit)
		4380	mask \|= GEN6_PM_RP_UP_THRESHOLD;
		4381
		4382	mask \|= dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_UP_EI_EXPIRED);
		4383	mask &= dev_priv->pm_rps_events;
		4384
		4385	/* IVB and SNB hard hangs on looping batchbuffer
		4386	* if GEN6_PM_UP_EI_EXPIRED is masked.
		4387	*/
		4388	if (INTEL_INFO(dev_priv->dev)->gen <= 7 && !IS_HASWELL(dev_priv->dev))
		4389	mask \|= GEN6_PM_RP_UP_EI_EXPIRED;
		4390
		4391	if (IS_GEN8(dev_priv->dev))
		4392	mask \|= GEN8_PMINTR_REDIRECT_TO_NON_DISP;
		4393
		4394	return ~mask;
		4395	}
		4396
		4397	/* gen6_set_rps is called to update the frequency request, but should also be
		4398	* called when the range (min_delay and max_delay) is modified so that we can
		4399	* update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
3031	serge	4400	void gen6_set_rps(struct drm_device *dev, u8 val)
		4401	{
		4402	struct drm_i915_private *dev_priv = dev->dev_private;
		4403
3243	Serge	4404	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5060	serge	4405	WARN_ON(val > dev_priv->rps.max_freq_softlimit);
		4406	WARN_ON(val < dev_priv->rps.min_freq_softlimit);
3031	serge	4407
5060	serge	4408	/* min/max delay may still have been modified so be sure to
		4409	* write the limits value.
		4410	*/
		4411	if (val != dev_priv->rps.cur_freq) {
5354	serge	4412	gen6_set_rps_thresholds(dev_priv, val);
4560	Serge	4413
5060	serge	4414	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
5354	serge	4415	I915_WRITE(GEN6_RPNSWREQ,
		4416	HSW_FREQUENCY(val));
		4417	else
		4418	I915_WRITE(GEN6_RPNSWREQ,
		4419	GEN6_FREQUENCY(val) \|
		4420	GEN6_OFFSET(0) \|
		4421	GEN6_AGGRESSIVE_TURBO);
5060	serge	4422	}
3031	serge	4423
		4424	/* Make sure we continue to get interrupts
		4425	* until we hit the minimum or maximum frequencies.
		4426	*/
5060	serge	4427	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, gen6_rps_limits(dev_priv, val));
		4428	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3031	serge	4429
		4430	POSTING_READ(GEN6_RPNSWREQ);
		4431
5060	serge	4432	dev_priv->rps.cur_freq = val;
3031	serge	4433	trace_intel_gpu_freq_change(val * 50);
		4434	}
		4435
5060	serge	4436	/* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down
		4437	*
		4438	* * If Gfx is Idle, then
		4439	* 1. Mask Turbo interrupts
		4440	* 2. Bring up Gfx clock
		4441	* 3. Change the freq to Rpn and wait till P-Unit updates freq
		4442	* 4. Clear the Force GFX CLK ON bit so that Gfx can down
		4443	* 5. Unmask Turbo interrupts
		4444	*/
		4445	static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
		4446	{
		4447	struct drm_device *dev = dev_priv->dev;
		4448
		4449	/* Latest VLV doesn't need to force the gfx clock */
		4450	if (dev->pdev->revision >= 0xd) {
		4451	valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
		4452	return;
		4453	}
		4454
		4455	/*
		4456	* When we are idle. Drop to min voltage state.
		4457	*/
		4458
		4459	if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit)
		4460	return;
		4461
		4462	/* Mask turbo interrupt so that they will not come in between */
		4463	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
		4464
		4465	vlv_force_gfx_clock(dev_priv, true);
		4466
		4467	dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit;
		4468
		4469	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ,
		4470	dev_priv->rps.min_freq_softlimit);
		4471
		4472	if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS))
5354	serge	4473	& GENFREQSTATUS) == 0, 100))
5060	serge	4474	DRM_ERROR("timed out waiting for Punit\n");
		4475
		4476	vlv_force_gfx_clock(dev_priv, false);
		4477
		4478	I915_WRITE(GEN6_PMINTRMSK,
		4479	gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
		4480	}
		4481
4560	Serge	4482	void gen6_rps_idle(struct drm_i915_private *dev_priv)
3031	serge	4483	{
4560	Serge	4484	struct drm_device *dev = dev_priv->dev;
4104	Serge	4485
4560	Serge	4486	mutex_lock(&dev_priv->rps.hw_lock);
		4487	if (dev_priv->rps.enabled) {
5060	serge	4488	if (IS_CHERRYVIEW(dev))
		4489	valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
		4490	else if (IS_VALLEYVIEW(dev))
		4491	vlv_set_rps_idle(dev_priv);
4560	Serge	4492	else
5060	serge	4493	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4560	Serge	4494	dev_priv->rps.last_adj = 0;
		4495	}
		4496	mutex_unlock(&dev_priv->rps.hw_lock);
		4497	}
4104	Serge	4498
4560	Serge	4499	void gen6_rps_boost(struct drm_i915_private *dev_priv)
		4500	{
		4501	struct drm_device *dev = dev_priv->dev;
4104	Serge	4502
4560	Serge	4503	mutex_lock(&dev_priv->rps.hw_lock);
		4504	if (dev_priv->rps.enabled) {
		4505	if (IS_VALLEYVIEW(dev))
5060	serge	4506	valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
4560	Serge	4507	else
5060	serge	4508	gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
4560	Serge	4509	dev_priv->rps.last_adj = 0;
		4510	}
		4511	mutex_unlock(&dev_priv->rps.hw_lock);
4104	Serge	4512	}
		4513
		4514	void valleyview_set_rps(struct drm_device *dev, u8 val)
		4515	{
3031	serge	4516	struct drm_i915_private *dev_priv = dev->dev_private;
		4517
4104	Serge	4518	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5060	serge	4519	WARN_ON(val > dev_priv->rps.max_freq_softlimit);
		4520	WARN_ON(val < dev_priv->rps.min_freq_softlimit);
4104	Serge	4521
5354	serge	4522	if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
		4523	"Odd GPU freq value\n"))
		4524	val &= ~1;
4104	Serge	4525
5060	serge	4526	if (val != dev_priv->rps.cur_freq)
5354	serge	4527	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
4104	Serge	4528
5060	serge	4529	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
4104	Serge	4530
5060	serge	4531	dev_priv->rps.cur_freq = val;
4560	Serge	4532	trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val));
4104	Serge	4533	}
		4534
5354	serge	4535	static void gen9_disable_rps(struct drm_device *dev)
5060	serge	4536	{
		4537	struct drm_i915_private *dev_priv = dev->dev_private;
		4538
5354	serge	4539	I915_WRITE(GEN6_RC_CONTROL, 0);
5060	serge	4540	}
		4541
4104	Serge	4542	static void gen6_disable_rps(struct drm_device *dev)
		4543	{
		4544	struct drm_i915_private *dev_priv = dev->dev_private;
		4545
		4546	I915_WRITE(GEN6_RC_CONTROL, 0);
		4547	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		4548	}
		4549
5060	serge	4550	static void cherryview_disable_rps(struct drm_device *dev)
		4551	{
		4552	struct drm_i915_private *dev_priv = dev->dev_private;
		4553
		4554	I915_WRITE(GEN6_RC_CONTROL, 0);
		4555	}
		4556
4104	Serge	4557	static void valleyview_disable_rps(struct drm_device *dev)
		4558	{
		4559	struct drm_i915_private *dev_priv = dev->dev_private;
		4560
5354	serge	4561	/* we're doing forcewake before Disabling RC6,
		4562	* This what the BIOS expects when going into suspend */
		4563	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
		4564
4104	Serge	4565	I915_WRITE(GEN6_RC_CONTROL, 0);
		4566
5354	serge	4567	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	4568	}
		4569
4560	Serge	4570	static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
		4571	{
5060	serge	4572	if (IS_VALLEYVIEW(dev)) {
		4573	if (mode & (GEN7_RC_CTL_TO_MODE \| GEN6_RC_CTL_EI_MODE(1)))
		4574	mode = GEN6_RC_CTL_RC6_ENABLE;
		4575	else
		4576	mode = 0;
		4577	}
5354	serge	4578	if (HAS_RC6p(dev))
		4579	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
4560	Serge	4580	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		4581	(mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		4582	(mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
5354	serge	4583
		4584	else
		4585	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
		4586	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
4560	Serge	4587	}
		4588
5060	serge	4589	static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
3031	serge	4590	{
4104	Serge	4591	/* No RC6 before Ironlake */
		4592	if (INTEL_INFO(dev)->gen < 5)
		4593	return 0;
		4594
5060	serge	4595	/* RC6 is only on Ironlake mobile not on desktop */
		4596	if (INTEL_INFO(dev)->gen == 5 && !IS_IRONLAKE_M(dev))
		4597	return 0;
		4598
3031	serge	4599	/* Respect the kernel parameter if it is set */
5060	serge	4600	if (enable_rc6 >= 0) {
		4601	int mask;
3031	serge	4602
5354	serge	4603	if (HAS_RC6p(dev))
5060	serge	4604	mask = INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE \|
		4605	INTEL_RC6pp_ENABLE;
		4606	else
		4607	mask = INTEL_RC6_ENABLE;
		4608
		4609	if ((enable_rc6 & mask) != enable_rc6)
		4610	DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
		4611	enable_rc6 & mask, enable_rc6, mask);
		4612
		4613	return enable_rc6 & mask;
		4614	}
		4615
3120	serge	4616	/* Disable RC6 on Ironlake */
		4617	if (INTEL_INFO(dev)->gen == 5)
		4618	return 0;
3031	serge	4619
5060	serge	4620	if (IS_IVYBRIDGE(dev))
		4621	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
3031	serge	4622
		4623	return INTEL_RC6_ENABLE;
5060	serge	4624	}
3031	serge	4625
5060	serge	4626	int intel_enable_rc6(const struct drm_device *dev)
		4627	{
		4628	return i915.enable_rc6;
3031	serge	4629	}
		4630
5354	serge	4631	static void gen6_init_rps_frequencies(struct drm_device *dev)
5060	serge	4632	{
		4633	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	4634	uint32_t rp_state_cap;
		4635	u32 ddcc_status = 0;
		4636	int ret;
5060	serge	4637
5354	serge	4638	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
5060	serge	4639	/* All of these values are in units of 50MHz */
		4640	dev_priv->rps.cur_freq = 0;
5354	serge	4641	/* static values from HW: RP0 > RP1 > RPn (min_freq) */
		4642	dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
5060	serge	4643	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		4644	dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
		4645	/* hw_max = RP0 until we check for overclocking */
		4646	dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
4104	Serge	4647
5354	serge	4648	dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
		4649	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
		4650	ret = sandybridge_pcode_read(dev_priv,
		4651	HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
		4652	&ddcc_status);
		4653	if (0 == ret)
		4654	dev_priv->rps.efficient_freq =
		4655	(ddcc_status >> 8) & 0xff;
		4656	}
		4657
5060	serge	4658	/* Preserve min/max settings in case of re-init */
		4659	if (dev_priv->rps.max_freq_softlimit == 0)
		4660	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4104	Serge	4661
5354	serge	4662	if (dev_priv->rps.min_freq_softlimit == 0) {
		4663	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		4664	dev_priv->rps.min_freq_softlimit =
		4665	/* max(RPe, 450 MHz) */
		4666	max(dev_priv->rps.efficient_freq, (u8) 9);
		4667	else
		4668	dev_priv->rps.min_freq_softlimit =
		4669	dev_priv->rps.min_freq;
		4670	}
4104	Serge	4671	}
		4672
5354	serge	4673	static void gen9_enable_rps(struct drm_device *dev)
		4674	{
		4675	struct drm_i915_private *dev_priv = dev->dev_private;
		4676	struct intel_engine_cs *ring;
		4677	uint32_t rc6_mask = 0;
		4678	int unused;
		4679
		4680	/* 1a: Software RC state - RC0 */
		4681	I915_WRITE(GEN6_RC_STATE, 0);
		4682
		4683	/* 1b: Get forcewake during program sequence. Although the driver
		4684	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
		4685	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
		4686
		4687	/* 2a: Disable RC states. */
		4688	I915_WRITE(GEN6_RC_CONTROL, 0);
		4689
		4690	/* 2b: Program RC6 thresholds.*/
		4691	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
		4692	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4693	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4694	for_each_ring(ring, dev_priv, unused)
		4695	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4696	I915_WRITE(GEN6_RC_SLEEP, 0);
		4697	I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
		4698
		4699	/* 3a: Enable RC6 */
		4700	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4701	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
		4702	DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
		4703	"on" : "off");
		4704	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4705	GEN6_RC_CTL_EI_MODE(1) \|
		4706	rc6_mask);
		4707
		4708	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
		4709
		4710	}
		4711
4560	Serge	4712	static void gen8_enable_rps(struct drm_device *dev)
		4713	{
		4714	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4715	struct intel_engine_cs *ring;
5354	serge	4716	uint32_t rc6_mask = 0;
4560	Serge	4717	int unused;
		4718
		4719	/* 1a: Software RC state - RC0 */
		4720	I915_WRITE(GEN6_RC_STATE, 0);
		4721
		4722	/* 1c & 1d: Get forcewake during program sequence. Although the driver
		4723	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
		4724	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
		4725
		4726	/* 2a: Disable RC states. */
		4727	I915_WRITE(GEN6_RC_CONTROL, 0);
		4728
5354	serge	4729	/* Initialize rps frequencies */
		4730	gen6_init_rps_frequencies(dev);
4560	Serge	4731
		4732	/* 2b: Program RC6 thresholds.*/
		4733	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		4734	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4735	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4736	for_each_ring(ring, dev_priv, unused)
		4737	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4738	I915_WRITE(GEN6_RC_SLEEP, 0);
5060	serge	4739	if (IS_BROADWELL(dev))
		4740	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
		4741	else
4560	Serge	4742	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
		4743
		4744	/* 3: Enable RC6 */
		4745	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4746	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
5060	serge	4747	intel_print_rc6_info(dev, rc6_mask);
		4748	if (IS_BROADWELL(dev))
		4749	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4750	GEN7_RC_CTL_TO_MODE \|
		4751	rc6_mask);
		4752	else
4560	Serge	4753	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		4754	GEN6_RC_CTL_EI_MODE(1) \|
		4755	rc6_mask);
		4756
		4757	/* 4 Program defaults and thresholds for RPS*/
5060	serge	4758	I915_WRITE(GEN6_RPNSWREQ,
		4759	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
		4760	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		4761	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4560	Serge	4762	/* NB: Docs say 1s, and 1000000 - which aren't equivalent */
		4763	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
		4764
		4765	/* Docs recommend 900MHz, and 300 MHz respectively */
		4766	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
5060	serge	4767	dev_priv->rps.max_freq_softlimit << 24 \|
		4768	dev_priv->rps.min_freq_softlimit << 16);
4560	Serge	4769
		4770	I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
		4771	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
		4772	I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
		4773	I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
		4774
		4775	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		4776
		4777	/* 5: Enable RPS */
		4778	I915_WRITE(GEN6_RP_CONTROL,
		4779	GEN6_RP_MEDIA_TURBO \|
		4780	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4781	GEN6_RP_MEDIA_IS_GFX \|
		4782	GEN6_RP_ENABLE \|
		4783	GEN6_RP_UP_BUSY_AVG \|
		4784	GEN6_RP_DOWN_IDLE_AVG);
		4785
		4786	/* 6: Ring frequency + overclocking (our driver does this later */
		4787
5354	serge	4788	dev_priv->rps.power = HIGH_POWER; /* force a reset */
		4789	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4560	Serge	4790
		4791	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
		4792	}
		4793
3031	serge	4794	static void gen6_enable_rps(struct drm_device *dev)
		4795	{
		4796	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4797	struct intel_engine_cs *ring;
		4798	u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
3031	serge	4799	u32 gtfifodbg;
		4800	int rc6_mode;
3243	Serge	4801	int i, ret;
3031	serge	4802
3243	Serge	4803	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4804
		4805	/* Here begins a magic sequence of register writes to enable
		4806	* auto-downclocking.
		4807	*
		4808	* Perhaps there might be some value in exposing these to
		4809	* userspace...
		4810	*/
		4811	I915_WRITE(GEN6_RC_STATE, 0);
		4812
		4813	/* Clear the DBG now so we don't confuse earlier errors */
		4814	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		4815	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		4816	I915_WRITE(GTFIFODBG, gtfifodbg);
		4817	}
		4818
4560	Serge	4819	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
3031	serge	4820
5354	serge	4821	/* Initialize rps frequencies */
		4822	gen6_init_rps_frequencies(dev);
3031	serge	4823
		4824	/* disable the counters and set deterministic thresholds */
		4825	I915_WRITE(GEN6_RC_CONTROL, 0);
		4826
		4827	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		4828	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		4829	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		4830	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		4831	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		4832
		4833	for_each_ring(ring, dev_priv, i)
		4834	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4835
		4836	I915_WRITE(GEN6_RC_SLEEP, 0);
		4837	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4560	Serge	4838	if (IS_IVYBRIDGE(dev))
4104	Serge	4839	I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
		4840	else
3031	serge	4841	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	4842	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	4843	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		4844
		4845	/* Check if we are enabling RC6 */
		4846	rc6_mode = intel_enable_rc6(dev_priv->dev);
		4847	if (rc6_mode & INTEL_RC6_ENABLE)
		4848	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		4849
		4850	/* We don't use those on Haswell */
		4851	if (!IS_HASWELL(dev)) {
		4852	if (rc6_mode & INTEL_RC6p_ENABLE)
		4853	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		4854
		4855	if (rc6_mode & INTEL_RC6pp_ENABLE)
		4856	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		4857	}
		4858
4560	Serge	4859	intel_print_rc6_info(dev, rc6_mask);
3031	serge	4860
		4861	I915_WRITE(GEN6_RC_CONTROL,
		4862	rc6_mask \|
		4863	GEN6_RC_CTL_EI_MODE(1) \|
		4864	GEN6_RC_CTL_HW_ENABLE);
		4865
4560	Serge	4866	/* Power down if completely idle for over 50ms */
		4867	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
3031	serge	4868	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		4869
3243	Serge	4870	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
5060	serge	4871	if (ret)
		4872	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		4873
3243	Serge	4874	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
3746	Serge	4875	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		4876	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
5060	serge	4877	(dev_priv->rps.max_freq_softlimit & 0xff) * 50,
3746	Serge	4878	(pcu_mbox & 0xff) * 50);
5060	serge	4879	dev_priv->rps.max_freq = pcu_mbox & 0xff;
3031	serge	4880	}
		4881
4560	Serge	4882	dev_priv->rps.power = HIGH_POWER; /* force a reset */
5060	serge	4883	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
3031	serge	4884
3243	Serge	4885	rc6vids = 0;
		4886	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		4887	if (IS_GEN6(dev) && ret) {
		4888	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		4889	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		4890	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		4891	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		4892	rc6vids &= 0xffff00;
		4893	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		4894	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		4895	if (ret)
		4896	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		4897	}
		4898
4560	Serge	4899	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
3031	serge	4900	}
		4901
5060	serge	4902	static void __gen6_update_ring_freq(struct drm_device *dev)
3031	serge	4903	{
		4904	struct drm_i915_private *dev_priv = dev->dev_private;
		4905	int min_freq = 15;
3746	Serge	4906	unsigned int gpu_freq;
		4907	unsigned int max_ia_freq, min_ring_freq;
3031	serge	4908	int scaling_factor = 180;
4560	Serge	4909	struct cpufreq_policy *policy;
3031	serge	4910
3243	Serge	4911	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4912
		4913	max_ia_freq = cpufreq_quick_get_max(0);
		4914	/*
5060	serge	4915	* Default to measured freq if none found, PCU will ensure we
		4916	* don't go over
3031	serge	4917	*/
		4918	max_ia_freq = tsc_khz;
		4919
		4920	/* Convert from kHz to MHz */
		4921	max_ia_freq /= 1000;
		4922
4560	Serge	4923	min_ring_freq = I915_READ(DCLK) & 0xf;
		4924	/* convert DDR frequency from units of 266.6MHz to bandwidth */
		4925	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3746	Serge	4926
3031	serge	4927	/*
		4928	* For each potential GPU frequency, load a ring frequency we'd like
		4929	* to use for memory access. We do this by specifying the IA frequency
		4930	* the PCU should use as a reference to determine the ring frequency.
		4931	*/
5354	serge	4932	for (gpu_freq = dev_priv->rps.max_freq; gpu_freq >= dev_priv->rps.min_freq;
3031	serge	4933	gpu_freq--) {
5354	serge	4934	int diff = dev_priv->rps.max_freq - gpu_freq;
3746	Serge	4935	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	4936
4560	Serge	4937	if (INTEL_INFO(dev)->gen >= 8) {
		4938	/* max(2 * GT, DDR). NB: GT is 50MHz units */
		4939	ring_freq = max(min_ring_freq, gpu_freq);
		4940	} else if (IS_HASWELL(dev)) {
		4941	ring_freq = mult_frac(gpu_freq, 5, 4);
3746	Serge	4942	ring_freq = max(min_ring_freq, ring_freq);
		4943	/* leave ia_freq as the default, chosen by cpufreq */
		4944	} else {
		4945	/* On older processors, there is no separate ring
		4946	* clock domain, so in order to boost the bandwidth
		4947	* of the ring, we need to upclock the CPU (ia_freq).
		4948	*
		4949	* For GPU frequencies less than 750MHz,
		4950	* just use the lowest ring freq.
3031	serge	4951	*/
		4952	if (gpu_freq < min_freq)
		4953	ia_freq = 800;
		4954	else
		4955	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		4956	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	4957	}
3031	serge	4958
3243	Serge	4959	sandybridge_pcode_write(dev_priv,
		4960	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	4961	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		4962	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		4963	gpu_freq);
3031	serge	4964	}
		4965	}
		4966
5060	serge	4967	void gen6_update_ring_freq(struct drm_device *dev)
4104	Serge	4968	{
5060	serge	4969	struct drm_i915_private *dev_priv = dev->dev_private;
		4970
		4971	if (INTEL_INFO(dev)->gen < 6 \|\| IS_VALLEYVIEW(dev))
		4972	return;
		4973
		4974	mutex_lock(&dev_priv->rps.hw_lock);
		4975	__gen6_update_ring_freq(dev);
		4976	mutex_unlock(&dev_priv->rps.hw_lock);
		4977	}
		4978
		4979	static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
		4980	{
4104	Serge	4981	u32 val, rp0;
		4982
5060	serge	4983	val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
		4984	rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & PUNIT_GPU_STATUS_MAX_FREQ_MASK;
		4985
		4986	return rp0;
		4987	}
		4988
		4989	static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		4990	{
		4991	u32 val, rpe;
		4992
		4993	val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
		4994	rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
		4995
		4996	return rpe;
		4997	}
		4998
		4999	static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5000	{
		5001	u32 val, rp1;
		5002
		5003	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5004	rp1 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & PUNIT_GPU_STATUS_MAX_FREQ_MASK;
		5005
		5006	return rp1;
		5007	}
		5008
		5009	static int cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
		5010	{
		5011	u32 val, rpn;
		5012
		5013	val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
		5014	rpn = (val >> PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT) & PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK;
		5015	return rpn;
		5016	}
		5017
		5018	static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
		5019	{
		5020	u32 val, rp1;
		5021
4104	Serge	5022	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5023
5060	serge	5024	rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
		5025
		5026	return rp1;
		5027	}
		5028
		5029	static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
		5030	{
		5031	u32 val, rp0;
		5032
		5033	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		5034
4104	Serge	5035	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
		5036	/* Clamp to max */
		5037	rp0 = min_t(u32, rp0, 0xea);
		5038
		5039	return rp0;
		5040	}
		5041
		5042	static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		5043	{
		5044	u32 val, rpe;
		5045
		5046	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
		5047	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
		5048	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
		5049	rpe \|= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
		5050
		5051	return rpe;
		5052	}
		5053
5060	serge	5054	static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
4104	Serge	5055	{
		5056	return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
		5057	}
		5058
5060	serge	5059	/* Check that the pctx buffer wasn't move under us. */
		5060	static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
		5061	{
		5062	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5063
		5064	WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
		5065	dev_priv->vlv_pctx->stolen->start);
		5066	}
		5067
		5068
		5069	/* Check that the pcbr address is not empty. */
		5070	static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
		5071	{
		5072	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		5073
		5074	WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
		5075	}
		5076
		5077	static void cherryview_setup_pctx(struct drm_device *dev)
		5078	{
		5079	struct drm_i915_private *dev_priv = dev->dev_private;
		5080	unsigned long pctx_paddr, paddr;
		5081	struct i915_gtt *gtt = &dev_priv->gtt;
		5082	u32 pcbr;
		5083	int pctx_size = 32*1024;
		5084
		5085	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5086
		5087	pcbr = I915_READ(VLV_PCBR);
		5088	if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
5354	serge	5089	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
5060	serge	5090	paddr = (dev_priv->mm.stolen_base +
		5091	(gtt->stolen_size - pctx_size));
		5092
		5093	pctx_paddr = (paddr & (~4095));
		5094	I915_WRITE(VLV_PCBR, pctx_paddr);
		5095	}
5354	serge	5096
		5097	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
5060	serge	5098	}
		5099
4104	Serge	5100	static void valleyview_setup_pctx(struct drm_device *dev)
		5101	{
		5102	struct drm_i915_private *dev_priv = dev->dev_private;
		5103	struct drm_i915_gem_object *pctx;
		5104	unsigned long pctx_paddr;
		5105	u32 pcbr;
		5106	int pctx_size = 24*1024;
		5107
5060	serge	5108	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5109
4104	Serge	5110	pcbr = I915_READ(VLV_PCBR);
		5111	if (pcbr) {
		5112	/* BIOS set it up already, grab the pre-alloc'd space */
		5113	int pcbr_offset;
		5114
		5115	pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
		5116	pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
		5117	pcbr_offset,
		5118	I915_GTT_OFFSET_NONE,
		5119	pctx_size);
		5120	goto out;
		5121	}
		5122
5354	serge	5123	DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
		5124
4104	Serge	5125	/*
		5126	* From the Gunit register HAS:
		5127	* The Gfx driver is expected to program this register and ensure
		5128	* proper allocation within Gfx stolen memory. For example, this
		5129	* register should be programmed such than the PCBR range does not
		5130	* overlap with other ranges, such as the frame buffer, protected
		5131	* memory, or any other relevant ranges.
		5132	*/
		5133	pctx = i915_gem_object_create_stolen(dev, pctx_size);
		5134	if (!pctx) {
		5135	DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
		5136	return;
		5137	}
		5138
		5139	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
		5140	I915_WRITE(VLV_PCBR, pctx_paddr);
		5141
		5142	out:
5354	serge	5143	DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
4104	Serge	5144	dev_priv->vlv_pctx = pctx;
		5145	}
		5146
5060	serge	5147	static void valleyview_cleanup_pctx(struct drm_device *dev)
		5148	{
		5149	struct drm_i915_private *dev_priv = dev->dev_private;
		5150
		5151	if (WARN_ON(!dev_priv->vlv_pctx))
		5152	return;
		5153
		5154	drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
		5155	dev_priv->vlv_pctx = NULL;
		5156	}
		5157
		5158	static void valleyview_init_gt_powersave(struct drm_device *dev)
		5159	{
		5160	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5161	u32 val;
5060	serge	5162
		5163	valleyview_setup_pctx(dev);
		5164
		5165	mutex_lock(&dev_priv->rps.hw_lock);
		5166
5354	serge	5167	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5168	switch ((val >> 6) & 3) {
		5169	case 0:
		5170	case 1:
		5171	dev_priv->mem_freq = 800;
		5172	break;
		5173	case 2:
		5174	dev_priv->mem_freq = 1066;
		5175	break;
		5176	case 3:
		5177	dev_priv->mem_freq = 1333;
		5178	break;
		5179	}
		5180	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5181
5060	serge	5182	dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
		5183	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5184	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
		5185	vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq),
		5186	dev_priv->rps.max_freq);
		5187
		5188	dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
		5189	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
		5190	vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
		5191	dev_priv->rps.efficient_freq);
		5192
		5193	dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
		5194	DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
		5195	vlv_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
		5196	dev_priv->rps.rp1_freq);
		5197
		5198	dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
		5199	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
		5200	vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq),
		5201	dev_priv->rps.min_freq);
		5202
		5203	/* Preserve min/max settings in case of re-init */
		5204	if (dev_priv->rps.max_freq_softlimit == 0)
		5205	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5206
		5207	if (dev_priv->rps.min_freq_softlimit == 0)
		5208	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5209
		5210	mutex_unlock(&dev_priv->rps.hw_lock);
		5211	}
		5212
		5213	static void cherryview_init_gt_powersave(struct drm_device *dev)
		5214	{
		5215	struct drm_i915_private *dev_priv = dev->dev_private;
5354	serge	5216	u32 val;
5060	serge	5217
		5218	cherryview_setup_pctx(dev);
		5219
		5220	mutex_lock(&dev_priv->rps.hw_lock);
		5221
5354	serge	5222	mutex_lock(&dev_priv->dpio_lock);
		5223	val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
		5224	mutex_unlock(&dev_priv->dpio_lock);
		5225
		5226	switch ((val >> 2) & 0x7) {
		5227	case 0:
		5228	case 1:
		5229	dev_priv->rps.cz_freq = 200;
		5230	dev_priv->mem_freq = 1600;
		5231	break;
		5232	case 2:
		5233	dev_priv->rps.cz_freq = 267;
		5234	dev_priv->mem_freq = 1600;
		5235	break;
		5236	case 3:
		5237	dev_priv->rps.cz_freq = 333;
		5238	dev_priv->mem_freq = 2000;
		5239	break;
		5240	case 4:
		5241	dev_priv->rps.cz_freq = 320;
		5242	dev_priv->mem_freq = 1600;
		5243	break;
		5244	case 5:
		5245	dev_priv->rps.cz_freq = 400;
		5246	dev_priv->mem_freq = 1600;
		5247	break;
		5248	}
		5249	DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
		5250
5060	serge	5251	dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
		5252	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		5253	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
		5254	vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq),
		5255	dev_priv->rps.max_freq);
		5256
		5257	dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
		5258	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
		5259	vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
		5260	dev_priv->rps.efficient_freq);
		5261
		5262	dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
		5263	DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
		5264	vlv_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
		5265	dev_priv->rps.rp1_freq);
		5266
		5267	dev_priv->rps.min_freq = cherryview_rps_min_freq(dev_priv);
		5268	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
		5269	vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq),
		5270	dev_priv->rps.min_freq);
		5271
5354	serge	5272	WARN_ONCE((dev_priv->rps.max_freq \|
		5273	dev_priv->rps.efficient_freq \|
		5274	dev_priv->rps.rp1_freq \|
		5275	dev_priv->rps.min_freq) & 1,
		5276	"Odd GPU freq values\n");
		5277
5060	serge	5278	/* Preserve min/max settings in case of re-init */
		5279	if (dev_priv->rps.max_freq_softlimit == 0)
		5280	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		5281
		5282	if (dev_priv->rps.min_freq_softlimit == 0)
		5283	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		5284
		5285	mutex_unlock(&dev_priv->rps.hw_lock);
		5286	}
		5287
		5288	static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
		5289	{
		5290	valleyview_cleanup_pctx(dev);
		5291	}
		5292
		5293	static void cherryview_enable_rps(struct drm_device *dev)
		5294	{
		5295	struct drm_i915_private *dev_priv = dev->dev_private;
		5296	struct intel_engine_cs *ring;
		5297	u32 gtfifodbg, val, rc6_mode = 0, pcbr;
		5298	int i;
		5299
		5300	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5301
		5302	gtfifodbg = I915_READ(GTFIFODBG);
		5303	if (gtfifodbg) {
		5304	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5305	gtfifodbg);
		5306	I915_WRITE(GTFIFODBG, gtfifodbg);
		5307	}
		5308
		5309	cherryview_check_pctx(dev_priv);
		5310
		5311	/* 1a & 1b: Get forcewake during program sequence. Although the driver
		5312	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
		5313	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
		5314
		5315	/* 2a: Program RC6 thresholds.*/
		5316	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		5317	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		5318	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		5319
		5320	for_each_ring(ring, dev_priv, i)
		5321	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5322	I915_WRITE(GEN6_RC_SLEEP, 0);
		5323
		5324	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
		5325
		5326	/* allows RC6 residency counter to work */
		5327	I915_WRITE(VLV_COUNTER_CONTROL,
		5328	_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH \|
		5329	VLV_MEDIA_RC6_COUNT_EN \|
		5330	VLV_RENDER_RC6_COUNT_EN));
		5331
		5332	/* For now we assume BIOS is allocating and populating the PCBR */
		5333	pcbr = I915_READ(VLV_PCBR);
		5334
		5335	/* 3: Enable RC6 */
		5336	if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
		5337	(pcbr >> VLV_PCBR_ADDR_SHIFT))
		5338	rc6_mode = GEN6_RC_CTL_EI_MODE(1);
		5339
		5340	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5341
		5342	/* 4 Program defaults and thresholds for RPS*/
		5343	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5344	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5345	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5346	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5347
		5348	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		5349
		5350	/* WaDisablePwrmtrEvent:chv (pre-production hw) */
		5351	I915_WRITE(0xA80C, I915_READ(0xA80C) & 0x00ffffff);
		5352	I915_WRITE(0xA810, I915_READ(0xA810) & 0xffffff00);
		5353
		5354	/* 5: Enable RPS */
		5355	I915_WRITE(GEN6_RP_CONTROL,
		5356	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		5357	GEN6_RP_MEDIA_IS_GFX \| /* WaSetMaskForGfxBusyness:chv (pre-production hw ?) */
		5358	GEN6_RP_ENABLE \|
		5359	GEN6_RP_UP_BUSY_AVG \|
		5360	GEN6_RP_DOWN_IDLE_AVG);
		5361
		5362	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5363
5354	serge	5364	/* RPS code assumes GPLL is used */
		5365	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5366
		5367	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no");
5060	serge	5368	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5369
		5370	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
		5371	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
		5372	vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
		5373	dev_priv->rps.cur_freq);
		5374
		5375	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
		5376	vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
		5377	dev_priv->rps.efficient_freq);
		5378
		5379	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
		5380
		5381	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
		5382	}
		5383
4104	Serge	5384	static void valleyview_enable_rps(struct drm_device *dev)
		5385	{
		5386	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	5387	struct intel_engine_cs *ring;
4560	Serge	5388	u32 gtfifodbg, val, rc6_mode = 0;
4104	Serge	5389	int i;
		5390
		5391	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5392
5060	serge	5393	valleyview_check_pctx(dev_priv);
		5394
4104	Serge	5395	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4560	Serge	5396	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		5397	gtfifodbg);
4104	Serge	5398	I915_WRITE(GTFIFODBG, gtfifodbg);
		5399	}
		5400
4560	Serge	5401	/* If VLV, Forcewake all wells, else re-direct to regular path */
		5402	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
4104	Serge	5403
		5404	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		5405	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		5406	I915_WRITE(GEN6_RP_UP_EI, 66000);
		5407	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		5408
		5409	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5060	serge	5410	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 0xf4240);
4104	Serge	5411
		5412	I915_WRITE(GEN6_RP_CONTROL,
		5413	GEN6_RP_MEDIA_TURBO \|
		5414	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		5415	GEN6_RP_MEDIA_IS_GFX \|
		5416	GEN6_RP_ENABLE \|
		5417	GEN6_RP_UP_BUSY_AVG \|
		5418	GEN6_RP_DOWN_IDLE_CONT);
		5419
		5420	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
		5421	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		5422	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		5423
		5424	for_each_ring(ring, dev_priv, i)
		5425	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		5426
4560	Serge	5427	I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4104	Serge	5428
		5429	/* allows RC6 residency counter to work */
4560	Serge	5430	I915_WRITE(VLV_COUNTER_CONTROL,
5060	serge	5431	_MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN \|
		5432	VLV_RENDER_RC0_COUNT_EN \|
4560	Serge	5433	VLV_MEDIA_RC6_COUNT_EN \|
		5434	VLV_RENDER_RC6_COUNT_EN));
5060	serge	5435
4560	Serge	5436	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		5437	rc6_mode = GEN7_RC_CTL_TO_MODE \| VLV_RC_CTL_CTX_RST_PARALLEL;
4104	Serge	5438
4560	Serge	5439	intel_print_rc6_info(dev, rc6_mode);
		5440
		5441	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		5442
4104	Serge	5443	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5444
5354	serge	5445	/* RPS code assumes GPLL is used */
		5446	WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
		5447
		5448	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no");
4104	Serge	5449	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		5450
5060	serge	5451	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4104	Serge	5452	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
5060	serge	5453	vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
		5454	dev_priv->rps.cur_freq);
4104	Serge	5455
		5456	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
5060	serge	5457	vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
		5458	dev_priv->rps.efficient_freq);
4104	Serge	5459
5060	serge	5460	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4104	Serge	5461
4560	Serge	5462	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	5463	}
		5464
3031	serge	5465	void ironlake_teardown_rc6(struct drm_device *dev)
		5466	{
		5467	struct drm_i915_private *dev_priv = dev->dev_private;
		5468
3243	Serge	5469	if (dev_priv->ips.renderctx) {
5060	serge	5470	i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx);
3243	Serge	5471	drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
		5472	dev_priv->ips.renderctx = NULL;
3031	serge	5473	}
		5474
3243	Serge	5475	if (dev_priv->ips.pwrctx) {
5060	serge	5476	i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx);
3243	Serge	5477	drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
		5478	dev_priv->ips.pwrctx = NULL;
3031	serge	5479	}
		5480	}
		5481
		5482	static void ironlake_disable_rc6(struct drm_device *dev)
		5483	{
		5484	struct drm_i915_private *dev_priv = dev->dev_private;
		5485
		5486	if (I915_READ(PWRCTXA)) {
		5487	/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
		5488	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) \| RCX_SW_EXIT);
		5489	wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
		5490	50);
		5491
		5492	I915_WRITE(PWRCTXA, 0);
		5493	POSTING_READ(PWRCTXA);
		5494
		5495	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		5496	POSTING_READ(RSTDBYCTL);
		5497	}
		5498	}
		5499
		5500	static int ironlake_setup_rc6(struct drm_device *dev)
		5501	{
		5502	struct drm_i915_private *dev_priv = dev->dev_private;
		5503
3243	Serge	5504	if (dev_priv->ips.renderctx == NULL)
		5505	dev_priv->ips.renderctx = intel_alloc_context_page(dev);
		5506	if (!dev_priv->ips.renderctx)
3031	serge	5507	return -ENOMEM;
		5508
3243	Serge	5509	if (dev_priv->ips.pwrctx == NULL)
		5510	dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
		5511	if (!dev_priv->ips.pwrctx) {
3031	serge	5512	ironlake_teardown_rc6(dev);
		5513	return -ENOMEM;
		5514	}
		5515
		5516	return 0;
		5517	}
		5518
		5519	static void ironlake_enable_rc6(struct drm_device *dev)
		5520	{
		5521	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	5522	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
3243	Serge	5523	bool was_interruptible;
3031	serge	5524	int ret;
		5525
		5526	/* rc6 disabled by default due to repeated reports of hanging during
		5527	* boot and resume.
		5528	*/
		5529	if (!intel_enable_rc6(dev))
		5530	return;
		5531
		5532	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		5533
		5534	ret = ironlake_setup_rc6(dev);
		5535	if (ret)
		5536	return;
		5537
3243	Serge	5538	was_interruptible = dev_priv->mm.interruptible;
		5539	dev_priv->mm.interruptible = false;
		5540
3031	serge	5541	/*
		5542	* GPU can automatically power down the render unit if given a page
		5543	* to save state.
		5544	*/
		5545	ret = intel_ring_begin(ring, 6);
		5546	if (ret) {
		5547	ironlake_teardown_rc6(dev);
3243	Serge	5548	dev_priv->mm.interruptible = was_interruptible;
3031	serge	5549	return;
		5550	}
		5551
		5552	intel_ring_emit(ring, MI_SUSPEND_FLUSH \| MI_SUSPEND_FLUSH_EN);
		5553	intel_ring_emit(ring, MI_SET_CONTEXT);
4104	Serge	5554	intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) \|
3031	serge	5555	MI_MM_SPACE_GTT \|
		5556	MI_SAVE_EXT_STATE_EN \|
		5557	MI_RESTORE_EXT_STATE_EN \|
		5558	MI_RESTORE_INHIBIT);
		5559	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
		5560	intel_ring_emit(ring, MI_NOOP);
		5561	intel_ring_emit(ring, MI_FLUSH);
		5562	intel_ring_advance(ring);
		5563
		5564	/*
		5565	* Wait for the command parser to advance past MI_SET_CONTEXT. The HW
		5566	* does an implicit flush, combined with MI_FLUSH above, it should be
		5567	* safe to assume that renderctx is valid
		5568	*/
3243	Serge	5569	ret = intel_ring_idle(ring);
		5570	dev_priv->mm.interruptible = was_interruptible;
3031	serge	5571	if (ret) {
3746	Serge	5572	DRM_ERROR("failed to enable ironlake power savings\n");
3031	serge	5573	ironlake_teardown_rc6(dev);
		5574	return;
		5575	}
		5576
4104	Serge	5577	I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) \| PWRCTX_EN);
3031	serge	5578	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
4560	Serge	5579
5060	serge	5580	intel_print_rc6_info(dev, GEN6_RC_CTL_RC6_ENABLE);
3031	serge	5581	}
		5582
		5583	static unsigned long intel_pxfreq(u32 vidfreq)
		5584	{
		5585	unsigned long freq;
		5586	int div = (vidfreq & 0x3f0000) >> 16;
		5587	int post = (vidfreq & 0x3000) >> 12;
		5588	int pre = (vidfreq & 0x7);
		5589
		5590	if (!pre)
		5591	return 0;
		5592
		5593	freq = ((div * 133333) / ((1<
		5594
		5595	return freq;
		5596	}
		5597
		5598	static const struct cparams {
		5599	u16 i;
		5600	u16 t;
		5601	u16 m;
		5602	u16 c;
		5603	} cparams[] = {
		5604	{ 1, 1333, 301, 28664 },
		5605	{ 1, 1066, 294, 24460 },
		5606	{ 1, 800, 294, 25192 },
		5607	{ 0, 1333, 276, 27605 },
		5608	{ 0, 1066, 276, 27605 },
		5609	{ 0, 800, 231, 23784 },
		5610	};
		5611
		5612	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		5613	{
		5614	u64 total_count, diff, ret;
		5615	u32 count1, count2, count3, m = 0, c = 0;
5060	serge	5616	unsigned long now = jiffies_to_msecs(jiffies), diff1;
3031	serge	5617	int i;
		5618
		5619	assert_spin_locked(&mchdev_lock);
		5620
		5621	diff1 = now - dev_priv->ips.last_time1;
		5622
		5623	/* Prevent division-by-zero if we are asking too fast.
		5624	* Also, we don't get interesting results if we are polling
		5625	* faster than once in 10ms, so just return the saved value
		5626	* in such cases.
		5627	*/
		5628	if (diff1 <= 10)
		5629	return dev_priv->ips.chipset_power;
		5630
		5631	count1 = I915_READ(DMIEC);
		5632	count2 = I915_READ(DDREC);
		5633	count3 = I915_READ(CSIEC);
		5634
		5635	total_count = count1 + count2 + count3;
		5636
		5637	/* FIXME: handle per-counter overflow */
		5638	if (total_count < dev_priv->ips.last_count1) {
		5639	diff = ~0UL - dev_priv->ips.last_count1;
		5640	diff += total_count;
		5641	} else {
		5642	diff = total_count - dev_priv->ips.last_count1;
		5643	}
		5644
		5645	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		5646	if (cparams[i].i == dev_priv->ips.c_m &&
		5647	cparams[i].t == dev_priv->ips.r_t) {
		5648	m = cparams[i].m;
		5649	c = cparams[i].c;
		5650	break;
		5651	}
		5652	}
		5653
		5654	diff = div_u64(diff, diff1);
		5655	ret = ((m * diff) + c);
		5656	ret = div_u64(ret, 10);
		5657
		5658	dev_priv->ips.last_count1 = total_count;
		5659	dev_priv->ips.last_time1 = now;
		5660
		5661	dev_priv->ips.chipset_power = ret;
		5662
		5663	return ret;
		5664	}
		5665
		5666	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		5667	{
5060	serge	5668	struct drm_device *dev = dev_priv->dev;
3031	serge	5669	unsigned long val;
		5670
5060	serge	5671	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5672	return 0;
		5673
		5674	spin_lock_irq(&mchdev_lock);
		5675
		5676	val = __i915_chipset_val(dev_priv);
		5677
		5678	spin_unlock_irq(&mchdev_lock);
		5679
		5680	return val;
		5681	}
		5682
		5683	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		5684	{
		5685	unsigned long m, x, b;
		5686	u32 tsfs;
		5687
		5688	tsfs = I915_READ(TSFS);
		5689
		5690	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		5691	x = I915_READ8(TR1);
		5692
		5693	b = tsfs & TSFS_INTR_MASK;
		5694
		5695	return ((m * x) / 127) - b;
		5696	}
		5697
		5698	static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		5699	{
5060	serge	5700	struct drm_device *dev = dev_priv->dev;
3031	serge	5701	static const struct v_table {
		5702	u16 vd; /* in .1 mil */
		5703	u16 vm; /* in .1 mil */
		5704	} v_table[] = {
		5705	{ 0, 0, },
		5706	{ 375, 0, },
		5707	{ 500, 0, },
		5708	{ 625, 0, },
		5709	{ 750, 0, },
		5710	{ 875, 0, },
		5711	{ 1000, 0, },
		5712	{ 1125, 0, },
		5713	{ 4125, 3000, },
		5714	{ 4125, 3000, },
		5715	{ 4125, 3000, },
		5716	{ 4125, 3000, },
		5717	{ 4125, 3000, },
		5718	{ 4125, 3000, },
		5719	{ 4125, 3000, },
		5720	{ 4125, 3000, },
		5721	{ 4125, 3000, },
		5722	{ 4125, 3000, },
		5723	{ 4125, 3000, },
		5724	{ 4125, 3000, },
		5725	{ 4125, 3000, },
		5726	{ 4125, 3000, },
		5727	{ 4125, 3000, },
		5728	{ 4125, 3000, },
		5729	{ 4125, 3000, },
		5730	{ 4125, 3000, },
		5731	{ 4125, 3000, },
		5732	{ 4125, 3000, },
		5733	{ 4125, 3000, },
		5734	{ 4125, 3000, },
		5735	{ 4125, 3000, },
		5736	{ 4125, 3000, },
		5737	{ 4250, 3125, },
		5738	{ 4375, 3250, },
		5739	{ 4500, 3375, },
		5740	{ 4625, 3500, },
		5741	{ 4750, 3625, },
		5742	{ 4875, 3750, },
		5743	{ 5000, 3875, },
		5744	{ 5125, 4000, },
		5745	{ 5250, 4125, },
		5746	{ 5375, 4250, },
		5747	{ 5500, 4375, },
		5748	{ 5625, 4500, },
		5749	{ 5750, 4625, },
		5750	{ 5875, 4750, },
		5751	{ 6000, 4875, },
		5752	{ 6125, 5000, },
		5753	{ 6250, 5125, },
		5754	{ 6375, 5250, },
		5755	{ 6500, 5375, },
		5756	{ 6625, 5500, },
		5757	{ 6750, 5625, },
		5758	{ 6875, 5750, },
		5759	{ 7000, 5875, },
		5760	{ 7125, 6000, },
		5761	{ 7250, 6125, },
		5762	{ 7375, 6250, },
		5763	{ 7500, 6375, },
		5764	{ 7625, 6500, },
		5765	{ 7750, 6625, },
		5766	{ 7875, 6750, },
		5767	{ 8000, 6875, },
		5768	{ 8125, 7000, },
		5769	{ 8250, 7125, },
		5770	{ 8375, 7250, },
		5771	{ 8500, 7375, },
		5772	{ 8625, 7500, },
		5773	{ 8750, 7625, },
		5774	{ 8875, 7750, },
		5775	{ 9000, 7875, },
		5776	{ 9125, 8000, },
		5777	{ 9250, 8125, },
		5778	{ 9375, 8250, },
		5779	{ 9500, 8375, },
		5780	{ 9625, 8500, },
		5781	{ 9750, 8625, },
		5782	{ 9875, 8750, },
		5783	{ 10000, 8875, },
		5784	{ 10125, 9000, },
		5785	{ 10250, 9125, },
		5786	{ 10375, 9250, },
		5787	{ 10500, 9375, },
		5788	{ 10625, 9500, },
		5789	{ 10750, 9625, },
		5790	{ 10875, 9750, },
		5791	{ 11000, 9875, },
		5792	{ 11125, 10000, },
		5793	{ 11250, 10125, },
		5794	{ 11375, 10250, },
		5795	{ 11500, 10375, },
		5796	{ 11625, 10500, },
		5797	{ 11750, 10625, },
		5798	{ 11875, 10750, },
		5799	{ 12000, 10875, },
		5800	{ 12125, 11000, },
		5801	{ 12250, 11125, },
		5802	{ 12375, 11250, },
		5803	{ 12500, 11375, },
		5804	{ 12625, 11500, },
		5805	{ 12750, 11625, },
		5806	{ 12875, 11750, },
		5807	{ 13000, 11875, },
		5808	{ 13125, 12000, },
		5809	{ 13250, 12125, },
		5810	{ 13375, 12250, },
		5811	{ 13500, 12375, },
		5812	{ 13625, 12500, },
		5813	{ 13750, 12625, },
		5814	{ 13875, 12750, },
		5815	{ 14000, 12875, },
		5816	{ 14125, 13000, },
		5817	{ 14250, 13125, },
		5818	{ 14375, 13250, },
		5819	{ 14500, 13375, },
		5820	{ 14625, 13500, },
		5821	{ 14750, 13625, },
		5822	{ 14875, 13750, },
		5823	{ 15000, 13875, },
		5824	{ 15125, 14000, },
		5825	{ 15250, 14125, },
		5826	{ 15375, 14250, },
		5827	{ 15500, 14375, },
		5828	{ 15625, 14500, },
		5829	{ 15750, 14625, },
		5830	{ 15875, 14750, },
		5831	{ 16000, 14875, },
		5832	{ 16125, 15000, },
		5833	};
5060	serge	5834	if (INTEL_INFO(dev)->is_mobile)
3031	serge	5835	return v_table[pxvid].vm;
		5836	else
		5837	return v_table[pxvid].vd;
		5838	}
		5839
		5840	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5841	{
5060	serge	5842	u64 now, diff, diffms;
3031	serge	5843	u32 count;
		5844
		5845	assert_spin_locked(&mchdev_lock);
		5846
5060	serge	5847	now = ktime_get_raw_ns();
		5848	diffms = now - dev_priv->ips.last_time2;
		5849	do_div(diffms, NSEC_PER_MSEC);
3031	serge	5850
		5851	/* Don't divide by 0 */
		5852	if (!diffms)
		5853	return;
		5854
		5855	count = I915_READ(GFXEC);
		5856
		5857	if (count < dev_priv->ips.last_count2) {
		5858	diff = ~0UL - dev_priv->ips.last_count2;
		5859	diff += count;
		5860	} else {
		5861	diff = count - dev_priv->ips.last_count2;
		5862	}
		5863
		5864	dev_priv->ips.last_count2 = count;
		5865	dev_priv->ips.last_time2 = now;
		5866
		5867	/* More magic constants... */
		5868	diff = diff * 1181;
		5869	diff = div_u64(diff, diffms * 10);
		5870	dev_priv->ips.gfx_power = diff;
		5871	}
		5872
		5873	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		5874	{
5060	serge	5875	struct drm_device *dev = dev_priv->dev;
		5876
		5877	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5878	return;
		5879
		5880	spin_lock_irq(&mchdev_lock);
		5881
		5882	__i915_update_gfx_val(dev_priv);
		5883
		5884	spin_unlock_irq(&mchdev_lock);
		5885	}
		5886
		5887	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		5888	{
		5889	unsigned long t, corr, state1, corr2, state2;
		5890	u32 pxvid, ext_v;
		5891
		5892	assert_spin_locked(&mchdev_lock);
		5893
5060	serge	5894	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4));
3031	serge	5895	pxvid = (pxvid >> 24) & 0x7f;
		5896	ext_v = pvid_to_extvid(dev_priv, pxvid);
		5897
		5898	state1 = ext_v;
		5899
		5900	t = i915_mch_val(dev_priv);
		5901
		5902	/* Revel in the empirically derived constants */
		5903
		5904	/* Correction factor in 1/100000 units */
		5905	if (t > 80)
		5906	corr = ((t * 2349) + 135940);
		5907	else if (t >= 50)
		5908	corr = ((t * 964) + 29317);
		5909	else /* < 50 */
		5910	corr = ((t * 301) + 1004);
		5911
		5912	corr = corr * ((150142 * state1) / 10000 - 78642);
		5913	corr /= 100000;
		5914	corr2 = (corr * dev_priv->ips.corr);
		5915
		5916	state2 = (corr2 * state1) / 10000;
		5917	state2 /= 100; /* convert to mW */
		5918
		5919	__i915_update_gfx_val(dev_priv);
		5920
		5921	return dev_priv->ips.gfx_power + state2;
		5922	}
		5923
		5924	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		5925	{
5060	serge	5926	struct drm_device *dev = dev_priv->dev;
3031	serge	5927	unsigned long val;
		5928
5060	serge	5929	if (INTEL_INFO(dev)->gen != 5)
3031	serge	5930	return 0;
		5931
		5932	spin_lock_irq(&mchdev_lock);
		5933
		5934	val = __i915_gfx_val(dev_priv);
		5935
		5936	spin_unlock_irq(&mchdev_lock);
		5937
		5938	return val;
		5939	}
		5940
		5941	/**
		5942	* i915_read_mch_val - return value for IPS use
		5943	*
		5944	* Calculate and return a value for the IPS driver to use when deciding whether
		5945	* we have thermal and power headroom to increase CPU or GPU power budget.
		5946	*/
		5947	unsigned long i915_read_mch_val(void)
		5948	{
		5949	struct drm_i915_private *dev_priv;
		5950	unsigned long chipset_val, graphics_val, ret = 0;
		5951
		5952	spin_lock_irq(&mchdev_lock);
		5953	if (!i915_mch_dev)
		5954	goto out_unlock;
		5955	dev_priv = i915_mch_dev;
		5956
		5957	chipset_val = __i915_chipset_val(dev_priv);
		5958	graphics_val = __i915_gfx_val(dev_priv);
		5959
		5960	ret = chipset_val + graphics_val;
		5961
		5962	out_unlock:
		5963	spin_unlock_irq(&mchdev_lock);
		5964
		5965	return ret;
		5966	}
		5967	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		5968
		5969	/**
		5970	* i915_gpu_raise - raise GPU frequency limit
		5971	*
		5972	* Raise the limit; IPS indicates we have thermal headroom.
		5973	*/
		5974	bool i915_gpu_raise(void)
		5975	{
		5976	struct drm_i915_private *dev_priv;
		5977	bool ret = true;
		5978
		5979	spin_lock_irq(&mchdev_lock);
		5980	if (!i915_mch_dev) {
		5981	ret = false;
		5982	goto out_unlock;
		5983	}
		5984	dev_priv = i915_mch_dev;
		5985
		5986	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		5987	dev_priv->ips.max_delay--;
		5988
		5989	out_unlock:
		5990	spin_unlock_irq(&mchdev_lock);
		5991
		5992	return ret;
		5993	}
		5994	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		5995
		5996	/**
		5997	* i915_gpu_lower - lower GPU frequency limit
		5998	*
		5999	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		6000	* frequency maximum.
		6001	*/
		6002	bool i915_gpu_lower(void)
		6003	{
		6004	struct drm_i915_private *dev_priv;
		6005	bool ret = true;
		6006
		6007	spin_lock_irq(&mchdev_lock);
		6008	if (!i915_mch_dev) {
		6009	ret = false;
		6010	goto out_unlock;
		6011	}
		6012	dev_priv = i915_mch_dev;
		6013
		6014	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		6015	dev_priv->ips.max_delay++;
		6016
		6017	out_unlock:
		6018	spin_unlock_irq(&mchdev_lock);
		6019
		6020	return ret;
		6021	}
		6022	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		6023
		6024	/**
		6025	* i915_gpu_busy - indicate GPU business to IPS
		6026	*
		6027	* Tell the IPS driver whether or not the GPU is busy.
		6028	*/
		6029	bool i915_gpu_busy(void)
		6030	{
		6031	struct drm_i915_private *dev_priv;
5060	serge	6032	struct intel_engine_cs *ring;
3031	serge	6033	bool ret = false;
		6034	int i;
		6035
		6036	spin_lock_irq(&mchdev_lock);
		6037	if (!i915_mch_dev)
		6038	goto out_unlock;
		6039	dev_priv = i915_mch_dev;
		6040
		6041	for_each_ring(ring, dev_priv, i)
		6042	ret \|= !list_empty(&ring->request_list);
		6043
		6044	out_unlock:
		6045	spin_unlock_irq(&mchdev_lock);
		6046
		6047	return ret;
		6048	}
		6049	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		6050
		6051	/**
		6052	* i915_gpu_turbo_disable - disable graphics turbo
		6053	*
		6054	* Disable graphics turbo by resetting the max frequency and setting the
		6055	* current frequency to the default.
		6056	*/
		6057	bool i915_gpu_turbo_disable(void)
		6058	{
		6059	struct drm_i915_private *dev_priv;
		6060	bool ret = true;
		6061
		6062	spin_lock_irq(&mchdev_lock);
		6063	if (!i915_mch_dev) {
		6064	ret = false;
		6065	goto out_unlock;
		6066	}
		6067	dev_priv = i915_mch_dev;
		6068
		6069	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		6070
		6071	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		6072	ret = false;
		6073
		6074	out_unlock:
		6075	spin_unlock_irq(&mchdev_lock);
		6076
		6077	return ret;
		6078	}
		6079	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		6080
		6081	/**
		6082	* Tells the intel_ips driver that the i915 driver is now loaded, if
		6083	* IPS got loaded first.
		6084	*
		6085	* This awkward dance is so that neither module has to depend on the
		6086	* other in order for IPS to do the appropriate communication of
		6087	* GPU turbo limits to i915.
		6088	*/
		6089	static void
		6090	ips_ping_for_i915_load(void)
		6091	{
		6092	void (*link)(void);
		6093
		6094	// link = symbol_get(ips_link_to_i915_driver);
		6095	// if (link) {
		6096	// link();
		6097	// symbol_put(ips_link_to_i915_driver);
		6098	// }
		6099	}
		6100
		6101	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		6102	{
		6103	/* We only register the i915 ips part with intel-ips once everything is
		6104	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		6105	spin_lock_irq(&mchdev_lock);
		6106	i915_mch_dev = dev_priv;
		6107	spin_unlock_irq(&mchdev_lock);
		6108
		6109	ips_ping_for_i915_load();
		6110	}
		6111
		6112	void intel_gpu_ips_teardown(void)
		6113	{
		6114	spin_lock_irq(&mchdev_lock);
		6115	i915_mch_dev = NULL;
		6116	spin_unlock_irq(&mchdev_lock);
		6117	}
5060	serge	6118
3031	serge	6119	static void intel_init_emon(struct drm_device *dev)
		6120	{
		6121	struct drm_i915_private *dev_priv = dev->dev_private;
		6122	u32 lcfuse;
		6123	u8 pxw[16];
		6124	int i;
		6125
		6126	/* Disable to program */
		6127	I915_WRITE(ECR, 0);
		6128	POSTING_READ(ECR);
		6129
		6130	/* Program energy weights for various events */
		6131	I915_WRITE(SDEW, 0x15040d00);
		6132	I915_WRITE(CSIEW0, 0x007f0000);
		6133	I915_WRITE(CSIEW1, 0x1e220004);
		6134	I915_WRITE(CSIEW2, 0x04000004);
		6135
		6136	for (i = 0; i < 5; i++)
		6137	I915_WRITE(PEW + (i * 4), 0);
		6138	for (i = 0; i < 3; i++)
		6139	I915_WRITE(DEW + (i * 4), 0);
		6140
		6141	/* Program P-state weights to account for frequency power adjustment */
		6142	for (i = 0; i < 16; i++) {
		6143	u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
		6144	unsigned long freq = intel_pxfreq(pxvidfreq);
		6145	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		6146	PXVFREQ_PX_SHIFT;
		6147	unsigned long val;
		6148
		6149	val = vid * vid;
		6150	val *= (freq / 1000);
		6151	val *= 255;
		6152	val /= (127127900);
		6153	if (val > 0xff)
		6154	DRM_ERROR("bad pxval: %ld\n", val);
		6155	pxw[i] = val;
		6156	}
		6157	/* Render standby states get 0 weight */
		6158	pxw[14] = 0;
		6159	pxw[15] = 0;
		6160
		6161	for (i = 0; i < 4; i++) {
		6162	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		6163	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
		6164	I915_WRITE(PXW + (i * 4), val);
		6165	}
		6166
		6167	/* Adjust magic regs to magic values (more experimental results) */
		6168	I915_WRITE(OGW0, 0);
		6169	I915_WRITE(OGW1, 0);
		6170	I915_WRITE(EG0, 0x00007f00);
		6171	I915_WRITE(EG1, 0x0000000e);
		6172	I915_WRITE(EG2, 0x000e0000);
		6173	I915_WRITE(EG3, 0x68000300);
		6174	I915_WRITE(EG4, 0x42000000);
		6175	I915_WRITE(EG5, 0x00140031);
		6176	I915_WRITE(EG6, 0);
		6177	I915_WRITE(EG7, 0);
		6178
		6179	for (i = 0; i < 8; i++)
		6180	I915_WRITE(PXWL + (i * 4), 0);
		6181
		6182	/* Enable PMON + select events */
		6183	I915_WRITE(ECR, 0x80000019);
		6184
		6185	lcfuse = I915_READ(LCFUSE02);
		6186
		6187	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		6188	}
		6189
5060	serge	6190	void intel_init_gt_powersave(struct drm_device *dev)
		6191	{
		6192	i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
		6193
		6194	if (IS_CHERRYVIEW(dev))
		6195	cherryview_init_gt_powersave(dev);
		6196	else if (IS_VALLEYVIEW(dev))
		6197	valleyview_init_gt_powersave(dev);
		6198	}
		6199
		6200	void intel_cleanup_gt_powersave(struct drm_device *dev)
		6201	{
		6202	if (IS_CHERRYVIEW(dev))
		6203	return;
		6204	else if (IS_VALLEYVIEW(dev))
		6205	valleyview_cleanup_gt_powersave(dev);
		6206	}
		6207
5354	serge	6208	static void gen6_suspend_rps(struct drm_device *dev)
		6209	{
		6210	struct drm_i915_private *dev_priv = dev->dev_private;
		6211
		6212	// flush_delayed_work(&dev_priv->rps.delayed_resume_work);
		6213
		6214	/*
		6215	* TODO: disable RPS interrupts on GEN9+ too once RPS support
		6216	* is added for it.
		6217	*/
		6218	if (INTEL_INFO(dev)->gen < 9)
		6219	gen6_disable_rps_interrupts(dev);
		6220	}
		6221
5060	serge	6222	/**
		6223	* intel_suspend_gt_powersave - suspend PM work and helper threads
		6224	* @dev: drm device
		6225	*
		6226	* We don't want to disable RC6 or other features here, we just want
		6227	* to make sure any work we've queued has finished and won't bother
		6228	* us while we're suspended.
		6229	*/
		6230	void intel_suspend_gt_powersave(struct drm_device *dev)
		6231	{
		6232	struct drm_i915_private *dev_priv = dev->dev_private;
		6233
5354	serge	6234	if (INTEL_INFO(dev)->gen < 6)
		6235	return;
5060	serge	6236
5354	serge	6237	gen6_suspend_rps(dev);
5060	serge	6238
		6239	/* Force GPU to min freq during suspend */
		6240	gen6_rps_idle(dev_priv);
		6241	}
		6242
3031	serge	6243	void intel_disable_gt_powersave(struct drm_device *dev)
		6244	{
3243	Serge	6245	struct drm_i915_private *dev_priv = dev->dev_private;
		6246
3031	serge	6247	if (IS_IRONLAKE_M(dev)) {
		6248	ironlake_disable_drps(dev);
		6249	ironlake_disable_rc6(dev);
4293	Serge	6250	} else if (INTEL_INFO(dev)->gen >= 6) {
5060	serge	6251	intel_suspend_gt_powersave(dev);
		6252
3482	Serge	6253	mutex_lock(&dev_priv->rps.hw_lock);
5354	serge	6254	if (INTEL_INFO(dev)->gen >= 9)
		6255	gen9_disable_rps(dev);
		6256	else if (IS_CHERRYVIEW(dev))
5060	serge	6257	cherryview_disable_rps(dev);
		6258	else if (IS_VALLEYVIEW(dev))
4104	Serge	6259	valleyview_disable_rps(dev);
		6260	else
3031	serge	6261	gen6_disable_rps(dev);
5354	serge	6262
4560	Serge	6263	dev_priv->rps.enabled = false;
3480	Serge	6264	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	6265	}
		6266	}
		6267
3482	Serge	6268	static void intel_gen6_powersave_work(struct work_struct *work)
		6269	{
		6270	struct drm_i915_private *dev_priv =
		6271	container_of(work, struct drm_i915_private,
		6272	rps.delayed_resume_work.work);
		6273	struct drm_device *dev = dev_priv->dev;
		6274
		6275	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	6276
5354	serge	6277	/*
		6278	* TODO: reset/enable RPS interrupts on GEN9+ too, once RPS support is
		6279	* added for it.
		6280	*/
		6281	if (INTEL_INFO(dev)->gen < 9)
		6282	gen6_reset_rps_interrupts(dev);
		6283
5060	serge	6284	if (IS_CHERRYVIEW(dev)) {
		6285	cherryview_enable_rps(dev);
		6286	} else if (IS_VALLEYVIEW(dev)) {
4104	Serge	6287	valleyview_enable_rps(dev);
5354	serge	6288	} else if (INTEL_INFO(dev)->gen >= 9) {
		6289	gen9_enable_rps(dev);
4560	Serge	6290	} else if (IS_BROADWELL(dev)) {
		6291	gen8_enable_rps(dev);
5060	serge	6292	__gen6_update_ring_freq(dev);
4104	Serge	6293	} else {
3482	Serge	6294	gen6_enable_rps(dev);
5060	serge	6295	__gen6_update_ring_freq(dev);
4104	Serge	6296	}
4560	Serge	6297	dev_priv->rps.enabled = true;
5354	serge	6298
		6299	if (INTEL_INFO(dev)->gen < 9)
		6300	gen6_enable_rps_interrupts(dev);
		6301
3482	Serge	6302	mutex_unlock(&dev_priv->rps.hw_lock);
5060	serge	6303
		6304	intel_runtime_pm_put(dev_priv);
3482	Serge	6305	}
		6306
3031	serge	6307	void intel_enable_gt_powersave(struct drm_device *dev)
		6308	{
3243	Serge	6309	struct drm_i915_private *dev_priv = dev->dev_private;
		6310
3031	serge	6311	if (IS_IRONLAKE_M(dev)) {
5060	serge	6312	mutex_lock(&dev->struct_mutex);
3031	serge	6313	ironlake_enable_drps(dev);
		6314	ironlake_enable_rc6(dev);
		6315	intel_init_emon(dev);
5060	serge	6316	mutex_unlock(&dev->struct_mutex);
		6317	} else if (INTEL_INFO(dev)->gen >= 6) {
3243	Serge	6318	/*
		6319	* PCU communication is slow and this doesn't need to be
		6320	* done at any specific time, so do this out of our fast path
		6321	* to make resume and init faster.
5060	serge	6322	*
		6323	* We depend on the HW RC6 power context save/restore
		6324	* mechanism when entering D3 through runtime PM suspend. So
		6325	* disable RPM until RPS/RC6 is properly setup. We can only
		6326	* get here via the driver load/system resume/runtime resume
		6327	* paths, so the _noresume version is enough (and in case of
		6328	* runtime resume it's necessary).
3243	Serge	6329	*/
5060	serge	6330	if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		6331	round_jiffies_up_relative(HZ)))
		6332	intel_runtime_pm_get_noresume(dev_priv);
3031	serge	6333	}
		6334	}
		6335
5060	serge	6336	void intel_reset_gt_powersave(struct drm_device *dev)
		6337	{
		6338	struct drm_i915_private *dev_priv = dev->dev_private;
		6339
5354	serge	6340	if (INTEL_INFO(dev)->gen < 6)
		6341	return;
		6342
		6343	gen6_suspend_rps(dev);
5060	serge	6344	dev_priv->rps.enabled = false;
		6345	}
		6346
3243	Serge	6347	static void ibx_init_clock_gating(struct drm_device *dev)
		6348	{
		6349	struct drm_i915_private *dev_priv = dev->dev_private;
		6350
		6351	/*
		6352	* On Ibex Peak and Cougar Point, we need to disable clock
		6353	* gating for the panel power sequencer or it will fail to
		6354	* start up when no ports are active.
		6355	*/
		6356	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		6357	}
		6358
4104	Serge	6359	static void g4x_disable_trickle_feed(struct drm_device *dev)
		6360	{
		6361	struct drm_i915_private *dev_priv = dev->dev_private;
		6362	int pipe;
		6363
5354	serge	6364	for_each_pipe(dev_priv, pipe) {
4104	Serge	6365	I915_WRITE(DSPCNTR(pipe),
		6366	I915_READ(DSPCNTR(pipe)) \|
		6367	DISPPLANE_TRICKLE_FEED_DISABLE);
4560	Serge	6368	intel_flush_primary_plane(dev_priv, pipe);
4104	Serge	6369	}
		6370	}
		6371
4560	Serge	6372	static void ilk_init_lp_watermarks(struct drm_device *dev)
		6373	{
		6374	struct drm_i915_private *dev_priv = dev->dev_private;
		6375
		6376	I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
		6377	I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
		6378	I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
		6379
		6380	/*
		6381	* Don't touch WM1S_LP_EN here.
		6382	* Doing so could cause underruns.
		6383	*/
		6384	}
		6385
3031	serge	6386	static void ironlake_init_clock_gating(struct drm_device *dev)
		6387	{
		6388	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6389	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6390
4104	Serge	6391	/*
		6392	* Required for FBC
		6393	* WaFbcDisableDpfcClockGating:ilk
		6394	*/
3243	Serge	6395	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		6396	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		6397	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	6398
		6399	I915_WRITE(PCH_3DCGDIS0,
		6400	MARIUNIT_CLOCK_GATE_DISABLE \|
		6401	SVSMUNIT_CLOCK_GATE_DISABLE);
		6402	I915_WRITE(PCH_3DCGDIS1,
		6403	VFMUNIT_CLOCK_GATE_DISABLE);
		6404
		6405	/*
		6406	* According to the spec the following bits should be set in
		6407	* order to enable memory self-refresh
		6408	* The bit 22/21 of 0x42004
		6409	* The bit 5 of 0x42020
		6410	* The bit 15 of 0x45000
		6411	*/
		6412	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6413	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6414	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	6415	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	6416	I915_WRITE(DISP_ARB_CTL,
		6417	(I915_READ(DISP_ARB_CTL) \|
		6418	DISP_FBC_WM_DIS));
		6419
4560	Serge	6420	ilk_init_lp_watermarks(dev);
		6421
3031	serge	6422	/*
		6423	* Based on the document from hardware guys the following bits
		6424	* should be set unconditionally in order to enable FBC.
		6425	* The bit 22 of 0x42000
		6426	* The bit 22 of 0x42004
		6427	* The bit 7,8,9 of 0x42020.
		6428	*/
		6429	if (IS_IRONLAKE_M(dev)) {
4104	Serge	6430	/* WaFbcAsynchFlipDisableFbcQueue:ilk */
3031	serge	6431	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6432	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6433	ILK_FBCQ_DIS);
		6434	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6435	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6436	ILK_DPARB_GATE);
		6437	}
		6438
3243	Serge	6439	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		6440
3031	serge	6441	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6442	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6443	ILK_ELPIN_409_SELECT);
		6444	I915_WRITE(_3D_CHICKEN2,
		6445	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		6446	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	6447
4104	Serge	6448	/* WaDisableRenderCachePipelinedFlush:ilk */
3243	Serge	6449	I915_WRITE(CACHE_MODE_0,
		6450	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		6451
5060	serge	6452	/* WaDisable_RenderCache_OperationalFlush:ilk */
		6453	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6454
4104	Serge	6455	g4x_disable_trickle_feed(dev);
		6456
3243	Serge	6457	ibx_init_clock_gating(dev);
3031	serge	6458	}
		6459
3243	Serge	6460	static void cpt_init_clock_gating(struct drm_device *dev)
		6461	{
		6462	struct drm_i915_private *dev_priv = dev->dev_private;
		6463	int pipe;
3746	Serge	6464	uint32_t val;
3243	Serge	6465
		6466	/*
		6467	* On Ibex Peak and Cougar Point, we need to disable clock
		6468	* gating for the panel power sequencer or it will fail to
		6469	* start up when no ports are active.
		6470	*/
4280	Serge	6471	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE \|
		6472	PCH_DPLUNIT_CLOCK_GATE_DISABLE \|
		6473	PCH_CPUNIT_CLOCK_GATE_DISABLE);
3243	Serge	6474	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		6475	DPLS_EDP_PPS_FIX_DIS);
		6476	/* The below fixes the weird display corruption, a few pixels shifted
		6477	* downward, on (only) LVDS of some HP laptops with IVY.
		6478	*/
5354	serge	6479	for_each_pipe(dev_priv, pipe) {
3746	Serge	6480	val = I915_READ(TRANS_CHICKEN2(pipe));
		6481	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		6482	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4104	Serge	6483	if (dev_priv->vbt.fdi_rx_polarity_inverted)
3746	Serge	6484	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		6485	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		6486	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		6487	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		6488	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		6489	}
3243	Serge	6490	/* WADP0ClockGatingDisable */
5354	serge	6491	for_each_pipe(dev_priv, pipe) {
3243	Serge	6492	I915_WRITE(TRANS_CHICKEN1(pipe),
		6493	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		6494	}
		6495	}
		6496
3480	Serge	6497	static void gen6_check_mch_setup(struct drm_device *dev)
		6498	{
		6499	struct drm_i915_private *dev_priv = dev->dev_private;
		6500	uint32_t tmp;
		6501
		6502	tmp = I915_READ(MCH_SSKPD);
5060	serge	6503	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
		6504	DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
		6505	tmp);
3480	Serge	6506	}
		6507
3031	serge	6508	static void gen6_init_clock_gating(struct drm_device *dev)
		6509	{
		6510	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	6511	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	6512
3243	Serge	6513	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	6514
		6515	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6516	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6517	ILK_ELPIN_409_SELECT);
		6518
4104	Serge	6519	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3243	Serge	6520	I915_WRITE(_3D_CHICKEN,
		6521	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		6522
5060	serge	6523	/* WaDisable_RenderCache_OperationalFlush:snb */
		6524	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6525
		6526	/*
		6527	* BSpec recoomends 8x4 when MSAA is used,
		6528	* however in practice 16x4 seems fastest.
		6529	*
		6530	* Note that PS/WM thread counts depend on the WIZ hashing
		6531	* disable bit, which we don't touch here, but it's good
		6532	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6533	*/
		6534	I915_WRITE(GEN6_GT_MODE,
5354	serge	6535	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6536
4560	Serge	6537	ilk_init_lp_watermarks(dev);
3031	serge	6538
		6539	I915_WRITE(CACHE_MODE_0,
		6540	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		6541
		6542	I915_WRITE(GEN6_UCGCTL1,
		6543	I915_READ(GEN6_UCGCTL1) \|
		6544	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		6545	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6546
		6547	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		6548	* gating disable must be set. Failure to set it results in
		6549	* flickering pixels due to Z write ordering failures after
		6550	* some amount of runtime in the Mesa "fire" demo, and Unigine
		6551	* Sanctuary and Tropics, and apparently anything else with
		6552	* alpha test or pixel discard.
		6553	*
		6554	* According to the spec, bit 11 (RCCUNIT) must also be set,
		6555	* but we didn't debug actual testcases to find it out.
		6556	*
5060	serge	6557	* WaDisableRCCUnitClockGating:snb
		6558	* WaDisableRCPBUnitClockGating:snb
3031	serge	6559	*/
		6560	I915_WRITE(GEN6_UCGCTL2,
		6561	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		6562	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		6563
5060	serge	6564	/* WaStripsFansDisableFastClipPerformanceFix:snb */
		6565	I915_WRITE(_3D_CHICKEN3,
		6566	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
3031	serge	6567
		6568	/*
5060	serge	6569	* Bspec says:
		6570	* "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
		6571	* 3DSTATE_SF number of SF output attributes is more than 16."
		6572	*/
		6573	I915_WRITE(_3D_CHICKEN3,
		6574	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
		6575
		6576	/*
3031	serge	6577	* According to the spec the following bits should be
		6578	* set in order to enable memory self-refresh and fbc:
		6579	* The bit21 and bit22 of 0x42000
		6580	* The bit21 and bit22 of 0x42004
		6581	* The bit5 and bit7 of 0x42020
		6582	* The bit14 of 0x70180
		6583	* The bit14 of 0x71180
4104	Serge	6584	*
		6585	* WaFbcAsynchFlipDisableFbcQueue:snb
3031	serge	6586	*/
		6587	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		6588	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		6589	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		6590	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		6591	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		6592	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	6593	I915_WRITE(ILK_DSPCLK_GATE_D,
		6594	I915_READ(ILK_DSPCLK_GATE_D) \|
		6595	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		6596	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	6597
4104	Serge	6598	g4x_disable_trickle_feed(dev);
3031	serge	6599
3243	Serge	6600	cpt_init_clock_gating(dev);
3480	Serge	6601
		6602	gen6_check_mch_setup(dev);
3031	serge	6603	}
		6604
		6605	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		6606	{
		6607	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		6608
5060	serge	6609	/*
		6610	* WaVSThreadDispatchOverride:ivb,vlv
		6611	*
		6612	* This actually overrides the dispatch
		6613	* mode for all thread types.
		6614	*/
3031	serge	6615	reg &= ~GEN7_FF_SCHED_MASK;
		6616	reg \|= GEN7_FF_TS_SCHED_HW;
		6617	reg \|= GEN7_FF_VS_SCHED_HW;
		6618	reg \|= GEN7_FF_DS_SCHED_HW;
		6619
		6620	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		6621	}
		6622
3243	Serge	6623	static void lpt_init_clock_gating(struct drm_device *dev)
		6624	{
		6625	struct drm_i915_private *dev_priv = dev->dev_private;
		6626
		6627	/*
		6628	* TODO: this bit should only be enabled when really needed, then
		6629	* disabled when not needed anymore in order to save power.
		6630	*/
		6631	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
		6632	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		6633	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		6634	PCH_LP_PARTITION_LEVEL_DISABLE);
4104	Serge	6635
		6636	/* WADPOClockGatingDisable:hsw */
		6637	I915_WRITE(_TRANSA_CHICKEN1,
		6638	I915_READ(_TRANSA_CHICKEN1) \|
		6639	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3243	Serge	6640	}
		6641
4104	Serge	6642	static void lpt_suspend_hw(struct drm_device *dev)
		6643	{
		6644	struct drm_i915_private *dev_priv = dev->dev_private;
		6645
		6646	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
		6647	uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
		6648
		6649	val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		6650	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
		6651	}
		6652	}
		6653
5354	serge	6654	static void broadwell_init_clock_gating(struct drm_device *dev)
3031	serge	6655	{
		6656	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	6657	enum pipe pipe;
3031	serge	6658
		6659	I915_WRITE(WM3_LP_ILK, 0);
		6660	I915_WRITE(WM2_LP_ILK, 0);
		6661	I915_WRITE(WM1_LP_ILK, 0);
		6662
4560	Serge	6663	/* WaSwitchSolVfFArbitrationPriority:bdw */
		6664	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6665
		6666	/* WaPsrDPAMaskVBlankInSRD:bdw */
		6667	I915_WRITE(CHICKEN_PAR1_1,
		6668	I915_READ(CHICKEN_PAR1_1) \| DPA_MASK_VBLANK_SRD);
		6669
		6670	/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
5354	serge	6671	for_each_pipe(dev_priv, pipe) {
5060	serge	6672	I915_WRITE(CHICKEN_PIPESL_1(pipe),
		6673	I915_READ(CHICKEN_PIPESL_1(pipe)) \|
		6674	BDW_DPRS_MASK_VBLANK_SRD);
4560	Serge	6675	}
		6676
		6677	/* WaVSRefCountFullforceMissDisable:bdw */
		6678	/* WaDSRefCountFullforceMissDisable:bdw */
		6679	I915_WRITE(GEN7_FF_THREAD_MODE,
		6680	I915_READ(GEN7_FF_THREAD_MODE) &
		6681	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
5060	serge	6682
		6683	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6684	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6685
		6686	/* WaDisableSDEUnitClockGating:bdw */
		6687	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6688	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		6689
5354	serge	6690	lpt_init_clock_gating(dev);
4560	Serge	6691	}
		6692
		6693	static void haswell_init_clock_gating(struct drm_device *dev)
		6694	{
		6695	struct drm_i915_private *dev_priv = dev->dev_private;
		6696
		6697	ilk_init_lp_watermarks(dev);
		6698
4104	Serge	6699	/* L3 caching of data atomics doesn't work -- disable it. */
		6700	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
		6701	I915_WRITE(HSW_ROW_CHICKEN3,
		6702	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
		6703
		6704	/* This is required by WaCatErrorRejectionIssue:hsw */
3031	serge	6705	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6706	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6707	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6708
4104	Serge	6709	/* WaVSRefCountFullforceMissDisable:hsw */
5060	serge	6710	I915_WRITE(GEN7_FF_THREAD_MODE,
		6711	I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
3031	serge	6712
5060	serge	6713	/* WaDisable_RenderCache_OperationalFlush:hsw */
		6714	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6715
		6716	/* enable HiZ Raw Stall Optimization */
		6717	I915_WRITE(CACHE_MODE_0_GEN7,
		6718	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6719
4104	Serge	6720	/* WaDisable4x2SubspanOptimization:hsw */
3031	serge	6721	I915_WRITE(CACHE_MODE_1,
		6722	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6723
5060	serge	6724	/*
		6725	* BSpec recommends 8x4 when MSAA is used,
		6726	* however in practice 16x4 seems fastest.
		6727	*
		6728	* Note that PS/WM thread counts depend on the WIZ hashing
		6729	* disable bit, which we don't touch here, but it's good
		6730	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6731	*/
		6732	I915_WRITE(GEN7_GT_MODE,
5354	serge	6733	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6734
4104	Serge	6735	/* WaSwitchSolVfFArbitrationPriority:hsw */
3746	Serge	6736	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		6737
4104	Serge	6738	/* WaRsPkgCStateDisplayPMReq:hsw */
		6739	I915_WRITE(CHICKEN_PAR1_1,
		6740	I915_READ(CHICKEN_PAR1_1) \| FORCE_ARB_IDLE_PLANES);
3031	serge	6741
3243	Serge	6742	lpt_init_clock_gating(dev);
3031	serge	6743	}
		6744
		6745	static void ivybridge_init_clock_gating(struct drm_device *dev)
		6746	{
		6747	struct drm_i915_private *dev_priv = dev->dev_private;
		6748	uint32_t snpcr;
		6749
4560	Serge	6750	ilk_init_lp_watermarks(dev);
3031	serge	6751
3243	Serge	6752	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	6753
4104	Serge	6754	/* WaDisableEarlyCull:ivb */
3243	Serge	6755	I915_WRITE(_3D_CHICKEN3,
		6756	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6757
4104	Serge	6758	/* WaDisableBackToBackFlipFix:ivb */
3031	serge	6759	I915_WRITE(IVB_CHICKEN3,
		6760	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6761	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6762
4104	Serge	6763	/* WaDisablePSDDualDispatchEnable:ivb */
3243	Serge	6764	if (IS_IVB_GT1(dev))
		6765	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		6766	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		6767
5060	serge	6768	/* WaDisable_RenderCache_OperationalFlush:ivb */
		6769	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6770
4104	Serge	6771	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
3031	serge	6772	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		6773	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		6774
4104	Serge	6775	/* WaApplyL3ControlAndL3ChickenMode:ivb */
3031	serge	6776	I915_WRITE(GEN7_L3CNTLREG1,
		6777	GEN7_WA_FOR_GEN7_L3_CONTROL);
		6778	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		6779	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	6780	if (IS_IVB_GT1(dev))
		6781	I915_WRITE(GEN7_ROW_CHICKEN2,
		6782	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6783	else {
		6784	/* must write both registers */
		6785	I915_WRITE(GEN7_ROW_CHICKEN2,
		6786	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3243	Serge	6787	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		6788	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	6789	}
3031	serge	6790
4104	Serge	6791	/* WaForceL3Serialization:ivb */
3243	Serge	6792	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6793	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6794
5060	serge	6795	/*
3031	serge	6796	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6797	* This implements the WaDisableRCZUnitClockGating:ivb workaround.
3031	serge	6798	*/
		6799	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6800	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6801
4104	Serge	6802	/* This is required by WaCatErrorRejectionIssue:ivb */
3031	serge	6803	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6804	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6805	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6806
4104	Serge	6807	g4x_disable_trickle_feed(dev);
3031	serge	6808
		6809	gen7_setup_fixed_func_scheduler(dev_priv);
		6810
5060	serge	6811	if (0) { /* causes HiZ corruption on ivb:gt1 */
		6812	/* enable HiZ Raw Stall Optimization */
		6813	I915_WRITE(CACHE_MODE_0_GEN7,
		6814	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		6815	}
		6816
4104	Serge	6817	/* WaDisable4x2SubspanOptimization:ivb */
3031	serge	6818	I915_WRITE(CACHE_MODE_1,
		6819	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6820
5060	serge	6821	/*
		6822	* BSpec recommends 8x4 when MSAA is used,
		6823	* however in practice 16x4 seems fastest.
		6824	*
		6825	* Note that PS/WM thread counts depend on the WIZ hashing
		6826	* disable bit, which we don't touch here, but it's good
		6827	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		6828	*/
		6829	I915_WRITE(GEN7_GT_MODE,
5354	serge	6830	_MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5060	serge	6831
3031	serge	6832	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		6833	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		6834	snpcr \|= GEN6_MBC_SNPCR_MED;
		6835	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	6836
3746	Serge	6837	if (!HAS_PCH_NOP(dev))
3243	Serge	6838	cpt_init_clock_gating(dev);
3480	Serge	6839
		6840	gen6_check_mch_setup(dev);
3031	serge	6841	}
		6842
		6843	static void valleyview_init_clock_gating(struct drm_device *dev)
		6844	{
		6845	struct drm_i915_private *dev_priv = dev->dev_private;
		6846
4104	Serge	6847	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	6848
4104	Serge	6849	/* WaDisableEarlyCull:vlv */
3243	Serge	6850	I915_WRITE(_3D_CHICKEN3,
		6851	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		6852
4104	Serge	6853	/* WaDisableBackToBackFlipFix:vlv */
3031	serge	6854	I915_WRITE(IVB_CHICKEN3,
		6855	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		6856	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		6857
5060	serge	6858	/* WaPsdDispatchEnable:vlv */
4104	Serge	6859	/* WaDisablePSDDualDispatchEnable:vlv */
3243	Serge	6860	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	6861	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		6862	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	6863
5060	serge	6864	/* WaDisable_RenderCache_OperationalFlush:vlv */
		6865	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6866
4104	Serge	6867	/* WaForceL3Serialization:vlv */
3243	Serge	6868	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		6869	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		6870
4104	Serge	6871	/* WaDisableDopClockGating:vlv */
3243	Serge	6872	I915_WRITE(GEN7_ROW_CHICKEN2,
		6873	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		6874
4104	Serge	6875	/* This is required by WaCatErrorRejectionIssue:vlv */
3031	serge	6876	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		6877	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		6878	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		6879
5060	serge	6880	gen7_setup_fixed_func_scheduler(dev_priv);
		6881
		6882	/*
3031	serge	6883	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	6884	* This implements the WaDisableRCZUnitClockGating:vlv workaround.
3031	serge	6885	*/
		6886	I915_WRITE(GEN6_UCGCTL2,
5060	serge	6887	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	6888
5060	serge	6889	/* WaDisableL3Bank2xClockGate:vlv
		6890	* Disabling L3 clock gating- MMIO 940c[25] = 1
		6891	* Set bit 25, to disable L3_BANK_2x_CLK_GATING */
		6892	I915_WRITE(GEN7_UCGCTL4,
		6893	I915_READ(GEN7_UCGCTL4) \| GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
3031	serge	6894
4104	Serge	6895	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
3031	serge	6896
5060	serge	6897	/*
		6898	* BSpec says this must be set, even though
		6899	* WaDisable4x2SubspanOptimization isn't listed for VLV.
		6900	*/
3031	serge	6901	I915_WRITE(CACHE_MODE_1,
		6902	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		6903
		6904	/*
5060	serge	6905	* WaIncreaseL3CreditsForVLVB0:vlv
		6906	* This is the hardware default actually.
		6907	*/
		6908	I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
		6909
		6910	/*
4104	Serge	6911	* WaDisableVLVClockGating_VBIIssue:vlv
3243	Serge	6912	* Disable clock gating on th GCFG unit to prevent a delay
		6913	* in the reporting of vblank events.
		6914	*/
5060	serge	6915	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
		6916	}
3746	Serge	6917
5060	serge	6918	static void cherryview_init_clock_gating(struct drm_device *dev)
		6919	{
		6920	struct drm_i915_private *dev_priv = dev->dev_private;
		6921
		6922	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
		6923
		6924	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
		6925
		6926	/* WaVSRefCountFullforceMissDisable:chv */
		6927	/* WaDSRefCountFullforceMissDisable:chv */
		6928	I915_WRITE(GEN7_FF_THREAD_MODE,
		6929	I915_READ(GEN7_FF_THREAD_MODE) &
		6930	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
		6931
		6932	/* WaDisableSemaphoreAndSyncFlipWait:chv */
		6933	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		6934	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		6935
		6936	/* WaDisableCSUnitClockGating:chv */
		6937	I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) \|
		6938	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		6939
		6940	/* WaDisableSDEUnitClockGating:chv */
		6941	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		6942	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
3031	serge	6943	}
		6944
		6945	static void g4x_init_clock_gating(struct drm_device *dev)
		6946	{
		6947	struct drm_i915_private *dev_priv = dev->dev_private;
		6948	uint32_t dspclk_gate;
		6949
		6950	I915_WRITE(RENCLK_GATE_D1, 0);
		6951	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		6952	GS_UNIT_CLOCK_GATE_DISABLE \|
		6953	CL_UNIT_CLOCK_GATE_DISABLE);
		6954	I915_WRITE(RAMCLK_GATE_D, 0);
		6955	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		6956	OVRUNIT_CLOCK_GATE_DISABLE \|
		6957	OVCUNIT_CLOCK_GATE_DISABLE;
		6958	if (IS_GM45(dev))
		6959	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		6960	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	6961
		6962	/* WaDisableRenderCachePipelinedFlush */
		6963	I915_WRITE(CACHE_MODE_0,
		6964	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4104	Serge	6965
5060	serge	6966	/* WaDisable_RenderCache_OperationalFlush:g4x */
		6967	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		6968
4104	Serge	6969	g4x_disable_trickle_feed(dev);
3031	serge	6970	}
		6971
		6972	static void crestline_init_clock_gating(struct drm_device *dev)
		6973	{
		6974	struct drm_i915_private *dev_priv = dev->dev_private;
		6975
		6976	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		6977	I915_WRITE(RENCLK_GATE_D2, 0);
		6978	I915_WRITE(DSPCLK_GATE_D, 0);
		6979	I915_WRITE(RAMCLK_GATE_D, 0);
		6980	I915_WRITE16(DEUC, 0);
4104	Serge	6981	I915_WRITE(MI_ARB_STATE,
		6982	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	6983
		6984	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		6985	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	6986	}
		6987
		6988	static void broadwater_init_clock_gating(struct drm_device *dev)
		6989	{
		6990	struct drm_i915_private *dev_priv = dev->dev_private;
		6991
		6992	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		6993	I965_RCC_CLOCK_GATE_DISABLE \|
		6994	I965_RCPB_CLOCK_GATE_DISABLE \|
		6995	I965_ISC_CLOCK_GATE_DISABLE \|
		6996	I965_FBC_CLOCK_GATE_DISABLE);
		6997	I915_WRITE(RENCLK_GATE_D2, 0);
4104	Serge	6998	I915_WRITE(MI_ARB_STATE,
		6999	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	7000
		7001	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		7002	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	7003	}
		7004
		7005	static void gen3_init_clock_gating(struct drm_device *dev)
		7006	{
		7007	struct drm_i915_private *dev_priv = dev->dev_private;
		7008	u32 dstate = I915_READ(D_STATE);
		7009
		7010	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		7011	DSTATE_DOT_CLOCK_GATING;
		7012	I915_WRITE(D_STATE, dstate);
		7013
		7014	if (IS_PINEVIEW(dev))
		7015	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		7016
		7017	/* IIR "flip pending" means done if this bit is set */
		7018	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
5060	serge	7019
		7020	/* interrupts should cause a wake up from C3 */
		7021	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
		7022
		7023	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		7024	I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
5354	serge	7025
		7026	I915_WRITE(MI_ARB_STATE,
		7027	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	7028	}
		7029
		7030	static void i85x_init_clock_gating(struct drm_device *dev)
		7031	{
		7032	struct drm_i915_private *dev_priv = dev->dev_private;
		7033
		7034	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
5060	serge	7035
		7036	/* interrupts should cause a wake up from C3 */
		7037	I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) \|
		7038	_MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
5354	serge	7039
		7040	I915_WRITE(MEM_MODE,
		7041	_MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	7042	}
		7043
		7044	static void i830_init_clock_gating(struct drm_device *dev)
		7045	{
		7046	struct drm_i915_private *dev_priv = dev->dev_private;
		7047
		7048	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
5354	serge	7049
		7050	I915_WRITE(MEM_MODE,
		7051	_MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) \|
		7052	_MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
3031	serge	7053	}
		7054
		7055	void intel_init_clock_gating(struct drm_device *dev)
		7056	{
		7057	struct drm_i915_private *dev_priv = dev->dev_private;
		7058
		7059	dev_priv->display.init_clock_gating(dev);
		7060	}
		7061
4104	Serge	7062	void intel_suspend_hw(struct drm_device *dev)
		7063	{
		7064	if (HAS_PCH_LPT(dev))
		7065	lpt_suspend_hw(dev);
		7066	}
		7067
5354	serge	7068	static void intel_init_fbc(struct drm_i915_private *dev_priv)
4560	Serge	7069	{
5354	serge	7070	if (!HAS_FBC(dev_priv)) {
		7071	dev_priv->fbc.enabled = false;
5060	serge	7072	return;
4560	Serge	7073	}
5060	serge	7074
5354	serge	7075	if (INTEL_INFO(dev_priv)->gen >= 7) {
3031	serge	7076	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
4560	Serge	7077	dev_priv->display.enable_fbc = gen7_enable_fbc;
3031	serge	7078	dev_priv->display.disable_fbc = ironlake_disable_fbc;
5354	serge	7079	} else if (INTEL_INFO(dev_priv)->gen >= 5) {
4560	Serge	7080	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
		7081	dev_priv->display.enable_fbc = ironlake_enable_fbc;
		7082	dev_priv->display.disable_fbc = ironlake_disable_fbc;
5354	serge	7083	} else if (IS_GM45(dev_priv)) {
3031	serge	7084	dev_priv->display.fbc_enabled = g4x_fbc_enabled;
		7085	dev_priv->display.enable_fbc = g4x_enable_fbc;
		7086	dev_priv->display.disable_fbc = g4x_disable_fbc;
4560	Serge	7087	} else {
3031	serge	7088	dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
		7089	dev_priv->display.enable_fbc = i8xx_enable_fbc;
		7090	dev_priv->display.disable_fbc = i8xx_disable_fbc;
4560	Serge	7091
		7092	/* This value was pulled out of someone's hat */
		7093	I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT);
3031	serge	7094	}
		7095
5354	serge	7096	dev_priv->fbc.enabled = dev_priv->display.fbc_enabled(dev_priv->dev);
		7097	}
		7098
		7099	/* Set up chip specific power management-related functions */
		7100	void intel_init_pm(struct drm_device *dev)
		7101	{
		7102	struct drm_i915_private *dev_priv = dev->dev_private;
		7103
		7104	intel_init_fbc(dev_priv);
		7105
3031	serge	7106	/* For cxsr */
		7107	if (IS_PINEVIEW(dev))
		7108	i915_pineview_get_mem_freq(dev);
		7109	else if (IS_GEN5(dev))
		7110	i915_ironlake_get_mem_freq(dev);
		7111
		7112	/* For FIFO watermark updates */
5354	serge	7113	if (INTEL_INFO(dev)->gen >= 9) {
		7114	skl_setup_wm_latency(dev);
		7115
		7116	dev_priv->display.init_clock_gating = gen9_init_clock_gating;
		7117	dev_priv->display.update_wm = skl_update_wm;
		7118	dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
		7119	} else if (HAS_PCH_SPLIT(dev)) {
5060	serge	7120	ilk_setup_wm_latency(dev);
4104	Serge	7121
4560	Serge	7122	if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
		7123	dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) \|\|
		7124	(!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
		7125	dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
		7126	dev_priv->display.update_wm = ilk_update_wm;
		7127	dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
3031	serge	7128	} else {
		7129	DRM_DEBUG_KMS("Failed to read display plane latency. "
		7130	"Disable CxSR\n");
		7131	}
4560	Serge	7132
		7133	if (IS_GEN5(dev))
		7134	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		7135	else if (IS_GEN6(dev))
3031	serge	7136	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
4560	Serge	7137	else if (IS_IVYBRIDGE(dev))
3031	serge	7138	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
4560	Serge	7139	else if (IS_HASWELL(dev))
3031	serge	7140	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
4560	Serge	7141	else if (INTEL_INFO(dev)->gen == 8)
5354	serge	7142	dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
5060	serge	7143	} else if (IS_CHERRYVIEW(dev)) {
5354	serge	7144	dev_priv->display.update_wm = cherryview_update_wm;
		7145	dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
5060	serge	7146	dev_priv->display.init_clock_gating =
		7147	cherryview_init_clock_gating;
3031	serge	7148	} else if (IS_VALLEYVIEW(dev)) {
		7149	dev_priv->display.update_wm = valleyview_update_wm;
5354	serge	7150	dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
3031	serge	7151	dev_priv->display.init_clock_gating =
		7152	valleyview_init_clock_gating;
		7153	} else if (IS_PINEVIEW(dev)) {
		7154	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		7155	dev_priv->is_ddr3,
		7156	dev_priv->fsb_freq,
		7157	dev_priv->mem_freq)) {
		7158	DRM_INFO("failed to find known CxSR latency "
		7159	"(found ddr%s fsb freq %d, mem freq %d), "
		7160	"disabling CxSR\n",
		7161	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		7162	dev_priv->fsb_freq, dev_priv->mem_freq);
		7163	/* Disable CxSR and never update its watermark again */
5060	serge	7164	intel_set_memory_cxsr(dev_priv, false);
3031	serge	7165	dev_priv->display.update_wm = NULL;
		7166	} else
		7167	dev_priv->display.update_wm = pineview_update_wm;
		7168	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		7169	} else if (IS_G4X(dev)) {
		7170	dev_priv->display.update_wm = g4x_update_wm;
		7171	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		7172	} else if (IS_GEN4(dev)) {
		7173	dev_priv->display.update_wm = i965_update_wm;
		7174	if (IS_CRESTLINE(dev))
		7175	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		7176	else if (IS_BROADWATER(dev))
		7177	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		7178	} else if (IS_GEN3(dev)) {
		7179	dev_priv->display.update_wm = i9xx_update_wm;
		7180	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		7181	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
4560	Serge	7182	} else if (IS_GEN2(dev)) {
		7183	if (INTEL_INFO(dev)->num_pipes == 1) {
		7184	dev_priv->display.update_wm = i845_update_wm;
		7185	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		7186	} else {
		7187	dev_priv->display.update_wm = i9xx_update_wm;
3031	serge	7188	dev_priv->display.get_fifo_size = i830_get_fifo_size;
4560	Serge	7189	}
		7190
		7191	if (IS_I85X(dev) \|\| IS_I865G(dev))
3031	serge	7192	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
4560	Serge	7193	else
		7194	dev_priv->display.init_clock_gating = i830_init_clock_gating;
3031	serge	7195	} else {
4560	Serge	7196	DRM_ERROR("unexpected fall-through in intel_init_pm\n");
3031	serge	7197	}
		7198	}
		7199
5354	serge	7200	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u32 mbox, u32 val)
3243	Serge	7201	{
		7202	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	7203
3243	Serge	7204	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7205	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		7206	return -EAGAIN;
		7207	}
3031	serge	7208
3243	Serge	7209	I915_WRITE(GEN6_PCODE_DATA, *val);
5354	serge	7210	I915_WRITE(GEN6_PCODE_DATA1, 0);
3243	Serge	7211	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7212
		7213	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7214	500)) {
		7215	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		7216	return -ETIMEDOUT;
3031	serge	7217	}
3243	Serge	7218
		7219	*val = I915_READ(GEN6_PCODE_DATA);
		7220	I915_WRITE(GEN6_PCODE_DATA, 0);
		7221
		7222	return 0;
		7223	}
		7224
5354	serge	7225	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
3243	Serge	7226	{
		7227	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		7228
		7229	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		7230	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		7231	return -EAGAIN;
3031	serge	7232	}
3243	Serge	7233
		7234	I915_WRITE(GEN6_PCODE_DATA, val);
		7235	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		7236
		7237	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		7238	500)) {
		7239	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		7240	return -ETIMEDOUT;
3031	serge	7241	}
3243	Serge	7242
		7243	I915_WRITE(GEN6_PCODE_DATA, 0);
		7244
		7245	return 0;
3031	serge	7246	}
3746	Serge	7247
5354	serge	7248	static int vlv_gpu_freq_div(unsigned int czclk_freq)
3746	Serge	7249	{
5354	serge	7250	switch (czclk_freq) {
		7251	case 200:
		7252	return 10;
		7253	case 267:
		7254	return 12;
		7255	case 320:
		7256	case 333:
		7257	return 16;
		7258	case 400:
		7259	return 20;
4104	Serge	7260	default:
		7261	return -1;
		7262	}
5354	serge	7263	}
3746	Serge	7264
5354	serge	7265	static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7266	{
		7267	int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
		7268
		7269	div = vlv_gpu_freq_div(czclk_freq);
		7270	if (div < 0)
		7271	return div;
		7272
		7273	return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
4104	Serge	7274	}
3746	Serge	7275
5060	serge	7276	static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
4104	Serge	7277	{
5354	serge	7278	int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
3746	Serge	7279
5354	serge	7280	mul = vlv_gpu_freq_div(czclk_freq);
		7281	if (mul < 0)
		7282	return mul;
3746	Serge	7283
5354	serge	7284	return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
3746	Serge	7285	}
		7286
5060	serge	7287	static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7288	{
5354	serge	7289	int div, czclk_freq = dev_priv->rps.cz_freq;
5060	serge	7290
5354	serge	7291	div = vlv_gpu_freq_div(czclk_freq) / 2;
		7292	if (div < 0)
		7293	return div;
5060	serge	7294
5354	serge	7295	return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
5060	serge	7296	}
		7297
		7298	static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7299	{
5354	serge	7300	int mul, czclk_freq = dev_priv->rps.cz_freq;
5060	serge	7301
5354	serge	7302	mul = vlv_gpu_freq_div(czclk_freq) / 2;
		7303	if (mul < 0)
		7304	return mul;
5060	serge	7305
5354	serge	7306	/* CHV needs even values */
		7307	return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
5060	serge	7308	}
		7309
		7310	int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7311	{
		7312	int ret = -1;
		7313
		7314	if (IS_CHERRYVIEW(dev_priv->dev))
		7315	ret = chv_gpu_freq(dev_priv, val);
		7316	else if (IS_VALLEYVIEW(dev_priv->dev))
		7317	ret = byt_gpu_freq(dev_priv, val);
		7318
		7319	return ret;
		7320	}
		7321
		7322	int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7323	{
		7324	int ret = -1;
		7325
		7326	if (IS_CHERRYVIEW(dev_priv->dev))
		7327	ret = chv_freq_opcode(dev_priv, val);
		7328	else if (IS_VALLEYVIEW(dev_priv->dev))
		7329	ret = byt_freq_opcode(dev_priv, val);
		7330
		7331	return ret;
		7332	}
		7333
4560	Serge	7334	void intel_pm_setup(struct drm_device *dev)
3746	Serge	7335	{
4104	Serge	7336	struct drm_i915_private *dev_priv = dev->dev_private;
		7337
4560	Serge	7338	mutex_init(&dev_priv->rps.hw_lock);
		7339
4104	Serge	7340	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		7341	intel_gen6_powersave_work);
5060	serge	7342
		7343	dev_priv->pm.suspended = false;
3746	Serge	7344	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c @ 6660 – Rev 5354