WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	#include
		32	//#include "../../../platform/x86/intel_ips.h"
		33	#include
		34
		35	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		36
		37	#define assert_spin_locked(x)
		38
		39	void getrawmonotonic(struct timespec *ts);
		40
		41
		42
		43	/* FBC, or Frame Buffer Compression, is a technique employed to compress the
		44	* framebuffer contents in-memory, aiming at reducing the required bandwidth
		45	* during in-memory transfers and, therefore, reduce the power packet.
		46	*
		47	* The benefits of FBC are mostly visible with solid backgrounds and
		48	* variation-less patterns.
		49	*
		50	* FBC-related functionality can be enabled by the means of the
		51	* i915.i915_enable_fbc parameter
		52	*/
		53
3243	Serge	54	static bool intel_crtc_active(struct drm_crtc *crtc)
		55	{
		56	/* Be paranoid as we can arrive here with only partial
		57	* state retrieved from the hardware during setup.
		58	*/
		59	return to_intel_crtc(crtc)->active && crtc->fb && crtc->mode.clock;
		60	}
		61
3031	serge	62	static void i8xx_disable_fbc(struct drm_device *dev)
		63	{
		64	struct drm_i915_private *dev_priv = dev->dev_private;
		65	u32 fbc_ctl;
		66
		67	/* Disable compression */
		68	fbc_ctl = I915_READ(FBC_CONTROL);
		69	if ((fbc_ctl & FBC_CTL_EN) == 0)
		70	return;
		71
		72	fbc_ctl &= ~FBC_CTL_EN;
		73	I915_WRITE(FBC_CONTROL, fbc_ctl);
		74
		75	/* Wait for compressing bit to clear */
		76	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
		77	DRM_DEBUG_KMS("FBC idle timed out\n");
		78	return;
		79	}
		80
		81	DRM_DEBUG_KMS("disabled FBC\n");
		82	}
		83
		84	static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		85	{
		86	struct drm_device *dev = crtc->dev;
		87	struct drm_i915_private *dev_priv = dev->dev_private;
		88	struct drm_framebuffer *fb = crtc->fb;
		89	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		90	struct drm_i915_gem_object *obj = intel_fb->obj;
		91	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		92	int cfb_pitch;
		93	int plane, i;
		94	u32 fbc_ctl, fbc_ctl2;
		95
4104	Serge	96	cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE;
3031	serge	97	if (fb->pitches[0] < cfb_pitch)
		98	cfb_pitch = fb->pitches[0];
		99
		100	/* FBC_CTL wants 64B units */
		101	cfb_pitch = (cfb_pitch / 64) - 1;
		102	plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
		103
		104	/* Clear old tags */
		105	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
		106	I915_WRITE(FBC_TAG + (i * 4), 0);
		107
		108	/* Set it up... */
		109	fbc_ctl2 = FBC_CTL_FENCE_DBL \| FBC_CTL_IDLE_IMM \| FBC_CTL_CPU_FENCE;
		110	fbc_ctl2 \|= plane;
		111	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
		112	I915_WRITE(FBC_FENCE_OFF, crtc->y);
		113
		114	/* enable it... */
		115	fbc_ctl = FBC_CTL_EN \| FBC_CTL_PERIODIC;
		116	if (IS_I945GM(dev))
		117	fbc_ctl \|= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
		118	fbc_ctl \|= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
		119	fbc_ctl \|= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
		120	fbc_ctl \|= obj->fence_reg;
		121	I915_WRITE(FBC_CONTROL, fbc_ctl);
		122
4104	Serge	123	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ",
		124	cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
3031	serge	125	}
		126
		127	static bool i8xx_fbc_enabled(struct drm_device *dev)
		128	{
		129	struct drm_i915_private *dev_priv = dev->dev_private;
		130
		131	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
		132	}
		133
		134	static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		135	{
		136	struct drm_device *dev = crtc->dev;
		137	struct drm_i915_private *dev_priv = dev->dev_private;
		138	struct drm_framebuffer *fb = crtc->fb;
		139	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		140	struct drm_i915_gem_object *obj = intel_fb->obj;
		141	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		142	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		143	unsigned long stall_watermark = 200;
		144	u32 dpfc_ctl;
		145
		146	dpfc_ctl = plane \| DPFC_SR_EN \| DPFC_CTL_LIMIT_1X;
		147	dpfc_ctl \|= DPFC_CTL_FENCE_EN \| obj->fence_reg;
		148	I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
		149
		150	I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		151	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		152	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		153	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
		154
		155	/* enable it... */
		156	I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) \| DPFC_CTL_EN);
		157
4104	Serge	158	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
3031	serge	159	}
		160
		161	static void g4x_disable_fbc(struct drm_device *dev)
		162	{
		163	struct drm_i915_private *dev_priv = dev->dev_private;
		164	u32 dpfc_ctl;
		165
		166	/* Disable compression */
		167	dpfc_ctl = I915_READ(DPFC_CONTROL);
		168	if (dpfc_ctl & DPFC_CTL_EN) {
		169	dpfc_ctl &= ~DPFC_CTL_EN;
		170	I915_WRITE(DPFC_CONTROL, dpfc_ctl);
		171
		172	DRM_DEBUG_KMS("disabled FBC\n");
		173	}
		174	}
		175
		176	static bool g4x_fbc_enabled(struct drm_device *dev)
		177	{
		178	struct drm_i915_private *dev_priv = dev->dev_private;
		179
		180	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
		181	}
		182
		183	static void sandybridge_blit_fbc_update(struct drm_device *dev)
		184	{
		185	struct drm_i915_private *dev_priv = dev->dev_private;
		186	u32 blt_ecoskpd;
		187
		188	/* Make sure blitter notifies FBC of writes */
		189	gen6_gt_force_wake_get(dev_priv);
		190	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
		191	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY <<
		192	GEN6_BLITTER_LOCK_SHIFT;
		193	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		194	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY;
		195	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		196	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
		197	GEN6_BLITTER_LOCK_SHIFT);
		198	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		199	POSTING_READ(GEN6_BLITTER_ECOSKPD);
		200	gen6_gt_force_wake_put(dev_priv);
		201	}
		202
		203	static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		204	{
		205	struct drm_device *dev = crtc->dev;
		206	struct drm_i915_private *dev_priv = dev->dev_private;
		207	struct drm_framebuffer *fb = crtc->fb;
		208	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		209	struct drm_i915_gem_object *obj = intel_fb->obj;
		210	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		211	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		212	unsigned long stall_watermark = 200;
		213	u32 dpfc_ctl;
		214
		215	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		216	dpfc_ctl &= DPFC_RESERVED;
		217	dpfc_ctl \|= (plane \| DPFC_CTL_LIMIT_1X);
		218	/* Set persistent mode for front-buffer rendering, ala X. */
		219	dpfc_ctl \|= DPFC_CTL_PERSISTENT_MODE;
		220	dpfc_ctl \|= (DPFC_CTL_FENCE_EN \| obj->fence_reg);
		221	I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
		222
		223	I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		224	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		225	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		226	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
4104	Serge	227	I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) \| ILK_FBC_RT_VALID);
3031	serge	228	/* enable it... */
		229	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		230
		231	if (IS_GEN6(dev)) {
		232	I915_WRITE(SNB_DPFC_CTL_SA,
		233	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		234	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		235	sandybridge_blit_fbc_update(dev);
		236	}
		237
4104	Serge	238	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
3031	serge	239	}
		240
		241	static void ironlake_disable_fbc(struct drm_device *dev)
		242	{
		243	struct drm_i915_private *dev_priv = dev->dev_private;
		244	u32 dpfc_ctl;
		245
		246	/* Disable compression */
		247	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		248	if (dpfc_ctl & DPFC_CTL_EN) {
		249	dpfc_ctl &= ~DPFC_CTL_EN;
		250	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
		251
4104	Serge	252	if (IS_IVYBRIDGE(dev))
		253	/* WaFbcDisableDpfcClockGating:ivb */
		254	I915_WRITE(ILK_DSPCLK_GATE_D,
		255	I915_READ(ILK_DSPCLK_GATE_D) &
		256	~ILK_DPFCUNIT_CLOCK_GATE_DISABLE);
		257
		258	if (IS_HASWELL(dev))
		259	/* WaFbcDisableDpfcClockGating:hsw */
		260	I915_WRITE(HSW_CLKGATE_DISABLE_PART_1,
		261	I915_READ(HSW_CLKGATE_DISABLE_PART_1) &
		262	~HSW_DPFC_GATING_DISABLE);
		263
3031	serge	264	DRM_DEBUG_KMS("disabled FBC\n");
		265	}
		266	}
		267
		268	static bool ironlake_fbc_enabled(struct drm_device *dev)
		269	{
		270	struct drm_i915_private *dev_priv = dev->dev_private;
		271
		272	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
		273	}
		274
4104	Serge	275	static void gen7_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		276	{
		277	struct drm_device *dev = crtc->dev;
		278	struct drm_i915_private *dev_priv = dev->dev_private;
		279	struct drm_framebuffer *fb = crtc->fb;
		280	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		281	struct drm_i915_gem_object *obj = intel_fb->obj;
		282	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		283
		284	I915_WRITE(IVB_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj));
		285
		286	I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN \| DPFC_CTL_LIMIT_1X \|
		287	IVB_DPFC_CTL_FENCE_EN \|
		288	intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT);
		289
		290	if (IS_IVYBRIDGE(dev)) {
		291	/* WaFbcAsynchFlipDisableFbcQueue:ivb */
		292	I915_WRITE(ILK_DISPLAY_CHICKEN1, ILK_FBCQ_DIS);
		293	/* WaFbcDisableDpfcClockGating:ivb */
		294	I915_WRITE(ILK_DSPCLK_GATE_D,
		295	I915_READ(ILK_DSPCLK_GATE_D) \|
		296	ILK_DPFCUNIT_CLOCK_GATE_DISABLE);
		297	} else {
		298	/* WaFbcAsynchFlipDisableFbcQueue:hsw */
		299	I915_WRITE(HSW_PIPE_SLICE_CHICKEN_1(intel_crtc->pipe),
		300	HSW_BYPASS_FBC_QUEUE);
		301	/* WaFbcDisableDpfcClockGating:hsw */
		302	I915_WRITE(HSW_CLKGATE_DISABLE_PART_1,
		303	I915_READ(HSW_CLKGATE_DISABLE_PART_1) \|
		304	HSW_DPFC_GATING_DISABLE);
		305	}
		306
		307	I915_WRITE(SNB_DPFC_CTL_SA,
		308	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		309	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		310
		311	sandybridge_blit_fbc_update(dev);
		312
		313	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		314	}
		315
3031	serge	316	bool intel_fbc_enabled(struct drm_device *dev)
		317	{
		318	struct drm_i915_private *dev_priv = dev->dev_private;
		319
		320	if (!dev_priv->display.fbc_enabled)
		321	return false;
		322
		323	return dev_priv->display.fbc_enabled(dev);
		324	}
		325
		326	static void intel_fbc_work_fn(struct work_struct *__work)
		327	{
		328	struct intel_fbc_work *work =
		329	container_of(to_delayed_work(__work),
		330	struct intel_fbc_work, work);
		331	struct drm_device *dev = work->crtc->dev;
		332	struct drm_i915_private *dev_priv = dev->dev_private;
		333
		334	mutex_lock(&dev->struct_mutex);
4104	Serge	335	if (work == dev_priv->fbc.fbc_work) {
3031	serge	336	/* Double check that we haven't switched fb without cancelling
		337	* the prior work.
		338	*/
		339	if (work->crtc->fb == work->fb) {
		340	dev_priv->display.enable_fbc(work->crtc,
		341	work->interval);
		342
4104	Serge	343	dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane;
		344	dev_priv->fbc.fb_id = work->crtc->fb->base.id;
		345	dev_priv->fbc.y = work->crtc->y;
3031	serge	346	}
		347
4104	Serge	348	dev_priv->fbc.fbc_work = NULL;
3031	serge	349	}
		350	mutex_unlock(&dev->struct_mutex);
		351
		352	kfree(work);
		353	}
		354
		355	static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
		356	{
4104	Serge	357	if (dev_priv->fbc.fbc_work == NULL)
3031	serge	358	return;
		359
		360	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
		361
		362	/* Synchronisation is provided by struct_mutex and checking of
4104	Serge	363	* dev_priv->fbc.fbc_work, so we can perform the cancellation
3031	serge	364	* entirely asynchronously.
		365	*/
3482	Serge	366	// if (cancel_delayed_work(&dev_priv->fbc_work->work))
3031	serge	367	/* tasklet was killed before being run, clean up */
3482	Serge	368	// kfree(dev_priv->fbc_work);
3031	serge	369
		370	/* Mark the work as no longer wanted so that if it does
		371	* wake-up (because the work was already running and waiting
		372	* for our mutex), it will discover that is no longer
		373	* necessary to run.
		374	*/
4104	Serge	375	dev_priv->fbc.fbc_work = NULL;
3031	serge	376	}
		377
4104	Serge	378	static void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
3031	serge	379	{
		380	struct intel_fbc_work *work;
		381	struct drm_device *dev = crtc->dev;
		382	struct drm_i915_private *dev_priv = dev->dev_private;
		383
3482	Serge	384	if (!dev_priv->display.enable_fbc)
3031	serge	385	return;
3482	Serge	386
3031	serge	387	intel_cancel_fbc_work(dev_priv);
		388
		389	work = kzalloc(sizeof *work, GFP_KERNEL);
		390	if (work == NULL) {
4104	Serge	391	DRM_ERROR("Failed to allocate FBC work structure\n");
3031	serge	392	dev_priv->display.enable_fbc(crtc, interval);
		393	return;
		394	}
		395
		396	work->crtc = crtc;
		397	work->fb = crtc->fb;
		398	work->interval = interval;
		399	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
		400
4104	Serge	401	dev_priv->fbc.fbc_work = work;
3031	serge	402
		403	DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
		404
		405	/* Delay the actual enabling to let pageflipping cease and the
		406	* display to settle before starting the compression. Note that
		407	* this delay also serves a second purpose: it allows for a
		408	* vblank to pass after disabling the FBC before we attempt
		409	* to modify the control registers.
		410	*
		411	* A more complicated solution would involve tracking vblanks
		412	* following the termination of the page-flipping sequence
		413	* and indeed performing the enable as a co-routine and not
		414	* waiting synchronously upon the vblank.
4104	Serge	415	*
		416	* WaFbcWaitForVBlankBeforeEnable:ilk,snb
3031	serge	417	*/
		418	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
		419	}
		420
		421	void intel_disable_fbc(struct drm_device *dev)
		422	{
		423	struct drm_i915_private *dev_priv = dev->dev_private;
		424
3482	Serge	425	intel_cancel_fbc_work(dev_priv);
3031	serge	426
3482	Serge	427	if (!dev_priv->display.disable_fbc)
		428	return;
3031	serge	429
3482	Serge	430	dev_priv->display.disable_fbc(dev);
4104	Serge	431	dev_priv->fbc.plane = -1;
3031	serge	432	}
		433
4104	Serge	434	static bool set_no_fbc_reason(struct drm_i915_private *dev_priv,
		435	enum no_fbc_reason reason)
		436	{
		437	if (dev_priv->fbc.no_fbc_reason == reason)
		438	return false;
		439
		440	dev_priv->fbc.no_fbc_reason = reason;
		441	return true;
		442	}
		443
3031	serge	444	/**
		445	* intel_update_fbc - enable/disable FBC as needed
		446	* @dev: the drm_device
		447	*
		448	* Set up the framebuffer compression hardware at mode set time. We
		449	* enable it if possible:
		450	* - plane A only (on pre-965)
		451	* - no pixel mulitply/line duplication
		452	* - no alpha buffer discard
		453	* - no dual wide
4104	Serge	454	* - framebuffer <= max_hdisplay in width, max_vdisplay in height
3031	serge	455	*
		456	* We can't assume that any compression will take place (worst case),
		457	* so the compressed buffer has to be the same size as the uncompressed
		458	* one. It also must reside (along with the line length buffer) in
		459	* stolen memory.
		460	*
		461	* We need to enable/disable FBC on a global basis.
		462	*/
		463	void intel_update_fbc(struct drm_device *dev)
		464	{
		465	struct drm_i915_private *dev_priv = dev->dev_private;
		466	struct drm_crtc crtc = NULL, tmp_crtc;
		467	struct intel_crtc *intel_crtc;
		468	struct drm_framebuffer *fb;
		469	struct intel_framebuffer *intel_fb;
		470	struct drm_i915_gem_object *obj;
4104	Serge	471	unsigned int max_hdisplay, max_vdisplay;
3031	serge	472
4104	Serge	473	if (!I915_HAS_FBC(dev)) {
		474	set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED);
3031	serge	475	return;
4104	Serge	476	}
3031	serge	477
4104	Serge	478	if (!i915_powersave) {
		479	if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
		480	DRM_DEBUG_KMS("fbc disabled per module param\n");
3031	serge	481	return;
4104	Serge	482	}
3031	serge	483
		484	/*
		485	* If FBC is already on, we just have to verify that we can
		486	* keep it that way...
		487	* Need to disable if:
		488	* - more than one pipe is active
		489	* - changing FBC params (stride, fence, mode)
		490	* - new fb is too large to fit in compressed buffer
		491	* - going to an unsupported config (interlace, pixel multiply, etc.)
		492	*/
		493	list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	494	if (intel_crtc_active(tmp_crtc) &&
		495	!to_intel_crtc(tmp_crtc)->primary_disabled) {
3031	serge	496	if (crtc) {
4104	Serge	497	if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES))
3031	serge	498	DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
		499	goto out_disable;
		500	}
		501	crtc = tmp_crtc;
		502	}
		503	}
		504
		505	if (!crtc \|\| crtc->fb == NULL) {
4104	Serge	506	if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT))
3031	serge	507	DRM_DEBUG_KMS("no output, disabling\n");
		508	goto out_disable;
		509	}
		510
		511	intel_crtc = to_intel_crtc(crtc);
		512	fb = crtc->fb;
		513	intel_fb = to_intel_framebuffer(fb);
		514	obj = intel_fb->obj;
		515
4104	Serge	516	if (i915_enable_fbc < 0 &&
		517	INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) {
		518	if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT))
		519	DRM_DEBUG_KMS("disabled per chip default\n");
		520	goto out_disable;
3031	serge	521	}
4104	Serge	522	if (!i915_enable_fbc) {
		523	if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
3031	serge	524	DRM_DEBUG_KMS("fbc disabled per module param\n");
		525	goto out_disable;
		526	}
		527	if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) \|\|
		528	(crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
4104	Serge	529	if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
3031	serge	530	DRM_DEBUG_KMS("mode incompatible with compression, "
		531	"disabling\n");
		532	goto out_disable;
		533	}
4104	Serge	534
		535	if (IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
		536	max_hdisplay = 4096;
		537	max_vdisplay = 2048;
		538	} else {
		539	max_hdisplay = 2048;
		540	max_vdisplay = 1536;
		541	}
		542	if ((crtc->mode.hdisplay > max_hdisplay) \|\|
		543	(crtc->mode.vdisplay > max_vdisplay)) {
		544	if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE))
3031	serge	545	DRM_DEBUG_KMS("mode too large for compression, disabling\n");
		546	goto out_disable;
		547	}
4104	Serge	548	if ((IS_I915GM(dev) \|\| IS_I945GM(dev) \|\| IS_HASWELL(dev)) &&
		549	intel_crtc->plane != 0) {
		550	if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE))
3031	serge	551	DRM_DEBUG_KMS("plane not 0, disabling compression\n");
		552	goto out_disable;
		553	}
		554
		555	/* The use of a CPU fence is mandatory in order to detect writes
		556	* by the CPU to the scanout and trigger updates to the FBC.
		557	*/
		558	if (obj->tiling_mode != I915_TILING_X \|\|
		559	obj->fence_reg == I915_FENCE_REG_NONE) {
4104	Serge	560	if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED))
3031	serge	561	DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
		562	goto out_disable;
		563	}
		564
		565	/* If the kernel debugger is active, always disable compression */
		566	if (in_dbg_master())
		567	goto out_disable;
		568
3480	Serge	569	if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
4104	Serge	570	if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL))
3480	Serge	571	DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
		572	goto out_disable;
		573	}
		574
3031	serge	575	/* If the scanout has not changed, don't modify the FBC settings.
		576	* Note that we make the fundamental assumption that the fb->obj
		577	* cannot be unpinned (and have its GTT offset and fence revoked)
		578	* without first being decoupled from the scanout and FBC disabled.
		579	*/
4104	Serge	580	if (dev_priv->fbc.plane == intel_crtc->plane &&
		581	dev_priv->fbc.fb_id == fb->base.id &&
		582	dev_priv->fbc.y == crtc->y)
3031	serge	583	return;
		584
		585	if (intel_fbc_enabled(dev)) {
		586	/* We update FBC along two paths, after changing fb/crtc
		587	* configuration (modeswitching) and after page-flipping
		588	* finishes. For the latter, we know that not only did
		589	* we disable the FBC at the start of the page-flip
		590	* sequence, but also more than one vblank has passed.
		591	*
		592	* For the former case of modeswitching, it is possible
		593	* to switch between two FBC valid configurations
		594	* instantaneously so we do need to disable the FBC
		595	* before we can modify its control registers. We also
		596	* have to wait for the next vblank for that to take
		597	* effect. However, since we delay enabling FBC we can
		598	* assume that a vblank has passed since disabling and
		599	* that we can safely alter the registers in the deferred
		600	* callback.
		601	*
		602	* In the scenario that we go from a valid to invalid
		603	* and then back to valid FBC configuration we have
		604	* no strict enforcement that a vblank occurred since
		605	* disabling the FBC. However, along all current pipe
		606	* disabling paths we do need to wait for a vblank at
		607	* some point. And we wait before enabling FBC anyway.
		608	*/
		609	DRM_DEBUG_KMS("disabling active FBC for update\n");
		610	intel_disable_fbc(dev);
		611	}
		612
		613	intel_enable_fbc(crtc, 500);
4104	Serge	614	dev_priv->fbc.no_fbc_reason = FBC_OK;
3031	serge	615	return;
		616
		617	out_disable:
		618	/* Multiple disables should be harmless */
		619	if (intel_fbc_enabled(dev)) {
		620	DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
		621	intel_disable_fbc(dev);
		622	}
3480	Serge	623	i915_gem_stolen_cleanup_compression(dev);
3031	serge	624	}
		625
		626	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		627	{
		628	drm_i915_private_t *dev_priv = dev->dev_private;
		629	u32 tmp;
		630
		631	tmp = I915_READ(CLKCFG);
		632
		633	switch (tmp & CLKCFG_FSB_MASK) {
		634	case CLKCFG_FSB_533:
		635	dev_priv->fsb_freq = 533; /* 1334 /
		636	break;
		637	case CLKCFG_FSB_800:
		638	dev_priv->fsb_freq = 800; /* 2004 /
		639	break;
		640	case CLKCFG_FSB_667:
		641	dev_priv->fsb_freq = 667; /* 1674 /
		642	break;
		643	case CLKCFG_FSB_400:
		644	dev_priv->fsb_freq = 400; /* 1004 /
		645	break;
		646	}
		647
		648	switch (tmp & CLKCFG_MEM_MASK) {
		649	case CLKCFG_MEM_533:
		650	dev_priv->mem_freq = 533;
		651	break;
		652	case CLKCFG_MEM_667:
		653	dev_priv->mem_freq = 667;
		654	break;
		655	case CLKCFG_MEM_800:
		656	dev_priv->mem_freq = 800;
		657	break;
		658	}
		659
		660	/* detect pineview DDR3 setting */
		661	tmp = I915_READ(CSHRDDR3CTL);
		662	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		663	}
		664
		665	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		666	{
		667	drm_i915_private_t *dev_priv = dev->dev_private;
		668	u16 ddrpll, csipll;
		669
		670	ddrpll = I915_READ16(DDRMPLL1);
		671	csipll = I915_READ16(CSIPLL0);
		672
		673	switch (ddrpll & 0xff) {
		674	case 0xc:
		675	dev_priv->mem_freq = 800;
		676	break;
		677	case 0x10:
		678	dev_priv->mem_freq = 1066;
		679	break;
		680	case 0x14:
		681	dev_priv->mem_freq = 1333;
		682	break;
		683	case 0x18:
		684	dev_priv->mem_freq = 1600;
		685	break;
		686	default:
		687	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		688	ddrpll & 0xff);
		689	dev_priv->mem_freq = 0;
		690	break;
		691	}
		692
		693	dev_priv->ips.r_t = dev_priv->mem_freq;
		694
		695	switch (csipll & 0x3ff) {
		696	case 0x00c:
		697	dev_priv->fsb_freq = 3200;
		698	break;
		699	case 0x00e:
		700	dev_priv->fsb_freq = 3733;
		701	break;
		702	case 0x010:
		703	dev_priv->fsb_freq = 4266;
		704	break;
		705	case 0x012:
		706	dev_priv->fsb_freq = 4800;
		707	break;
		708	case 0x014:
		709	dev_priv->fsb_freq = 5333;
		710	break;
		711	case 0x016:
		712	dev_priv->fsb_freq = 5866;
		713	break;
		714	case 0x018:
		715	dev_priv->fsb_freq = 6400;
		716	break;
		717	default:
		718	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		719	csipll & 0x3ff);
		720	dev_priv->fsb_freq = 0;
		721	break;
		722	}
		723
		724	if (dev_priv->fsb_freq == 3200) {
		725	dev_priv->ips.c_m = 0;
		726	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		727	dev_priv->ips.c_m = 1;
		728	} else {
		729	dev_priv->ips.c_m = 2;
		730	}
		731	}
		732
		733	static const struct cxsr_latency cxsr_latency_table[] = {
		734	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		735	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		736	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		737	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		738	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		739
		740	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		741	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		742	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		743	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		744	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		745
		746	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		747	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		748	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		749	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		750	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		751
		752	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		753	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		754	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		755	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		756	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		757
		758	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		759	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		760	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		761	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		762	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		763
		764	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		765	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		766	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		767	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		768	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		769	};
		770
		771	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		772	int is_ddr3,
		773	int fsb,
		774	int mem)
		775	{
		776	const struct cxsr_latency *latency;
		777	int i;
		778
		779	if (fsb == 0 \|\| mem == 0)
		780	return NULL;
		781
		782	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		783	latency = &cxsr_latency_table[i];
		784	if (is_desktop == latency->is_desktop &&
		785	is_ddr3 == latency->is_ddr3 &&
		786	fsb == latency->fsb_freq && mem == latency->mem_freq)
		787	return latency;
		788	}
		789
		790	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		791
		792	return NULL;
		793	}
		794
		795	static void pineview_disable_cxsr(struct drm_device *dev)
		796	{
		797	struct drm_i915_private *dev_priv = dev->dev_private;
		798
		799	/* deactivate cxsr */
		800	I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
		801	}
		802
		803	/*
		804	* Latency for FIFO fetches is dependent on several factors:
		805	* - memory configuration (speed, channels)
		806	* - chipset
		807	* - current MCH state
		808	* It can be fairly high in some situations, so here we assume a fairly
		809	* pessimal value. It's a tradeoff between extra memory fetches (if we
		810	* set this value too high, the FIFO will fetch frequently to stay full)
		811	* and power consumption (set it too low to save power and we might see
		812	* FIFO underruns and display "flicker").
		813	*
		814	* A value of 5us seems to be a good balance; safe for very low end
		815	* platforms but not overly aggressive on lower latency configs.
		816	*/
		817	static const int latency_ns = 5000;
		818
		819	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		820	{
		821	struct drm_i915_private *dev_priv = dev->dev_private;
		822	uint32_t dsparb = I915_READ(DSPARB);
		823	int size;
		824
		825	size = dsparb & 0x7f;
		826	if (plane)
		827	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		828
		829	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		830	plane ? "B" : "A", size);
		831
		832	return size;
		833	}
		834
		835	static int i85x_get_fifo_size(struct drm_device *dev, int plane)
		836	{
		837	struct drm_i915_private *dev_priv = dev->dev_private;
		838	uint32_t dsparb = I915_READ(DSPARB);
		839	int size;
		840
		841	size = dsparb & 0x1ff;
		842	if (plane)
		843	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		844	size >>= 1; /* Convert to cachelines */
		845
		846	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		847	plane ? "B" : "A", size);
		848
		849	return size;
		850	}
		851
		852	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		853	{
		854	struct drm_i915_private *dev_priv = dev->dev_private;
		855	uint32_t dsparb = I915_READ(DSPARB);
		856	int size;
		857
		858	size = dsparb & 0x7f;
		859	size >>= 2; /* Convert to cachelines */
		860
		861	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		862	plane ? "B" : "A",
		863	size);
		864
		865	return size;
		866	}
		867
		868	static int i830_get_fifo_size(struct drm_device *dev, int plane)
		869	{
		870	struct drm_i915_private *dev_priv = dev->dev_private;
		871	uint32_t dsparb = I915_READ(DSPARB);
		872	int size;
		873
		874	size = dsparb & 0x7f;
		875	size >>= 1; /* Convert to cachelines */
		876
		877	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		878	plane ? "B" : "A", size);
		879
		880	return size;
		881	}
		882
		883	/* Pineview has different values for various configs */
		884	static const struct intel_watermark_params pineview_display_wm = {
		885	PINEVIEW_DISPLAY_FIFO,
		886	PINEVIEW_MAX_WM,
		887	PINEVIEW_DFT_WM,
		888	PINEVIEW_GUARD_WM,
		889	PINEVIEW_FIFO_LINE_SIZE
		890	};
		891	static const struct intel_watermark_params pineview_display_hplloff_wm = {
		892	PINEVIEW_DISPLAY_FIFO,
		893	PINEVIEW_MAX_WM,
		894	PINEVIEW_DFT_HPLLOFF_WM,
		895	PINEVIEW_GUARD_WM,
		896	PINEVIEW_FIFO_LINE_SIZE
		897	};
		898	static const struct intel_watermark_params pineview_cursor_wm = {
		899	PINEVIEW_CURSOR_FIFO,
		900	PINEVIEW_CURSOR_MAX_WM,
		901	PINEVIEW_CURSOR_DFT_WM,
		902	PINEVIEW_CURSOR_GUARD_WM,
		903	PINEVIEW_FIFO_LINE_SIZE,
		904	};
		905	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
		906	PINEVIEW_CURSOR_FIFO,
		907	PINEVIEW_CURSOR_MAX_WM,
		908	PINEVIEW_CURSOR_DFT_WM,
		909	PINEVIEW_CURSOR_GUARD_WM,
		910	PINEVIEW_FIFO_LINE_SIZE
		911	};
		912	static const struct intel_watermark_params g4x_wm_info = {
		913	G4X_FIFO_SIZE,
		914	G4X_MAX_WM,
		915	G4X_MAX_WM,
		916	2,
		917	G4X_FIFO_LINE_SIZE,
		918	};
		919	static const struct intel_watermark_params g4x_cursor_wm_info = {
		920	I965_CURSOR_FIFO,
		921	I965_CURSOR_MAX_WM,
		922	I965_CURSOR_DFT_WM,
		923	2,
		924	G4X_FIFO_LINE_SIZE,
		925	};
		926	static const struct intel_watermark_params valleyview_wm_info = {
		927	VALLEYVIEW_FIFO_SIZE,
		928	VALLEYVIEW_MAX_WM,
		929	VALLEYVIEW_MAX_WM,
		930	2,
		931	G4X_FIFO_LINE_SIZE,
		932	};
		933	static const struct intel_watermark_params valleyview_cursor_wm_info = {
		934	I965_CURSOR_FIFO,
		935	VALLEYVIEW_CURSOR_MAX_WM,
		936	I965_CURSOR_DFT_WM,
		937	2,
		938	G4X_FIFO_LINE_SIZE,
		939	};
		940	static const struct intel_watermark_params i965_cursor_wm_info = {
		941	I965_CURSOR_FIFO,
		942	I965_CURSOR_MAX_WM,
		943	I965_CURSOR_DFT_WM,
		944	2,
		945	I915_FIFO_LINE_SIZE,
		946	};
		947	static const struct intel_watermark_params i945_wm_info = {
		948	I945_FIFO_SIZE,
		949	I915_MAX_WM,
		950	1,
		951	2,
		952	I915_FIFO_LINE_SIZE
		953	};
		954	static const struct intel_watermark_params i915_wm_info = {
		955	I915_FIFO_SIZE,
		956	I915_MAX_WM,
		957	1,
		958	2,
		959	I915_FIFO_LINE_SIZE
		960	};
		961	static const struct intel_watermark_params i855_wm_info = {
		962	I855GM_FIFO_SIZE,
		963	I915_MAX_WM,
		964	1,
		965	2,
		966	I830_FIFO_LINE_SIZE
		967	};
		968	static const struct intel_watermark_params i830_wm_info = {
		969	I830_FIFO_SIZE,
		970	I915_MAX_WM,
		971	1,
		972	2,
		973	I830_FIFO_LINE_SIZE
		974	};
		975
		976	static const struct intel_watermark_params ironlake_display_wm_info = {
		977	ILK_DISPLAY_FIFO,
		978	ILK_DISPLAY_MAXWM,
		979	ILK_DISPLAY_DFTWM,
		980	2,
		981	ILK_FIFO_LINE_SIZE
		982	};
		983	static const struct intel_watermark_params ironlake_cursor_wm_info = {
		984	ILK_CURSOR_FIFO,
		985	ILK_CURSOR_MAXWM,
		986	ILK_CURSOR_DFTWM,
		987	2,
		988	ILK_FIFO_LINE_SIZE
		989	};
		990	static const struct intel_watermark_params ironlake_display_srwm_info = {
		991	ILK_DISPLAY_SR_FIFO,
		992	ILK_DISPLAY_MAX_SRWM,
		993	ILK_DISPLAY_DFT_SRWM,
		994	2,
		995	ILK_FIFO_LINE_SIZE
		996	};
		997	static const struct intel_watermark_params ironlake_cursor_srwm_info = {
		998	ILK_CURSOR_SR_FIFO,
		999	ILK_CURSOR_MAX_SRWM,
		1000	ILK_CURSOR_DFT_SRWM,
		1001	2,
		1002	ILK_FIFO_LINE_SIZE
		1003	};
		1004
		1005	static const struct intel_watermark_params sandybridge_display_wm_info = {
		1006	SNB_DISPLAY_FIFO,
		1007	SNB_DISPLAY_MAXWM,
		1008	SNB_DISPLAY_DFTWM,
		1009	2,
		1010	SNB_FIFO_LINE_SIZE
		1011	};
		1012	static const struct intel_watermark_params sandybridge_cursor_wm_info = {
		1013	SNB_CURSOR_FIFO,
		1014	SNB_CURSOR_MAXWM,
		1015	SNB_CURSOR_DFTWM,
		1016	2,
		1017	SNB_FIFO_LINE_SIZE
		1018	};
		1019	static const struct intel_watermark_params sandybridge_display_srwm_info = {
		1020	SNB_DISPLAY_SR_FIFO,
		1021	SNB_DISPLAY_MAX_SRWM,
		1022	SNB_DISPLAY_DFT_SRWM,
		1023	2,
		1024	SNB_FIFO_LINE_SIZE
		1025	};
		1026	static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
		1027	SNB_CURSOR_SR_FIFO,
		1028	SNB_CURSOR_MAX_SRWM,
		1029	SNB_CURSOR_DFT_SRWM,
		1030	2,
		1031	SNB_FIFO_LINE_SIZE
		1032	};
		1033
		1034
		1035	/**
		1036	* intel_calculate_wm - calculate watermark level
		1037	* @clock_in_khz: pixel clock
		1038	* @wm: chip FIFO params
		1039	* @pixel_size: display pixel size
		1040	* @latency_ns: memory latency for the platform
		1041	*
		1042	* Calculate the watermark level (the level at which the display plane will
		1043	* start fetching from memory again). Each chip has a different display
		1044	* FIFO size and allocation, so the caller needs to figure that out and pass
		1045	* in the correct intel_watermark_params structure.
		1046	*
		1047	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		1048	* on the pixel size. When it reaches the watermark level, it'll start
		1049	* fetching FIFO line sized based chunks from memory until the FIFO fills
		1050	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		1051	* will occur, and a display engine hang could result.
		1052	*/
		1053	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		1054	const struct intel_watermark_params *wm,
		1055	int fifo_size,
		1056	int pixel_size,
		1057	unsigned long latency_ns)
		1058	{
		1059	long entries_required, wm_size;
		1060
		1061	/*
		1062	* Note: we need to make sure we don't overflow for various clock &
		1063	* latency values.
		1064	* clocks go from a few thousand to several hundred thousand.
		1065	* latency is usually a few thousand
		1066	*/
		1067	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		1068	1000;
		1069	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		1070
		1071	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		1072
		1073	wm_size = fifo_size - (entries_required + wm->guard_size);
		1074
		1075	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		1076
		1077	/* Don't promote wm_size to unsigned... */
		1078	if (wm_size > (long)wm->max_wm)
		1079	wm_size = wm->max_wm;
		1080	if (wm_size <= 0)
		1081	wm_size = wm->default_wm;
		1082	return wm_size;
		1083	}
		1084
		1085	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		1086	{
		1087	struct drm_crtc crtc, enabled = NULL;
		1088
		1089	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	1090	if (intel_crtc_active(crtc)) {
3031	serge	1091	if (enabled)
		1092	return NULL;
		1093	enabled = crtc;
		1094	}
		1095	}
		1096
		1097	return enabled;
		1098	}
		1099
		1100	static void pineview_update_wm(struct drm_device *dev)
		1101	{
		1102	struct drm_i915_private *dev_priv = dev->dev_private;
		1103	struct drm_crtc *crtc;
		1104	const struct cxsr_latency *latency;
		1105	u32 reg;
		1106	unsigned long wm;
		1107
		1108	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		1109	dev_priv->fsb_freq, dev_priv->mem_freq);
		1110	if (!latency) {
		1111	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		1112	pineview_disable_cxsr(dev);
		1113	return;
		1114	}
		1115
		1116	crtc = single_enabled_crtc(dev);
		1117	if (crtc) {
		1118	int clock = crtc->mode.clock;
		1119	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1120
		1121	/* Display SR */
		1122	wm = intel_calculate_wm(clock, &pineview_display_wm,
		1123	pineview_display_wm.fifo_size,
		1124	pixel_size, latency->display_sr);
		1125	reg = I915_READ(DSPFW1);
		1126	reg &= ~DSPFW_SR_MASK;
		1127	reg \|= wm << DSPFW_SR_SHIFT;
		1128	I915_WRITE(DSPFW1, reg);
		1129	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		1130
		1131	/* cursor SR */
		1132	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		1133	pineview_display_wm.fifo_size,
		1134	pixel_size, latency->cursor_sr);
		1135	reg = I915_READ(DSPFW3);
		1136	reg &= ~DSPFW_CURSOR_SR_MASK;
		1137	reg \|= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
		1138	I915_WRITE(DSPFW3, reg);
		1139
		1140	/* Display HPLL off SR */
		1141	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		1142	pineview_display_hplloff_wm.fifo_size,
		1143	pixel_size, latency->display_hpll_disable);
		1144	reg = I915_READ(DSPFW3);
		1145	reg &= ~DSPFW_HPLL_SR_MASK;
		1146	reg \|= wm & DSPFW_HPLL_SR_MASK;
		1147	I915_WRITE(DSPFW3, reg);
		1148
		1149	/* cursor HPLL off SR */
		1150	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		1151	pineview_display_hplloff_wm.fifo_size,
		1152	pixel_size, latency->cursor_hpll_disable);
		1153	reg = I915_READ(DSPFW3);
		1154	reg &= ~DSPFW_HPLL_CURSOR_MASK;
		1155	reg \|= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
		1156	I915_WRITE(DSPFW3, reg);
		1157	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		1158
		1159	/* activate cxsr */
		1160	I915_WRITE(DSPFW3,
		1161	I915_READ(DSPFW3) \| PINEVIEW_SELF_REFRESH_EN);
		1162	DRM_DEBUG_KMS("Self-refresh is enabled\n");
		1163	} else {
		1164	pineview_disable_cxsr(dev);
		1165	DRM_DEBUG_KMS("Self-refresh is disabled\n");
		1166	}
		1167	}
		1168
		1169	static bool g4x_compute_wm0(struct drm_device *dev,
		1170	int plane,
		1171	const struct intel_watermark_params *display,
		1172	int display_latency_ns,
		1173	const struct intel_watermark_params *cursor,
		1174	int cursor_latency_ns,
		1175	int *plane_wm,
		1176	int *cursor_wm)
		1177	{
		1178	struct drm_crtc *crtc;
		1179	int htotal, hdisplay, clock, pixel_size;
		1180	int line_time_us, line_count;
		1181	int entries, tlb_miss;
		1182
		1183	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1184	if (!intel_crtc_active(crtc)) {
3031	serge	1185	*cursor_wm = cursor->guard_size;
		1186	*plane_wm = display->guard_size;
		1187	return false;
		1188	}
		1189
		1190	htotal = crtc->mode.htotal;
		1191	hdisplay = crtc->mode.hdisplay;
		1192	clock = crtc->mode.clock;
		1193	pixel_size = crtc->fb->bits_per_pixel / 8;
		1194
		1195	/* Use the small buffer method to calculate plane watermark */
		1196	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1197	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		1198	if (tlb_miss > 0)
		1199	entries += tlb_miss;
		1200	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1201	*plane_wm = entries + display->guard_size;
		1202	if (*plane_wm > (int)display->max_wm)
		1203	*plane_wm = display->max_wm;
		1204
		1205	/* Use the large buffer method to calculate cursor watermark */
		1206	line_time_us = ((htotal * 1000) / clock);
		1207	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
		1208	entries = line_count * 64 * pixel_size;
		1209	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		1210	if (tlb_miss > 0)
		1211	entries += tlb_miss;
		1212	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1213	*cursor_wm = entries + cursor->guard_size;
		1214	if (*cursor_wm > (int)cursor->max_wm)
		1215	*cursor_wm = (int)cursor->max_wm;
		1216
		1217	return true;
		1218	}
		1219
		1220	/*
		1221	* Check the wm result.
		1222	*
		1223	* If any calculated watermark values is larger than the maximum value that
		1224	* can be programmed into the associated watermark register, that watermark
		1225	* must be disabled.
		1226	*/
		1227	static bool g4x_check_srwm(struct drm_device *dev,
		1228	int display_wm, int cursor_wm,
		1229	const struct intel_watermark_params *display,
		1230	const struct intel_watermark_params *cursor)
		1231	{
		1232	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		1233	display_wm, cursor_wm);
		1234
		1235	if (display_wm > display->max_wm) {
		1236	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		1237	display_wm, display->max_wm);
		1238	return false;
		1239	}
		1240
		1241	if (cursor_wm > cursor->max_wm) {
		1242	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		1243	cursor_wm, cursor->max_wm);
		1244	return false;
		1245	}
		1246
		1247	if (!(display_wm \|\| cursor_wm)) {
		1248	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		1249	return false;
		1250	}
		1251
		1252	return true;
		1253	}
		1254
		1255	static bool g4x_compute_srwm(struct drm_device *dev,
		1256	int plane,
		1257	int latency_ns,
		1258	const struct intel_watermark_params *display,
		1259	const struct intel_watermark_params *cursor,
		1260	int display_wm, int cursor_wm)
		1261	{
		1262	struct drm_crtc *crtc;
		1263	int hdisplay, htotal, pixel_size, clock;
		1264	unsigned long line_time_us;
		1265	int line_count, line_size;
		1266	int small, large;
		1267	int entries;
		1268
		1269	if (!latency_ns) {
		1270	display_wm = cursor_wm = 0;
		1271	return false;
		1272	}
		1273
		1274	crtc = intel_get_crtc_for_plane(dev, plane);
		1275	hdisplay = crtc->mode.hdisplay;
		1276	htotal = crtc->mode.htotal;
		1277	clock = crtc->mode.clock;
		1278	pixel_size = crtc->fb->bits_per_pixel / 8;
		1279
		1280	line_time_us = (htotal * 1000) / clock;
		1281	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1282	line_size = hdisplay * pixel_size;
		1283
		1284	/* Use the minimum of the small and large buffer method for primary */
		1285	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1286	large = line_count * line_size;
		1287
		1288	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1289	*display_wm = entries + display->guard_size;
		1290
		1291	/* calculate the self-refresh watermark for display cursor */
		1292	entries = line_count * pixel_size * 64;
		1293	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1294	*cursor_wm = entries + cursor->guard_size;
		1295
		1296	return g4x_check_srwm(dev,
		1297	display_wm, cursor_wm,
		1298	display, cursor);
		1299	}
		1300
		1301	static bool vlv_compute_drain_latency(struct drm_device *dev,
		1302	int plane,
		1303	int *plane_prec_mult,
		1304	int *plane_dl,
		1305	int *cursor_prec_mult,
		1306	int *cursor_dl)
		1307	{
		1308	struct drm_crtc *crtc;
		1309	int clock, pixel_size;
		1310	int entries;
		1311
		1312	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1313	if (!intel_crtc_active(crtc))
3031	serge	1314	return false;
		1315
		1316	clock = crtc->mode.clock; /* VESA DOT Clock */
		1317	pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */
		1318
		1319	entries = (clock / 1000) * pixel_size;
		1320	*plane_prec_mult = (entries > 256) ?
		1321	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1322	plane_dl = (64 (plane_prec_mult) 4) / ((clock / 1000) *
		1323	pixel_size);
		1324
		1325	entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
		1326	*cursor_prec_mult = (entries > 256) ?
		1327	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1328	cursor_dl = (64 (cursor_prec_mult) 4) / ((clock / 1000) * 4);
		1329
		1330	return true;
		1331	}
		1332
		1333	/*
		1334	* Update drain latency registers of memory arbiter
		1335	*
		1336	* Valleyview SoC has a new memory arbiter and needs drain latency registers
		1337	* to be programmed. Each plane has a drain latency multiplier and a drain
		1338	* latency value.
		1339	*/
		1340
		1341	static void vlv_update_drain_latency(struct drm_device *dev)
		1342	{
		1343	struct drm_i915_private *dev_priv = dev->dev_private;
		1344	int planea_prec, planea_dl, planeb_prec, planeb_dl;
		1345	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
		1346	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
		1347	either 16 or 32 */
		1348
		1349	/* For plane A, Cursor A */
		1350	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
		1351	&cursor_prec_mult, &cursora_dl)) {
		1352	cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1353	DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
		1354	planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1355	DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
		1356
		1357	I915_WRITE(VLV_DDL1, cursora_prec \|
		1358	(cursora_dl << DDL_CURSORA_SHIFT) \|
		1359	planea_prec \| planea_dl);
		1360	}
		1361
		1362	/* For plane B, Cursor B */
		1363	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
		1364	&cursor_prec_mult, &cursorb_dl)) {
		1365	cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1366	DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
		1367	planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1368	DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
		1369
		1370	I915_WRITE(VLV_DDL2, cursorb_prec \|
		1371	(cursorb_dl << DDL_CURSORB_SHIFT) \|
		1372	planeb_prec \| planeb_dl);
		1373	}
		1374	}
		1375
		1376	#define single_plane_enabled(mask) is_power_of_2(mask)
		1377
		1378	static void valleyview_update_wm(struct drm_device *dev)
		1379	{
		1380	static const int sr_latency_ns = 12000;
		1381	struct drm_i915_private *dev_priv = dev->dev_private;
		1382	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1383	int plane_sr, cursor_sr;
3243	Serge	1384	int ignore_plane_sr, ignore_cursor_sr;
3031	serge	1385	unsigned int enabled = 0;
		1386
		1387	vlv_update_drain_latency(dev);
		1388
3746	Serge	1389	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1390	&valleyview_wm_info, latency_ns,
		1391	&valleyview_cursor_wm_info, latency_ns,
		1392	&planea_wm, &cursora_wm))
3746	Serge	1393	enabled \|= 1 << PIPE_A;
3031	serge	1394
3746	Serge	1395	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1396	&valleyview_wm_info, latency_ns,
		1397	&valleyview_cursor_wm_info, latency_ns,
		1398	&planeb_wm, &cursorb_wm))
3746	Serge	1399	enabled \|= 1 << PIPE_B;
3031	serge	1400
		1401	if (single_plane_enabled(enabled) &&
		1402	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1403	sr_latency_ns,
		1404	&valleyview_wm_info,
		1405	&valleyview_cursor_wm_info,
3243	Serge	1406	&plane_sr, &ignore_cursor_sr) &&
		1407	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1408	2*sr_latency_ns,
		1409	&valleyview_wm_info,
		1410	&valleyview_cursor_wm_info,
		1411	&ignore_plane_sr, &cursor_sr)) {
3031	serge	1412	I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
3243	Serge	1413	} else {
3031	serge	1414	I915_WRITE(FW_BLC_SELF_VLV,
		1415	I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
3243	Serge	1416	plane_sr = cursor_sr = 0;
		1417	}
3031	serge	1418
		1419	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1420	planea_wm, cursora_wm,
		1421	planeb_wm, cursorb_wm,
		1422	plane_sr, cursor_sr);
		1423
		1424	I915_WRITE(DSPFW1,
		1425	(plane_sr << DSPFW_SR_SHIFT) \|
		1426	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1427	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1428	planea_wm);
		1429	I915_WRITE(DSPFW2,
3243	Serge	1430	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1431	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1432	I915_WRITE(DSPFW3,
3243	Serge	1433	(I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) \|
		1434	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
3031	serge	1435	}
		1436
		1437	static void g4x_update_wm(struct drm_device *dev)
		1438	{
		1439	static const int sr_latency_ns = 12000;
		1440	struct drm_i915_private *dev_priv = dev->dev_private;
		1441	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1442	int plane_sr, cursor_sr;
		1443	unsigned int enabled = 0;
		1444
3746	Serge	1445	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1446	&g4x_wm_info, latency_ns,
		1447	&g4x_cursor_wm_info, latency_ns,
		1448	&planea_wm, &cursora_wm))
3746	Serge	1449	enabled \|= 1 << PIPE_A;
3031	serge	1450
3746	Serge	1451	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1452	&g4x_wm_info, latency_ns,
		1453	&g4x_cursor_wm_info, latency_ns,
		1454	&planeb_wm, &cursorb_wm))
3746	Serge	1455	enabled \|= 1 << PIPE_B;
3031	serge	1456
		1457	if (single_plane_enabled(enabled) &&
		1458	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1459	sr_latency_ns,
		1460	&g4x_wm_info,
		1461	&g4x_cursor_wm_info,
3243	Serge	1462	&plane_sr, &cursor_sr)) {
3031	serge	1463	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
3243	Serge	1464	} else {
3031	serge	1465	I915_WRITE(FW_BLC_SELF,
		1466	I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
3243	Serge	1467	plane_sr = cursor_sr = 0;
		1468	}
3031	serge	1469
		1470	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1471	planea_wm, cursora_wm,
		1472	planeb_wm, cursorb_wm,
		1473	plane_sr, cursor_sr);
		1474
		1475	I915_WRITE(DSPFW1,
		1476	(plane_sr << DSPFW_SR_SHIFT) \|
		1477	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1478	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1479	planea_wm);
		1480	I915_WRITE(DSPFW2,
3243	Serge	1481	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1482	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1483	/* HPLL off in SR has some issues on G4x... disable it */
		1484	I915_WRITE(DSPFW3,
3243	Serge	1485	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
3031	serge	1486	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1487	}
		1488
		1489	static void i965_update_wm(struct drm_device *dev)
		1490	{
		1491	struct drm_i915_private *dev_priv = dev->dev_private;
		1492	struct drm_crtc *crtc;
		1493	int srwm = 1;
		1494	int cursor_sr = 16;
		1495
		1496	/* Calc sr entries for one plane configs */
		1497	crtc = single_enabled_crtc(dev);
		1498	if (crtc) {
		1499	/* self-refresh has much higher latency */
		1500	static const int sr_latency_ns = 12000;
		1501	int clock = crtc->mode.clock;
		1502	int htotal = crtc->mode.htotal;
		1503	int hdisplay = crtc->mode.hdisplay;
		1504	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1505	unsigned long line_time_us;
		1506	int entries;
		1507
		1508	line_time_us = ((htotal * 1000) / clock);
		1509
		1510	/* Use ns/us then divide to preserve precision */
		1511	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1512	pixel_size * hdisplay;
		1513	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1514	srwm = I965_FIFO_SIZE - entries;
		1515	if (srwm < 0)
		1516	srwm = 1;
		1517	srwm &= 0x1ff;
		1518	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1519	entries, srwm);
		1520
		1521	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1522	pixel_size * 64;
		1523	entries = DIV_ROUND_UP(entries,
		1524	i965_cursor_wm_info.cacheline_size);
		1525	cursor_sr = i965_cursor_wm_info.fifo_size -
		1526	(entries + i965_cursor_wm_info.guard_size);
		1527
		1528	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1529	cursor_sr = i965_cursor_wm_info.max_wm;
		1530
		1531	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1532	"cursor %d\n", srwm, cursor_sr);
		1533
		1534	if (IS_CRESTLINE(dev))
		1535	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
		1536	} else {
		1537	/* Turn off self refresh if both pipes are enabled */
		1538	if (IS_CRESTLINE(dev))
		1539	I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
		1540	& ~FW_BLC_SELF_EN);
		1541	}
		1542
		1543	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1544	srwm);
		1545
		1546	/* 965 has limitations... */
		1547	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) \|
		1548	(8 << 16) \| (8 << 8) \| (8 << 0));
		1549	I915_WRITE(DSPFW2, (8 << 8) \| (8 << 0));
		1550	/* update cursor SR watermark */
		1551	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1552	}
		1553
		1554	static void i9xx_update_wm(struct drm_device *dev)
		1555	{
		1556	struct drm_i915_private *dev_priv = dev->dev_private;
		1557	const struct intel_watermark_params *wm_info;
		1558	uint32_t fwater_lo;
		1559	uint32_t fwater_hi;
		1560	int cwm, srwm = 1;
		1561	int fifo_size;
		1562	int planea_wm, planeb_wm;
		1563	struct drm_crtc crtc, enabled = NULL;
		1564
		1565	if (IS_I945GM(dev))
		1566	wm_info = &i945_wm_info;
		1567	else if (!IS_GEN2(dev))
		1568	wm_info = &i915_wm_info;
		1569	else
		1570	wm_info = &i855_wm_info;
		1571
		1572	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1573	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1574	if (intel_crtc_active(crtc)) {
		1575	int cpp = crtc->fb->bits_per_pixel / 8;
		1576	if (IS_GEN2(dev))
		1577	cpp = 4;
		1578
3031	serge	1579	planea_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1580	wm_info, fifo_size, cpp,
3031	serge	1581	latency_ns);
		1582	enabled = crtc;
		1583	} else
		1584	planea_wm = fifo_size - wm_info->guard_size;
		1585
		1586	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1587	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1588	if (intel_crtc_active(crtc)) {
		1589	int cpp = crtc->fb->bits_per_pixel / 8;
		1590	if (IS_GEN2(dev))
		1591	cpp = 4;
		1592
3031	serge	1593	planeb_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1594	wm_info, fifo_size, cpp,
3031	serge	1595	latency_ns);
		1596	if (enabled == NULL)
		1597	enabled = crtc;
		1598	else
		1599	enabled = NULL;
		1600	} else
		1601	planeb_wm = fifo_size - wm_info->guard_size;
		1602
		1603	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1604
		1605	/*
		1606	* Overlay gets an aggressive default since video jitter is bad.
		1607	*/
		1608	cwm = 2;
		1609
		1610	/* Play safe and disable self-refresh before adjusting watermarks. */
		1611	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1612	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK \| 0);
		1613	else if (IS_I915GM(dev))
		1614	I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
		1615
		1616	/* Calc sr entries for one plane configs */
		1617	if (HAS_FW_BLC(dev) && enabled) {
		1618	/* self-refresh has much higher latency */
		1619	static const int sr_latency_ns = 6000;
		1620	int clock = enabled->mode.clock;
		1621	int htotal = enabled->mode.htotal;
		1622	int hdisplay = enabled->mode.hdisplay;
		1623	int pixel_size = enabled->fb->bits_per_pixel / 8;
		1624	unsigned long line_time_us;
		1625	int entries;
		1626
		1627	line_time_us = (htotal * 1000) / clock;
		1628
		1629	/* Use ns/us then divide to preserve precision */
		1630	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1631	pixel_size * hdisplay;
		1632	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1633	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1634	srwm = wm_info->fifo_size - entries;
		1635	if (srwm < 0)
		1636	srwm = 1;
		1637
		1638	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1639	I915_WRITE(FW_BLC_SELF,
		1640	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1641	else if (IS_I915GM(dev))
		1642	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1643	}
		1644
		1645	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1646	planea_wm, planeb_wm, cwm, srwm);
		1647
		1648	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1649	fwater_hi = (cwm & 0x1f);
		1650
		1651	/* Set request length to 8 cachelines per fetch */
		1652	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1653	fwater_hi = fwater_hi \| (1 << 8);
		1654
		1655	I915_WRITE(FW_BLC, fwater_lo);
		1656	I915_WRITE(FW_BLC2, fwater_hi);
		1657
		1658	if (HAS_FW_BLC(dev)) {
		1659	if (enabled) {
		1660	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1661	I915_WRITE(FW_BLC_SELF,
		1662	FW_BLC_SELF_EN_MASK \| FW_BLC_SELF_EN);
		1663	else if (IS_I915GM(dev))
		1664	I915_WRITE(INSTPM, I915_READ(INSTPM) \| INSTPM_SELF_EN);
		1665	DRM_DEBUG_KMS("memory self refresh enabled\n");
		1666	} else
		1667	DRM_DEBUG_KMS("memory self refresh disabled\n");
		1668	}
		1669	}
		1670
		1671	static void i830_update_wm(struct drm_device *dev)
		1672	{
		1673	struct drm_i915_private *dev_priv = dev->dev_private;
		1674	struct drm_crtc *crtc;
		1675	uint32_t fwater_lo;
		1676	int planea_wm;
		1677
		1678	crtc = single_enabled_crtc(dev);
		1679	if (crtc == NULL)
		1680	return;
		1681
		1682	planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
		1683	dev_priv->display.get_fifo_size(dev, 0),
3243	Serge	1684	4, latency_ns);
3031	serge	1685	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1686	fwater_lo \|= (3<<8) \| planea_wm;
		1687
		1688	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1689
		1690	I915_WRITE(FW_BLC, fwater_lo);
		1691	}
		1692
		1693	/*
		1694	* Check the wm result.
		1695	*
		1696	* If any calculated watermark values is larger than the maximum value that
		1697	* can be programmed into the associated watermark register, that watermark
		1698	* must be disabled.
		1699	*/
		1700	static bool ironlake_check_srwm(struct drm_device *dev, int level,
		1701	int fbc_wm, int display_wm, int cursor_wm,
		1702	const struct intel_watermark_params *display,
		1703	const struct intel_watermark_params *cursor)
		1704	{
		1705	struct drm_i915_private *dev_priv = dev->dev_private;
		1706
		1707	DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
		1708	" cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
		1709
		1710	if (fbc_wm > SNB_FBC_MAX_SRWM) {
		1711	DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
		1712	fbc_wm, SNB_FBC_MAX_SRWM, level);
		1713
		1714	/* fbc has it's own way to disable FBC WM */
		1715	I915_WRITE(DISP_ARB_CTL,
		1716	I915_READ(DISP_ARB_CTL) \| DISP_FBC_WM_DIS);
		1717	return false;
4104	Serge	1718	} else if (INTEL_INFO(dev)->gen >= 6) {
		1719	/* enable FBC WM (except on ILK, where it must remain off) */
		1720	I915_WRITE(DISP_ARB_CTL,
		1721	I915_READ(DISP_ARB_CTL) & ~DISP_FBC_WM_DIS);
3031	serge	1722	}
		1723
		1724	if (display_wm > display->max_wm) {
		1725	DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
		1726	display_wm, SNB_DISPLAY_MAX_SRWM, level);
		1727	return false;
		1728	}
		1729
		1730	if (cursor_wm > cursor->max_wm) {
		1731	DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
		1732	cursor_wm, SNB_CURSOR_MAX_SRWM, level);
		1733	return false;
		1734	}
		1735
		1736	if (!(fbc_wm \|\| display_wm \|\| cursor_wm)) {
		1737	DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
		1738	return false;
		1739	}
		1740
		1741	return true;
		1742	}
		1743
		1744	/*
		1745	* Compute watermark values of WM[1-3],
		1746	*/
		1747	static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
		1748	int latency_ns,
		1749	const struct intel_watermark_params *display,
		1750	const struct intel_watermark_params *cursor,
		1751	int fbc_wm, int display_wm, int *cursor_wm)
		1752	{
		1753	struct drm_crtc *crtc;
		1754	unsigned long line_time_us;
		1755	int hdisplay, htotal, pixel_size, clock;
		1756	int line_count, line_size;
		1757	int small, large;
		1758	int entries;
		1759
		1760	if (!latency_ns) {
		1761	fbc_wm = display_wm = *cursor_wm = 0;
		1762	return false;
		1763	}
		1764
		1765	crtc = intel_get_crtc_for_plane(dev, plane);
		1766	hdisplay = crtc->mode.hdisplay;
		1767	htotal = crtc->mode.htotal;
		1768	clock = crtc->mode.clock;
		1769	pixel_size = crtc->fb->bits_per_pixel / 8;
		1770
		1771	line_time_us = (htotal * 1000) / clock;
		1772	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1773	line_size = hdisplay * pixel_size;
		1774
		1775	/* Use the minimum of the small and large buffer method for primary */
		1776	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1777	large = line_count * line_size;
		1778
		1779	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1780	*display_wm = entries + display->guard_size;
		1781
		1782	/*
		1783	* Spec says:
		1784	* FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
		1785	*/
		1786	fbc_wm = DIV_ROUND_UP(display_wm * 64, line_size) + 2;
		1787
		1788	/* calculate the self-refresh watermark for display cursor */
		1789	entries = line_count * pixel_size * 64;
		1790	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1791	*cursor_wm = entries + cursor->guard_size;
		1792
		1793	return ironlake_check_srwm(dev, level,
		1794	fbc_wm, display_wm, *cursor_wm,
		1795	display, cursor);
		1796	}
		1797
		1798	static void ironlake_update_wm(struct drm_device *dev)
		1799	{
		1800	struct drm_i915_private *dev_priv = dev->dev_private;
		1801	int fbc_wm, plane_wm, cursor_wm;
		1802	unsigned int enabled;
		1803
		1804	enabled = 0;
3746	Serge	1805	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1806	&ironlake_display_wm_info,
4104	Serge	1807	dev_priv->wm.pri_latency[0] * 100,
3031	serge	1808	&ironlake_cursor_wm_info,
4104	Serge	1809	dev_priv->wm.cur_latency[0] * 100,
3031	serge	1810	&plane_wm, &cursor_wm)) {
		1811	I915_WRITE(WM0_PIPEA_ILK,
		1812	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1813	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1814	" plane %d, " "cursor: %d\n",
		1815	plane_wm, cursor_wm);
3746	Serge	1816	enabled \|= 1 << PIPE_A;
3031	serge	1817	}
		1818
3746	Serge	1819	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1820	&ironlake_display_wm_info,
4104	Serge	1821	dev_priv->wm.pri_latency[0] * 100,
3031	serge	1822	&ironlake_cursor_wm_info,
4104	Serge	1823	dev_priv->wm.cur_latency[0] * 100,
3031	serge	1824	&plane_wm, &cursor_wm)) {
		1825	I915_WRITE(WM0_PIPEB_ILK,
		1826	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1827	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1828	" plane %d, cursor: %d\n",
		1829	plane_wm, cursor_wm);
3746	Serge	1830	enabled \|= 1 << PIPE_B;
3031	serge	1831	}
		1832
		1833	/*
		1834	* Calculate and update the self-refresh watermark only when one
		1835	* display plane is used.
		1836	*/
		1837	I915_WRITE(WM3_LP_ILK, 0);
		1838	I915_WRITE(WM2_LP_ILK, 0);
		1839	I915_WRITE(WM1_LP_ILK, 0);
		1840
		1841	if (!single_plane_enabled(enabled))
		1842	return;
		1843	enabled = ffs(enabled) - 1;
		1844
		1845	/* WM1 */
		1846	if (!ironlake_compute_srwm(dev, 1, enabled,
4104	Serge	1847	dev_priv->wm.pri_latency[1] * 500,
3031	serge	1848	&ironlake_display_srwm_info,
		1849	&ironlake_cursor_srwm_info,
		1850	&fbc_wm, &plane_wm, &cursor_wm))
		1851	return;
		1852
		1853	I915_WRITE(WM1_LP_ILK,
		1854	WM1_LP_SR_EN \|
4104	Serge	1855	(dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) \|
3031	serge	1856	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1857	(plane_wm << WM1_LP_SR_SHIFT) \|
		1858	cursor_wm);
		1859
		1860	/* WM2 */
		1861	if (!ironlake_compute_srwm(dev, 2, enabled,
4104	Serge	1862	dev_priv->wm.pri_latency[2] * 500,
3031	serge	1863	&ironlake_display_srwm_info,
		1864	&ironlake_cursor_srwm_info,
		1865	&fbc_wm, &plane_wm, &cursor_wm))
		1866	return;
		1867
		1868	I915_WRITE(WM2_LP_ILK,
		1869	WM2_LP_EN \|
4104	Serge	1870	(dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) \|
3031	serge	1871	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1872	(plane_wm << WM1_LP_SR_SHIFT) \|
		1873	cursor_wm);
		1874
		1875	/*
		1876	* WM3 is unsupported on ILK, probably because we don't have latency
		1877	* data for that power state
		1878	*/
		1879	}
		1880
		1881	static void sandybridge_update_wm(struct drm_device *dev)
		1882	{
		1883	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	1884	int latency = dev_priv->wm.pri_latency[0] * 100; /* In unit 0.1us */
3031	serge	1885	u32 val;
		1886	int fbc_wm, plane_wm, cursor_wm;
		1887	unsigned int enabled;
		1888
		1889	enabled = 0;
3746	Serge	1890	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1891	&sandybridge_display_wm_info, latency,
		1892	&sandybridge_cursor_wm_info, latency,
		1893	&plane_wm, &cursor_wm)) {
		1894	val = I915_READ(WM0_PIPEA_ILK);
		1895	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1896	I915_WRITE(WM0_PIPEA_ILK, val \|
		1897	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1898	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1899	" plane %d, " "cursor: %d\n",
		1900	plane_wm, cursor_wm);
3746	Serge	1901	enabled \|= 1 << PIPE_A;
3031	serge	1902	}
		1903
3746	Serge	1904	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1905	&sandybridge_display_wm_info, latency,
		1906	&sandybridge_cursor_wm_info, latency,
		1907	&plane_wm, &cursor_wm)) {
		1908	val = I915_READ(WM0_PIPEB_ILK);
		1909	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1910	I915_WRITE(WM0_PIPEB_ILK, val \|
		1911	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1912	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1913	" plane %d, cursor: %d\n",
		1914	plane_wm, cursor_wm);
3746	Serge	1915	enabled \|= 1 << PIPE_B;
3031	serge	1916	}
		1917
3243	Serge	1918	/*
		1919	* Calculate and update the self-refresh watermark only when one
		1920	* display plane is used.
		1921	*
		1922	* SNB support 3 levels of watermark.
		1923	*
		1924	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1925	* and disabled in the descending order
		1926	*
		1927	*/
		1928	I915_WRITE(WM3_LP_ILK, 0);
		1929	I915_WRITE(WM2_LP_ILK, 0);
		1930	I915_WRITE(WM1_LP_ILK, 0);
		1931
		1932	if (!single_plane_enabled(enabled) \|\|
		1933	dev_priv->sprite_scaling_enabled)
		1934	return;
		1935	enabled = ffs(enabled) - 1;
		1936
		1937	/* WM1 */
		1938	if (!ironlake_compute_srwm(dev, 1, enabled,
4104	Serge	1939	dev_priv->wm.pri_latency[1] * 500,
3243	Serge	1940	&sandybridge_display_srwm_info,
		1941	&sandybridge_cursor_srwm_info,
		1942	&fbc_wm, &plane_wm, &cursor_wm))
		1943	return;
		1944
		1945	I915_WRITE(WM1_LP_ILK,
		1946	WM1_LP_SR_EN \|
4104	Serge	1947	(dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) \|
3243	Serge	1948	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1949	(plane_wm << WM1_LP_SR_SHIFT) \|
		1950	cursor_wm);
		1951
		1952	/* WM2 */
		1953	if (!ironlake_compute_srwm(dev, 2, enabled,
4104	Serge	1954	dev_priv->wm.pri_latency[2] * 500,
3243	Serge	1955	&sandybridge_display_srwm_info,
		1956	&sandybridge_cursor_srwm_info,
		1957	&fbc_wm, &plane_wm, &cursor_wm))
		1958	return;
		1959
		1960	I915_WRITE(WM2_LP_ILK,
		1961	WM2_LP_EN \|
4104	Serge	1962	(dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) \|
3243	Serge	1963	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1964	(plane_wm << WM1_LP_SR_SHIFT) \|
		1965	cursor_wm);
		1966
		1967	/* WM3 */
		1968	if (!ironlake_compute_srwm(dev, 3, enabled,
4104	Serge	1969	dev_priv->wm.pri_latency[3] * 500,
3243	Serge	1970	&sandybridge_display_srwm_info,
		1971	&sandybridge_cursor_srwm_info,
		1972	&fbc_wm, &plane_wm, &cursor_wm))
		1973	return;
		1974
		1975	I915_WRITE(WM3_LP_ILK,
		1976	WM3_LP_EN \|
4104	Serge	1977	(dev_priv->wm.pri_latency[3] << WM1_LP_LATENCY_SHIFT) \|
3243	Serge	1978	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1979	(plane_wm << WM1_LP_SR_SHIFT) \|
		1980	cursor_wm);
		1981	}
		1982
		1983	static void ivybridge_update_wm(struct drm_device *dev)
		1984	{
		1985	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	1986	int latency = dev_priv->wm.pri_latency[0] * 100; /* In unit 0.1us */
3243	Serge	1987	u32 val;
		1988	int fbc_wm, plane_wm, cursor_wm;
		1989	int ignore_fbc_wm, ignore_plane_wm, ignore_cursor_wm;
		1990	unsigned int enabled;
		1991
		1992	enabled = 0;
3746	Serge	1993	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1994	&sandybridge_display_wm_info, latency,
		1995	&sandybridge_cursor_wm_info, latency,
		1996	&plane_wm, &cursor_wm)) {
3243	Serge	1997	val = I915_READ(WM0_PIPEA_ILK);
		1998	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1999	I915_WRITE(WM0_PIPEA_ILK, val \|
		2000	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		2001	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		2002	" plane %d, " "cursor: %d\n",
		2003	plane_wm, cursor_wm);
3746	Serge	2004	enabled \|= 1 << PIPE_A;
3243	Serge	2005	}
		2006
3746	Serge	2007	if (g4x_compute_wm0(dev, PIPE_B,
3243	Serge	2008	&sandybridge_display_wm_info, latency,
		2009	&sandybridge_cursor_wm_info, latency,
		2010	&plane_wm, &cursor_wm)) {
		2011	val = I915_READ(WM0_PIPEB_ILK);
		2012	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		2013	I915_WRITE(WM0_PIPEB_ILK, val \|
		2014	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		2015	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		2016	" plane %d, cursor: %d\n",
		2017	plane_wm, cursor_wm);
3746	Serge	2018	enabled \|= 1 << PIPE_B;
3243	Serge	2019	}
		2020
3746	Serge	2021	if (g4x_compute_wm0(dev, PIPE_C,
3243	Serge	2022	&sandybridge_display_wm_info, latency,
		2023	&sandybridge_cursor_wm_info, latency,
		2024	&plane_wm, &cursor_wm)) {
3031	serge	2025	val = I915_READ(WM0_PIPEC_IVB);
		2026	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		2027	I915_WRITE(WM0_PIPEC_IVB, val \|
		2028	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		2029	DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
		2030	" plane %d, cursor: %d\n",
		2031	plane_wm, cursor_wm);
3746	Serge	2032	enabled \|= 1 << PIPE_C;
3031	serge	2033	}
		2034
		2035	/*
		2036	* Calculate and update the self-refresh watermark only when one
		2037	* display plane is used.
		2038	*
		2039	* SNB support 3 levels of watermark.
		2040	*
		2041	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		2042	* and disabled in the descending order
		2043	*
		2044	*/
		2045	I915_WRITE(WM3_LP_ILK, 0);
		2046	I915_WRITE(WM2_LP_ILK, 0);
		2047	I915_WRITE(WM1_LP_ILK, 0);
		2048
		2049	if (!single_plane_enabled(enabled) \|\|
		2050	dev_priv->sprite_scaling_enabled)
		2051	return;
		2052	enabled = ffs(enabled) - 1;
		2053
		2054	/* WM1 */
		2055	if (!ironlake_compute_srwm(dev, 1, enabled,
4104	Serge	2056	dev_priv->wm.pri_latency[1] * 500,
3031	serge	2057	&sandybridge_display_srwm_info,
		2058	&sandybridge_cursor_srwm_info,
		2059	&fbc_wm, &plane_wm, &cursor_wm))
		2060	return;
		2061
		2062	I915_WRITE(WM1_LP_ILK,
		2063	WM1_LP_SR_EN \|
4104	Serge	2064	(dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) \|
3031	serge	2065	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2066	(plane_wm << WM1_LP_SR_SHIFT) \|
		2067	cursor_wm);
		2068
		2069	/* WM2 */
		2070	if (!ironlake_compute_srwm(dev, 2, enabled,
4104	Serge	2071	dev_priv->wm.pri_latency[2] * 500,
3031	serge	2072	&sandybridge_display_srwm_info,
		2073	&sandybridge_cursor_srwm_info,
		2074	&fbc_wm, &plane_wm, &cursor_wm))
		2075	return;
		2076
		2077	I915_WRITE(WM2_LP_ILK,
		2078	WM2_LP_EN \|
4104	Serge	2079	(dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) \|
3031	serge	2080	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2081	(plane_wm << WM1_LP_SR_SHIFT) \|
		2082	cursor_wm);
		2083
3243	Serge	2084	/* WM3, note we have to correct the cursor latency */
3031	serge	2085	if (!ironlake_compute_srwm(dev, 3, enabled,
4104	Serge	2086	dev_priv->wm.pri_latency[3] * 500,
3031	serge	2087	&sandybridge_display_srwm_info,
		2088	&sandybridge_cursor_srwm_info,
3243	Serge	2089	&fbc_wm, &plane_wm, &ignore_cursor_wm) \|\|
		2090	!ironlake_compute_srwm(dev, 3, enabled,
4104	Serge	2091	dev_priv->wm.cur_latency[3] * 500,
3243	Serge	2092	&sandybridge_display_srwm_info,
		2093	&sandybridge_cursor_srwm_info,
		2094	&ignore_fbc_wm, &ignore_plane_wm, &cursor_wm))
3031	serge	2095	return;
		2096
		2097	I915_WRITE(WM3_LP_ILK,
		2098	WM3_LP_EN \|
4104	Serge	2099	(dev_priv->wm.pri_latency[3] << WM1_LP_LATENCY_SHIFT) \|
3031	serge	2100	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2101	(plane_wm << WM1_LP_SR_SHIFT) \|
		2102	cursor_wm);
		2103	}
		2104
4104	Serge	2105	static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
		2106	struct drm_crtc *crtc)
3031	serge	2107	{
4104	Serge	2108	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2109	uint32_t pixel_rate;
		2110
		2111	pixel_rate = intel_crtc->config.adjusted_mode.clock;
		2112
		2113	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
		2114	* adjust the pixel_rate here. */
		2115
		2116	if (intel_crtc->config.pch_pfit.enabled) {
		2117	uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
		2118	uint32_t pfit_size = intel_crtc->config.pch_pfit.size;
		2119
		2120	pipe_w = intel_crtc->config.requested_mode.hdisplay;
		2121	pipe_h = intel_crtc->config.requested_mode.vdisplay;
		2122	pfit_w = (pfit_size >> 16) & 0xFFFF;
		2123	pfit_h = pfit_size & 0xFFFF;
		2124	if (pipe_w < pfit_w)
		2125	pipe_w = pfit_w;
		2126	if (pipe_h < pfit_h)
		2127	pipe_h = pfit_h;
		2128
		2129	pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
		2130	pfit_w * pfit_h);
		2131	}
		2132
		2133	return pixel_rate;
		2134	}
		2135
		2136	/* latency must be in 0.1us units. */
		2137	static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		2138	uint32_t latency)
		2139	{
		2140	uint64_t ret;
		2141
		2142	if (WARN(latency == 0, "Latency value missing\n"))
		2143	return UINT_MAX;
		2144
		2145	ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
		2146	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
		2147
		2148	return ret;
		2149	}
		2150
		2151	/* latency must be in 0.1us units. */
		2152	static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		2153	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
		2154	uint32_t latency)
		2155	{
		2156	uint32_t ret;
		2157
		2158	if (WARN(latency == 0, "Latency value missing\n"))
		2159	return UINT_MAX;
		2160
		2161	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		2162	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		2163	ret = DIV_ROUND_UP(ret, 64) + 2;
		2164	return ret;
		2165	}
		2166
		2167	static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
		2168	uint8_t bytes_per_pixel)
		2169	{
		2170	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
		2171	}
		2172
		2173	struct hsw_pipe_wm_parameters {
		2174	bool active;
		2175	uint32_t pipe_htotal;
		2176	uint32_t pixel_rate;
		2177	struct intel_plane_wm_parameters pri;
		2178	struct intel_plane_wm_parameters spr;
		2179	struct intel_plane_wm_parameters cur;
		2180	};
		2181
		2182	struct hsw_wm_maximums {
		2183	uint16_t pri;
		2184	uint16_t spr;
		2185	uint16_t cur;
		2186	uint16_t fbc;
		2187	};
		2188
		2189	struct hsw_wm_values {
		2190	uint32_t wm_pipe[3];
		2191	uint32_t wm_lp[3];
		2192	uint32_t wm_lp_spr[3];
		2193	uint32_t wm_linetime[3];
		2194	bool enable_fbc_wm;
		2195	};
		2196
		2197	/* used in computing the new watermarks state */
		2198	struct intel_wm_config {
		2199	unsigned int num_pipes_active;
		2200	bool sprites_enabled;
		2201	bool sprites_scaled;
		2202	bool fbc_wm_enabled;
		2203	};
		2204
		2205	/*
		2206	* For both WM_PIPE and WM_LP.
		2207	* mem_value must be in 0.1us units.
		2208	*/
		2209	static uint32_t ilk_compute_pri_wm(struct hsw_pipe_wm_parameters *params,
		2210	uint32_t mem_value,
		2211	bool is_lp)
		2212	{
		2213	uint32_t method1, method2;
		2214
		2215	if (!params->active \|\| !params->pri.enabled)
		2216	return 0;
		2217
		2218	method1 = ilk_wm_method1(params->pixel_rate,
		2219	params->pri.bytes_per_pixel,
		2220	mem_value);
		2221
		2222	if (!is_lp)
		2223	return method1;
		2224
		2225	method2 = ilk_wm_method2(params->pixel_rate,
		2226	params->pipe_htotal,
		2227	params->pri.horiz_pixels,
		2228	params->pri.bytes_per_pixel,
		2229	mem_value);
		2230
		2231	return min(method1, method2);
		2232	}
		2233
		2234	/*
		2235	* For both WM_PIPE and WM_LP.
		2236	* mem_value must be in 0.1us units.
		2237	*/
		2238	static uint32_t ilk_compute_spr_wm(struct hsw_pipe_wm_parameters *params,
		2239	uint32_t mem_value)
		2240	{
		2241	uint32_t method1, method2;
		2242
		2243	if (!params->active \|\| !params->spr.enabled)
		2244	return 0;
		2245
		2246	method1 = ilk_wm_method1(params->pixel_rate,
		2247	params->spr.bytes_per_pixel,
		2248	mem_value);
		2249	method2 = ilk_wm_method2(params->pixel_rate,
		2250	params->pipe_htotal,
		2251	params->spr.horiz_pixels,
		2252	params->spr.bytes_per_pixel,
		2253	mem_value);
		2254	return min(method1, method2);
		2255	}
		2256
		2257	/*
		2258	* For both WM_PIPE and WM_LP.
		2259	* mem_value must be in 0.1us units.
		2260	*/
		2261	static uint32_t ilk_compute_cur_wm(struct hsw_pipe_wm_parameters *params,
		2262	uint32_t mem_value)
		2263	{
		2264	if (!params->active \|\| !params->cur.enabled)
		2265	return 0;
		2266
		2267	return ilk_wm_method2(params->pixel_rate,
		2268	params->pipe_htotal,
		2269	params->cur.horiz_pixels,
		2270	params->cur.bytes_per_pixel,
		2271	mem_value);
		2272	}
		2273
		2274	/* Only for WM_LP. */
		2275	static uint32_t ilk_compute_fbc_wm(struct hsw_pipe_wm_parameters *params,
		2276	uint32_t pri_val)
		2277	{
		2278	if (!params->active \|\| !params->pri.enabled)
		2279	return 0;
		2280
		2281	return ilk_wm_fbc(pri_val,
		2282	params->pri.horiz_pixels,
		2283	params->pri.bytes_per_pixel);
		2284	}
		2285
		2286	static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
		2287	{
		2288	if (INTEL_INFO(dev)->gen >= 7)
		2289	return 768;
		2290	else
		2291	return 512;
		2292	}
		2293
		2294	/* Calculate the maximum primary/sprite plane watermark */
		2295	static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
		2296	int level,
		2297	const struct intel_wm_config *config,
		2298	enum intel_ddb_partitioning ddb_partitioning,
		2299	bool is_sprite)
		2300	{
		2301	unsigned int fifo_size = ilk_display_fifo_size(dev);
		2302	unsigned int max;
		2303
		2304	/* if sprites aren't enabled, sprites get nothing */
		2305	if (is_sprite && !config->sprites_enabled)
		2306	return 0;
		2307
		2308	/* HSW allows LP1+ watermarks even with multiple pipes */
		2309	if (level == 0 \|\| config->num_pipes_active > 1) {
		2310	fifo_size /= INTEL_INFO(dev)->num_pipes;
		2311
		2312	/*
		2313	* For some reason the non self refresh
		2314	* FIFO size is only half of the self
		2315	* refresh FIFO size on ILK/SNB.
		2316	*/
		2317	if (INTEL_INFO(dev)->gen <= 6)
		2318	fifo_size /= 2;
		2319	}
		2320
		2321	if (config->sprites_enabled) {
		2322	/* level 0 is always calculated with 1:1 split */
		2323	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
		2324	if (is_sprite)
		2325	fifo_size *= 5;
		2326	fifo_size /= 6;
		2327	} else {
		2328	fifo_size /= 2;
		2329	}
		2330	}
		2331
		2332	/* clamp to max that the registers can hold */
		2333	if (INTEL_INFO(dev)->gen >= 7)
		2334	/* IVB/HSW primary/sprite plane watermarks */
		2335	max = level == 0 ? 127 : 1023;
		2336	else if (!is_sprite)
		2337	/* ILK/SNB primary plane watermarks */
		2338	max = level == 0 ? 127 : 511;
		2339	else
		2340	/* ILK/SNB sprite plane watermarks */
		2341	max = level == 0 ? 63 : 255;
		2342
		2343	return min(fifo_size, max);
		2344	}
		2345
		2346	/* Calculate the maximum cursor plane watermark */
		2347	static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
		2348	int level,
		2349	const struct intel_wm_config *config)
		2350	{
		2351	/* HSW LP1+ watermarks w/ multiple pipes */
		2352	if (level > 0 && config->num_pipes_active > 1)
		2353	return 64;
		2354
		2355	/* otherwise just report max that registers can hold */
		2356	if (INTEL_INFO(dev)->gen >= 7)
		2357	return level == 0 ? 63 : 255;
		2358	else
		2359	return level == 0 ? 31 : 63;
		2360	}
		2361
		2362	/* Calculate the maximum FBC watermark */
		2363	static unsigned int ilk_fbc_wm_max(void)
		2364	{
		2365	/* max that registers can hold */
		2366	return 15;
		2367	}
		2368
		2369	static void ilk_wm_max(struct drm_device *dev,
		2370	int level,
		2371	const struct intel_wm_config *config,
		2372	enum intel_ddb_partitioning ddb_partitioning,
		2373	struct hsw_wm_maximums *max)
		2374	{
		2375	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
		2376	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
		2377	max->cur = ilk_cursor_wm_max(dev, level, config);
		2378	max->fbc = ilk_fbc_wm_max();
		2379	}
		2380
		2381	static bool ilk_check_wm(int level,
		2382	const struct hsw_wm_maximums *max,
		2383	struct intel_wm_level *result)
		2384	{
		2385	bool ret;
		2386
		2387	/* already determined to be invalid? */
		2388	if (!result->enable)
		2389	return false;
		2390
		2391	result->enable = result->pri_val <= max->pri &&
		2392	result->spr_val <= max->spr &&
		2393	result->cur_val <= max->cur;
		2394
		2395	ret = result->enable;
		2396
		2397	/*
		2398	* HACK until we can pre-compute everything,
		2399	* and thus fail gracefully if LP0 watermarks
		2400	* are exceeded...
		2401	*/
		2402	if (level == 0 && !result->enable) {
		2403	if (result->pri_val > max->pri)
		2404	DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
		2405	level, result->pri_val, max->pri);
		2406	if (result->spr_val > max->spr)
		2407	DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
		2408	level, result->spr_val, max->spr);
		2409	if (result->cur_val > max->cur)
		2410	DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
		2411	level, result->cur_val, max->cur);
		2412
		2413	result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
		2414	result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
		2415	result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
		2416	result->enable = true;
		2417	}
		2418
		2419	DRM_DEBUG_KMS("WM%d: %sabled\n", level, result->enable ? "en" : "dis");
		2420
		2421	return ret;
		2422	}
		2423
		2424	static void ilk_compute_wm_level(struct drm_i915_private *dev_priv,
		2425	int level,
		2426	struct hsw_pipe_wm_parameters *p,
		2427	struct intel_wm_level *result)
		2428	{
		2429	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
		2430	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
		2431	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
		2432
		2433	/* WM1+ latency values stored in 0.5us units */
		2434	if (level > 0) {
		2435	pri_latency *= 5;
		2436	spr_latency *= 5;
		2437	cur_latency *= 5;
		2438	}
		2439
		2440	result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
		2441	result->spr_val = ilk_compute_spr_wm(p, spr_latency);
		2442	result->cur_val = ilk_compute_cur_wm(p, cur_latency);
		2443	result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
		2444	result->enable = true;
		2445	}
		2446
		2447	static bool hsw_compute_lp_wm(struct drm_i915_private *dev_priv,
		2448	int level, struct hsw_wm_maximums *max,
		2449	struct hsw_pipe_wm_parameters *params,
		2450	struct intel_wm_level *result)
		2451	{
		2452	enum pipe pipe;
		2453	struct intel_wm_level res[3];
		2454
		2455	for (pipe = PIPE_A; pipe <= PIPE_C; pipe++)
		2456	ilk_compute_wm_level(dev_priv, level, ¶ms[pipe], &res[pipe]);
		2457
		2458	result->pri_val = max3(res[0].pri_val, res[1].pri_val, res[2].pri_val);
		2459	result->spr_val = max3(res[0].spr_val, res[1].spr_val, res[2].spr_val);
		2460	result->cur_val = max3(res[0].cur_val, res[1].cur_val, res[2].cur_val);
		2461	result->fbc_val = max3(res[0].fbc_val, res[1].fbc_val, res[2].fbc_val);
		2462	result->enable = true;
		2463
		2464	return ilk_check_wm(level, max, result);
		2465	}
		2466
		2467	static uint32_t hsw_compute_wm_pipe(struct drm_i915_private *dev_priv,
		2468	enum pipe pipe,
		2469	struct hsw_pipe_wm_parameters *params)
		2470	{
		2471	uint32_t pri_val, cur_val, spr_val;
		2472	/* WM0 latency values stored in 0.1us units */
		2473	uint16_t pri_latency = dev_priv->wm.pri_latency[0];
		2474	uint16_t spr_latency = dev_priv->wm.spr_latency[0];
		2475	uint16_t cur_latency = dev_priv->wm.cur_latency[0];
		2476
		2477	pri_val = ilk_compute_pri_wm(params, pri_latency, false);
		2478	spr_val = ilk_compute_spr_wm(params, spr_latency);
		2479	cur_val = ilk_compute_cur_wm(params, cur_latency);
		2480
		2481	WARN(pri_val > 127,
		2482	"Primary WM error, mode not supported for pipe %c\n",
		2483	pipe_name(pipe));
		2484	WARN(spr_val > 127,
		2485	"Sprite WM error, mode not supported for pipe %c\n",
		2486	pipe_name(pipe));
		2487	WARN(cur_val > 63,
		2488	"Cursor WM error, mode not supported for pipe %c\n",
		2489	pipe_name(pipe));
		2490
		2491	return (pri_val << WM0_PIPE_PLANE_SHIFT) \|
		2492	(spr_val << WM0_PIPE_SPRITE_SHIFT) \|
		2493	cur_val;
		2494	}
		2495
		2496	static uint32_t
		2497	hsw_compute_linetime_wm(struct drm_device dev, struct drm_crtc crtc)
		2498	{
3031	serge	2499	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2500	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2501	struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
		2502	u32 linetime, ips_linetime;
3031	serge	2503
4104	Serge	2504	if (!intel_crtc_active(crtc))
		2505	return 0;
3031	serge	2506
		2507	/* The WM are computed with base on how long it takes to fill a single
		2508	* row at the given clock rate, multiplied by 8.
		2509	* */
4104	Serge	2510	linetime = DIV_ROUND_CLOSEST(mode->htotal * 1000 * 8, mode->clock);
		2511	ips_linetime = DIV_ROUND_CLOSEST(mode->htotal * 1000 * 8,
		2512	intel_ddi_get_cdclk_freq(dev_priv));
3031	serge	2513
4104	Serge	2514	return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) \|
		2515	PIPE_WM_LINETIME_TIME(linetime);
		2516	}
		2517
		2518	static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2519	{
		2520	struct drm_i915_private *dev_priv = dev->dev_private;
		2521
		2522	if (IS_HASWELL(dev)) {
		2523	uint64_t sskpd = I915_READ64(MCH_SSKPD);
		2524
		2525	wm[0] = (sskpd >> 56) & 0xFF;
		2526	if (wm[0] == 0)
		2527	wm[0] = sskpd & 0xF;
		2528	wm[1] = (sskpd >> 4) & 0xFF;
		2529	wm[2] = (sskpd >> 12) & 0xFF;
		2530	wm[3] = (sskpd >> 20) & 0x1FF;
		2531	wm[4] = (sskpd >> 32) & 0x1FF;
		2532	} else if (INTEL_INFO(dev)->gen >= 6) {
		2533	uint32_t sskpd = I915_READ(MCH_SSKPD);
		2534
		2535	wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		2536	wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		2537	wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		2538	wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
		2539	} else if (INTEL_INFO(dev)->gen >= 5) {
		2540	uint32_t mltr = I915_READ(MLTR_ILK);
		2541
		2542	/* ILK primary LP0 latency is 700 ns */
		2543	wm[0] = 7;
		2544	wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		2545	wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
		2546	}
		2547	}
		2548
		2549	static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2550	{
		2551	/* ILK sprite LP0 latency is 1300 ns */
		2552	if (INTEL_INFO(dev)->gen == 5)
		2553	wm[0] = 13;
		2554	}
		2555
		2556	static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2557	{
		2558	/* ILK cursor LP0 latency is 1300 ns */
		2559	if (INTEL_INFO(dev)->gen == 5)
		2560	wm[0] = 13;
		2561
		2562	/* WaDoubleCursorLP3Latency:ivb */
		2563	if (IS_IVYBRIDGE(dev))
		2564	wm[3] *= 2;
		2565	}
		2566
		2567	static void intel_print_wm_latency(struct drm_device *dev,
		2568	const char *name,
		2569	const uint16_t wm[5])
		2570	{
		2571	int level, max_level;
		2572
		2573	/* how many WM levels are we expecting */
		2574	if (IS_HASWELL(dev))
		2575	max_level = 4;
		2576	else if (INTEL_INFO(dev)->gen >= 6)
		2577	max_level = 3;
		2578	else
		2579	max_level = 2;
		2580
		2581	for (level = 0; level <= max_level; level++) {
		2582	unsigned int latency = wm[level];
		2583
		2584	if (latency == 0) {
		2585	DRM_ERROR("%s WM%d latency not provided\n",
		2586	name, level);
		2587	continue;
		2588	}
		2589
		2590	/* WM1+ latency values in 0.5us units */
		2591	if (level > 0)
		2592	latency *= 5;
		2593
		2594	DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
		2595	name, level, wm[level],
		2596	latency / 10, latency % 10);
		2597	}
		2598	}
		2599
		2600	static void intel_setup_wm_latency(struct drm_device *dev)
		2601	{
		2602	struct drm_i915_private *dev_priv = dev->dev_private;
		2603
		2604	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
		2605
		2606	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
		2607	sizeof(dev_priv->wm.pri_latency));
		2608	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
		2609	sizeof(dev_priv->wm.pri_latency));
		2610
		2611	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
		2612	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
		2613
		2614	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2615	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2616	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
		2617	}
		2618
		2619	static void hsw_compute_wm_parameters(struct drm_device *dev,
		2620	struct hsw_pipe_wm_parameters *params,
		2621	struct hsw_wm_maximums *lp_max_1_2,
		2622	struct hsw_wm_maximums *lp_max_5_6)
		2623	{
		2624	struct drm_crtc *crtc;
		2625	struct drm_plane *plane;
		2626	enum pipe pipe;
		2627	struct intel_wm_config config = {};
		2628
		2629	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
		2630	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2631	struct hsw_pipe_wm_parameters *p;
		2632
		2633	pipe = intel_crtc->pipe;
		2634	p = ¶ms[pipe];
		2635
		2636	p->active = intel_crtc_active(crtc);
		2637	if (!p->active)
		2638	continue;
		2639
		2640	config.num_pipes_active++;
		2641
		2642	p->pipe_htotal = intel_crtc->config.adjusted_mode.htotal;
		2643	p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
		2644	p->pri.bytes_per_pixel = crtc->fb->bits_per_pixel / 8;
		2645	p->cur.bytes_per_pixel = 4;
		2646	p->pri.horiz_pixels =
		2647	intel_crtc->config.requested_mode.hdisplay;
		2648	p->cur.horiz_pixels = 64;
		2649	/* TODO: for now, assume primary and cursor planes are always enabled. */
		2650	p->pri.enabled = true;
		2651	p->cur.enabled = true;
		2652	}
		2653
		2654	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		2655	struct intel_plane *intel_plane = to_intel_plane(plane);
		2656	struct hsw_pipe_wm_parameters *p;
		2657
		2658	pipe = intel_plane->pipe;
		2659	p = ¶ms[pipe];
		2660
		2661	p->spr = intel_plane->wm;
		2662
		2663	config.sprites_enabled \|= p->spr.enabled;
		2664	config.sprites_scaled \|= p->spr.scaled;
		2665	}
		2666
		2667	ilk_wm_max(dev, 1, &config, INTEL_DDB_PART_1_2, lp_max_1_2);
		2668
		2669	/* 5/6 split only in single pipe config on IVB+ */
		2670	if (INTEL_INFO(dev)->gen >= 7 && config.num_pipes_active <= 1)
		2671	ilk_wm_max(dev, 1, &config, INTEL_DDB_PART_5_6, lp_max_5_6);
		2672	else
		2673	lp_max_5_6 = lp_max_1_2;
		2674	}
		2675
		2676	static void hsw_compute_wm_results(struct drm_device *dev,
		2677	struct hsw_pipe_wm_parameters *params,
		2678	struct hsw_wm_maximums *lp_maximums,
		2679	struct hsw_wm_values *results)
		2680	{
		2681	struct drm_i915_private *dev_priv = dev->dev_private;
		2682	struct drm_crtc *crtc;
		2683	struct intel_wm_level lp_results[4] = {};
		2684	enum pipe pipe;
		2685	int level, max_level, wm_lp;
		2686
		2687	for (level = 1; level <= 4; level++)
		2688	if (!hsw_compute_lp_wm(dev_priv, level,
		2689	lp_maximums, params,
		2690	&lp_results[level - 1]))
		2691	break;
		2692	max_level = level - 1;
		2693
		2694	memset(results, 0, sizeof(*results));
		2695
		2696	/* The spec says it is preferred to disable FBC WMs instead of disabling
		2697	* a WM level. */
		2698	results->enable_fbc_wm = true;
		2699	for (level = 1; level <= max_level; level++) {
		2700	if (lp_results[level - 1].fbc_val > lp_maximums->fbc) {
		2701	results->enable_fbc_wm = false;
		2702	lp_results[level - 1].fbc_val = 0;
		2703	}
		2704	}
		2705
		2706	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2707	const struct intel_wm_level *r;
		2708
		2709	level = (max_level == 4 && wm_lp > 1) ? wm_lp + 1 : wm_lp;
		2710	if (level > max_level)
		2711	break;
		2712
		2713	r = &lp_results[level - 1];
		2714	results->wm_lp[wm_lp - 1] = HSW_WM_LP_VAL(level * 2,
		2715	r->fbc_val,
		2716	r->pri_val,
		2717	r->cur_val);
		2718	results->wm_lp_spr[wm_lp - 1] = r->spr_val;
		2719	}
		2720
		2721	for_each_pipe(pipe)
		2722	results->wm_pipe[pipe] = hsw_compute_wm_pipe(dev_priv, pipe,
		2723	¶ms[pipe]);
		2724
		2725	for_each_pipe(pipe) {
		2726	crtc = dev_priv->pipe_to_crtc_mapping[pipe];
		2727	results->wm_linetime[pipe] = hsw_compute_linetime_wm(dev, crtc);
		2728	}
		2729	}
		2730
		2731	/* Find the result with the highest level enabled. Check for enable_fbc_wm in
		2732	* case both are at the same level. Prefer r1 in case they're the same. */
		2733	static struct hsw_wm_values hsw_find_best_result(struct hsw_wm_values r1,
		2734	struct hsw_wm_values *r2)
		2735	{
		2736	int i, val_r1 = 0, val_r2 = 0;
		2737
		2738	for (i = 0; i < 3; i++) {
		2739	if (r1->wm_lp[i] & WM3_LP_EN)
		2740	val_r1 = r1->wm_lp[i] & WM1_LP_LATENCY_MASK;
		2741	if (r2->wm_lp[i] & WM3_LP_EN)
		2742	val_r2 = r2->wm_lp[i] & WM1_LP_LATENCY_MASK;
		2743	}
		2744
		2745	if (val_r1 == val_r2) {
		2746	if (r2->enable_fbc_wm && !r1->enable_fbc_wm)
		2747	return r2;
		2748	else
		2749	return r1;
		2750	} else if (val_r1 > val_r2) {
		2751	return r1;
		2752	} else {
		2753	return r2;
		2754	}
		2755	}
		2756
		2757	/*
		2758	* The spec says we shouldn't write when we don't need, because every write
		2759	* causes WMs to be re-evaluated, expending some power.
3031	serge	2760	*/
4104	Serge	2761	static void hsw_write_wm_values(struct drm_i915_private *dev_priv,
		2762	struct hsw_wm_values *results,
		2763	enum intel_ddb_partitioning partitioning)
		2764	{
		2765	struct hsw_wm_values previous;
		2766	uint32_t val;
		2767	enum intel_ddb_partitioning prev_partitioning;
		2768	bool prev_enable_fbc_wm;
3031	serge	2769
4104	Serge	2770	previous.wm_pipe[0] = I915_READ(WM0_PIPEA_ILK);
		2771	previous.wm_pipe[1] = I915_READ(WM0_PIPEB_ILK);
		2772	previous.wm_pipe[2] = I915_READ(WM0_PIPEC_IVB);
		2773	previous.wm_lp[0] = I915_READ(WM1_LP_ILK);
		2774	previous.wm_lp[1] = I915_READ(WM2_LP_ILK);
		2775	previous.wm_lp[2] = I915_READ(WM3_LP_ILK);
		2776	previous.wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
		2777	previous.wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		2778	previous.wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
		2779	previous.wm_linetime[0] = I915_READ(PIPE_WM_LINETIME(PIPE_A));
		2780	previous.wm_linetime[1] = I915_READ(PIPE_WM_LINETIME(PIPE_B));
		2781	previous.wm_linetime[2] = I915_READ(PIPE_WM_LINETIME(PIPE_C));
		2782
		2783	prev_partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
		2784	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
		2785
		2786	prev_enable_fbc_wm = !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
		2787
		2788	if (memcmp(results->wm_pipe, previous.wm_pipe,
		2789	sizeof(results->wm_pipe)) == 0 &&
		2790	memcmp(results->wm_lp, previous.wm_lp,
		2791	sizeof(results->wm_lp)) == 0 &&
		2792	memcmp(results->wm_lp_spr, previous.wm_lp_spr,
		2793	sizeof(results->wm_lp_spr)) == 0 &&
		2794	memcmp(results->wm_linetime, previous.wm_linetime,
		2795	sizeof(results->wm_linetime)) == 0 &&
		2796	partitioning == prev_partitioning &&
		2797	results->enable_fbc_wm == prev_enable_fbc_wm)
		2798	return;
		2799
		2800	if (previous.wm_lp[2] != 0)
		2801	I915_WRITE(WM3_LP_ILK, 0);
		2802	if (previous.wm_lp[1] != 0)
		2803	I915_WRITE(WM2_LP_ILK, 0);
		2804	if (previous.wm_lp[0] != 0)
		2805	I915_WRITE(WM1_LP_ILK, 0);
		2806
		2807	if (previous.wm_pipe[0] != results->wm_pipe[0])
		2808	I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
		2809	if (previous.wm_pipe[1] != results->wm_pipe[1])
		2810	I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
		2811	if (previous.wm_pipe[2] != results->wm_pipe[2])
		2812	I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
		2813
		2814	if (previous.wm_linetime[0] != results->wm_linetime[0])
		2815	I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
		2816	if (previous.wm_linetime[1] != results->wm_linetime[1])
		2817	I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
		2818	if (previous.wm_linetime[2] != results->wm_linetime[2])
		2819	I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
		2820
		2821	if (prev_partitioning != partitioning) {
		2822	val = I915_READ(WM_MISC);
		2823	if (partitioning == INTEL_DDB_PART_1_2)
		2824	val &= ~WM_MISC_DATA_PARTITION_5_6;
		2825	else
		2826	val \|= WM_MISC_DATA_PARTITION_5_6;
		2827	I915_WRITE(WM_MISC, val);
		2828	}
		2829
		2830	if (prev_enable_fbc_wm != results->enable_fbc_wm) {
		2831	val = I915_READ(DISP_ARB_CTL);
		2832	if (results->enable_fbc_wm)
		2833	val &= ~DISP_FBC_WM_DIS;
		2834	else
		2835	val \|= DISP_FBC_WM_DIS;
		2836	I915_WRITE(DISP_ARB_CTL, val);
		2837	}
		2838
		2839	if (previous.wm_lp_spr[0] != results->wm_lp_spr[0])
		2840	I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
		2841	if (previous.wm_lp_spr[1] != results->wm_lp_spr[1])
		2842	I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
		2843	if (previous.wm_lp_spr[2] != results->wm_lp_spr[2])
		2844	I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
		2845
		2846	if (results->wm_lp[0] != 0)
		2847	I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
		2848	if (results->wm_lp[1] != 0)
		2849	I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
		2850	if (results->wm_lp[2] != 0)
		2851	I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3031	serge	2852	}
		2853
4104	Serge	2854	static void haswell_update_wm(struct drm_device *dev)
		2855	{
		2856	struct drm_i915_private *dev_priv = dev->dev_private;
		2857	struct hsw_wm_maximums lp_max_1_2, lp_max_5_6;
		2858	struct hsw_pipe_wm_parameters params[3];
		2859	struct hsw_wm_values results_1_2, results_5_6, *best_results;
		2860	enum intel_ddb_partitioning partitioning;
		2861
		2862	hsw_compute_wm_parameters(dev, params, &lp_max_1_2, &lp_max_5_6);
		2863
		2864	hsw_compute_wm_results(dev, params,
		2865	&lp_max_1_2, &results_1_2);
		2866	if (lp_max_1_2.pri != lp_max_5_6.pri) {
		2867	hsw_compute_wm_results(dev, params,
		2868	&lp_max_5_6, &results_5_6);
		2869	best_results = hsw_find_best_result(&results_1_2, &results_5_6);
		2870	} else {
		2871	best_results = &results_1_2;
		2872	}
		2873
		2874	partitioning = (best_results == &results_1_2) ?
		2875	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
		2876
		2877	hsw_write_wm_values(dev_priv, best_results, partitioning);
		2878	}
		2879
		2880	static void haswell_update_sprite_wm(struct drm_plane *plane,
		2881	struct drm_crtc *crtc,
		2882	uint32_t sprite_width, int pixel_size,
		2883	bool enabled, bool scaled)
		2884	{
		2885	struct intel_plane *intel_plane = to_intel_plane(plane);
		2886
		2887	intel_plane->wm.enabled = enabled;
		2888	intel_plane->wm.scaled = scaled;
		2889	intel_plane->wm.horiz_pixels = sprite_width;
		2890	intel_plane->wm.bytes_per_pixel = pixel_size;
		2891
		2892	haswell_update_wm(plane->dev);
		2893	}
		2894
3031	serge	2895	static bool
		2896	sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
		2897	uint32_t sprite_width, int pixel_size,
		2898	const struct intel_watermark_params *display,
		2899	int display_latency_ns, int *sprite_wm)
		2900	{
		2901	struct drm_crtc *crtc;
		2902	int clock;
		2903	int entries, tlb_miss;
		2904
		2905	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	2906	if (!intel_crtc_active(crtc)) {
3031	serge	2907	*sprite_wm = display->guard_size;
		2908	return false;
		2909	}
		2910
		2911	clock = crtc->mode.clock;
		2912
		2913	/* Use the small buffer method to calculate the sprite watermark */
		2914	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		2915	tlb_miss = display->fifo_size*display->cacheline_size -
		2916	sprite_width * 8;
		2917	if (tlb_miss > 0)
		2918	entries += tlb_miss;
		2919	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		2920	*sprite_wm = entries + display->guard_size;
		2921	if (*sprite_wm > (int)display->max_wm)
		2922	*sprite_wm = display->max_wm;
		2923
		2924	return true;
		2925	}
		2926
		2927	static bool
		2928	sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
		2929	uint32_t sprite_width, int pixel_size,
		2930	const struct intel_watermark_params *display,
		2931	int latency_ns, int *sprite_wm)
		2932	{
		2933	struct drm_crtc *crtc;
		2934	unsigned long line_time_us;
		2935	int clock;
		2936	int line_count, line_size;
		2937	int small, large;
		2938	int entries;
		2939
		2940	if (!latency_ns) {
		2941	*sprite_wm = 0;
		2942	return false;
		2943	}
		2944
		2945	crtc = intel_get_crtc_for_plane(dev, plane);
		2946	clock = crtc->mode.clock;
		2947	if (!clock) {
		2948	*sprite_wm = 0;
		2949	return false;
		2950	}
		2951
		2952	line_time_us = (sprite_width * 1000) / clock;
		2953	if (!line_time_us) {
		2954	*sprite_wm = 0;
		2955	return false;
		2956	}
		2957
		2958	line_count = (latency_ns / line_time_us + 1000) / 1000;
		2959	line_size = sprite_width * pixel_size;
		2960
		2961	/* Use the minimum of the small and large buffer method for primary */
		2962	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		2963	large = line_count * line_size;
		2964
		2965	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		2966	*sprite_wm = entries + display->guard_size;
		2967
		2968	return *sprite_wm > 0x3ff ? false : true;
		2969	}
		2970
4104	Serge	2971	static void sandybridge_update_sprite_wm(struct drm_plane *plane,
		2972	struct drm_crtc *crtc,
		2973	uint32_t sprite_width, int pixel_size,
		2974	bool enabled, bool scaled)
3031	serge	2975	{
4104	Serge	2976	struct drm_device *dev = plane->dev;
3031	serge	2977	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2978	int pipe = to_intel_plane(plane)->pipe;
		2979	int latency = dev_priv->wm.spr_latency[0] * 100; /* In unit 0.1us */
3031	serge	2980	u32 val;
		2981	int sprite_wm, reg;
		2982	int ret;
		2983
4104	Serge	2984	if (!enabled)
		2985	return;
		2986
3031	serge	2987	switch (pipe) {
		2988	case 0:
		2989	reg = WM0_PIPEA_ILK;
		2990	break;
		2991	case 1:
		2992	reg = WM0_PIPEB_ILK;
		2993	break;
		2994	case 2:
		2995	reg = WM0_PIPEC_IVB;
		2996	break;
		2997	default:
		2998	return; /* bad pipe */
		2999	}
		3000
		3001	ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
		3002	&sandybridge_display_wm_info,
		3003	latency, &sprite_wm);
		3004	if (!ret) {
4104	Serge	3005	DRM_DEBUG_KMS("failed to compute sprite wm for pipe %c\n",
		3006	pipe_name(pipe));
3031	serge	3007	return;
		3008	}
		3009
		3010	val = I915_READ(reg);
		3011	val &= ~WM0_PIPE_SPRITE_MASK;
		3012	I915_WRITE(reg, val \| (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
4104	Serge	3013	DRM_DEBUG_KMS("sprite watermarks For pipe %c - %d\n", pipe_name(pipe), sprite_wm);
3031	serge	3014
		3015
		3016	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		3017	pixel_size,
		3018	&sandybridge_display_srwm_info,
4104	Serge	3019	dev_priv->wm.spr_latency[1] * 500,
3031	serge	3020	&sprite_wm);
		3021	if (!ret) {
4104	Serge	3022	DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %c\n",
		3023	pipe_name(pipe));
3031	serge	3024	return;
		3025	}
		3026	I915_WRITE(WM1S_LP_ILK, sprite_wm);
		3027
		3028	/* Only IVB has two more LP watermarks for sprite */
		3029	if (!IS_IVYBRIDGE(dev))
		3030	return;
		3031
		3032	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		3033	pixel_size,
		3034	&sandybridge_display_srwm_info,
4104	Serge	3035	dev_priv->wm.spr_latency[2] * 500,
3031	serge	3036	&sprite_wm);
		3037	if (!ret) {
4104	Serge	3038	DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %c\n",
		3039	pipe_name(pipe));
3031	serge	3040	return;
		3041	}
		3042	I915_WRITE(WM2S_LP_IVB, sprite_wm);
		3043
		3044	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		3045	pixel_size,
		3046	&sandybridge_display_srwm_info,
4104	Serge	3047	dev_priv->wm.spr_latency[3] * 500,
3031	serge	3048	&sprite_wm);
		3049	if (!ret) {
4104	Serge	3050	DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %c\n",
		3051	pipe_name(pipe));
3031	serge	3052	return;
		3053	}
		3054	I915_WRITE(WM3S_LP_IVB, sprite_wm);
		3055	}
		3056
		3057	/**
		3058	* intel_update_watermarks - update FIFO watermark values based on current modes
		3059	*
		3060	* Calculate watermark values for the various WM regs based on current mode
		3061	* and plane configuration.
		3062	*
		3063	* There are several cases to deal with here:
		3064	* - normal (i.e. non-self-refresh)
		3065	* - self-refresh (SR) mode
		3066	* - lines are large relative to FIFO size (buffer can hold up to 2)
		3067	* - lines are small relative to FIFO size (buffer can hold more than 2
		3068	* lines), so need to account for TLB latency
		3069	*
		3070	* The normal calculation is:
		3071	* watermark = dotclock * bytes per pixel * latency
		3072	* where latency is platform & configuration dependent (we assume pessimal
		3073	* values here).
		3074	*
		3075	* The SR calculation is:
		3076	* watermark = (trunc(latency/line time)+1) * surface width *
		3077	* bytes per pixel
		3078	* where
		3079	* line time = htotal / dotclock
		3080	* surface width = hdisplay for normal plane and 64 for cursor
		3081	* and latency is assumed to be high, as above.
		3082	*
		3083	* The final value programmed to the register should always be rounded up,
		3084	* and include an extra 2 entries to account for clock crossings.
		3085	*
		3086	* We don't use the sprite, so we can ignore that. And on Crestline we have
		3087	* to set the non-SR watermarks to 8.
		3088	*/
		3089	void intel_update_watermarks(struct drm_device *dev)
		3090	{
		3091	struct drm_i915_private *dev_priv = dev->dev_private;
		3092
		3093	if (dev_priv->display.update_wm)
		3094	dev_priv->display.update_wm(dev);
		3095	}
		3096
4104	Serge	3097	void intel_update_sprite_watermarks(struct drm_plane *plane,
		3098	struct drm_crtc *crtc,
		3099	uint32_t sprite_width, int pixel_size,
		3100	bool enabled, bool scaled)
3031	serge	3101	{
4104	Serge	3102	struct drm_i915_private *dev_priv = plane->dev->dev_private;
3031	serge	3103
		3104	if (dev_priv->display.update_sprite_wm)
4104	Serge	3105	dev_priv->display.update_sprite_wm(plane, crtc, sprite_width,
		3106	pixel_size, enabled, scaled);
3031	serge	3107	}
		3108
		3109	static struct drm_i915_gem_object *
		3110	intel_alloc_context_page(struct drm_device *dev)
		3111	{
		3112	struct drm_i915_gem_object *ctx;
		3113	int ret;
		3114
		3115	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		3116
		3117	ctx = i915_gem_alloc_object(dev, 4096);
		3118	if (!ctx) {
		3119	DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
		3120	return NULL;
		3121	}
		3122
4104	Serge	3123	ret = i915_gem_obj_ggtt_pin(ctx, 4096, true, false);
3031	serge	3124	if (ret) {
		3125	DRM_ERROR("failed to pin power context: %d\n", ret);
		3126	goto err_unref;
		3127	}
		3128
		3129	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
		3130	if (ret) {
		3131	DRM_ERROR("failed to set-domain on power context: %d\n", ret);
		3132	goto err_unpin;
		3133	}
		3134
		3135	return ctx;
		3136
		3137	err_unpin:
		3138	i915_gem_object_unpin(ctx);
		3139	err_unref:
		3140	drm_gem_object_unreference(&ctx->base);
		3141	return NULL;
		3142	}
		3143
		3144	/**
		3145	* Lock protecting IPS related data structures
		3146	*/
		3147	DEFINE_SPINLOCK(mchdev_lock);
		3148
		3149	/* Global for IPS driver to get at the current i915 device. Protected by
		3150	* mchdev_lock. */
		3151	static struct drm_i915_private *i915_mch_dev;
		3152
		3153	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		3154	{
		3155	struct drm_i915_private *dev_priv = dev->dev_private;
		3156	u16 rgvswctl;
		3157
		3158	assert_spin_locked(&mchdev_lock);
		3159
		3160	rgvswctl = I915_READ16(MEMSWCTL);
		3161	if (rgvswctl & MEMCTL_CMD_STS) {
		3162	DRM_DEBUG("gpu busy, RCS change rejected\n");
		3163	return false; /* still busy with another command */
		3164	}
		3165
		3166	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		3167	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		3168	I915_WRITE16(MEMSWCTL, rgvswctl);
		3169	POSTING_READ16(MEMSWCTL);
		3170
		3171	rgvswctl \|= MEMCTL_CMD_STS;
		3172	I915_WRITE16(MEMSWCTL, rgvswctl);
		3173
		3174	return true;
		3175	}
		3176
		3177	static void ironlake_enable_drps(struct drm_device *dev)
		3178	{
		3179	struct drm_i915_private *dev_priv = dev->dev_private;
		3180	u32 rgvmodectl = I915_READ(MEMMODECTL);
		3181	u8 fmax, fmin, fstart, vstart;
		3182
		3183	spin_lock_irq(&mchdev_lock);
		3184
		3185	/* Enable temp reporting */
		3186	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		3187	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		3188
		3189	/* 100ms RC evaluation intervals */
		3190	I915_WRITE(RCUPEI, 100000);
		3191	I915_WRITE(RCDNEI, 100000);
		3192
		3193	/* Set max/min thresholds to 90ms and 80ms respectively */
		3194	I915_WRITE(RCBMAXAVG, 90000);
		3195	I915_WRITE(RCBMINAVG, 80000);
		3196
		3197	I915_WRITE(MEMIHYST, 1);
		3198
		3199	/* Set up min, max, and cur for interrupt handling */
		3200	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		3201	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		3202	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		3203	MEMMODE_FSTART_SHIFT;
		3204
		3205	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
		3206	PXVFREQ_PX_SHIFT;
		3207
		3208	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		3209	dev_priv->ips.fstart = fstart;
		3210
		3211	dev_priv->ips.max_delay = fstart;
		3212	dev_priv->ips.min_delay = fmin;
		3213	dev_priv->ips.cur_delay = fstart;
		3214
		3215	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		3216	fmax, fmin, fstart);
		3217
		3218	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		3219
		3220	/*
		3221	* Interrupts will be enabled in ironlake_irq_postinstall
		3222	*/
		3223
		3224	I915_WRITE(VIDSTART, vstart);
		3225	POSTING_READ(VIDSTART);
		3226
		3227	rgvmodectl \|= MEMMODE_SWMODE_EN;
		3228	I915_WRITE(MEMMODECTL, rgvmodectl);
		3229
		3230	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		3231	DRM_ERROR("stuck trying to change perf mode\n");
		3232	mdelay(1);
		3233
		3234	ironlake_set_drps(dev, fstart);
		3235
		3236	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
		3237	I915_READ(0x112e0);
		3238	dev_priv->ips.last_time1 = jiffies_to_msecs(GetTimerTicks());
		3239	dev_priv->ips.last_count2 = I915_READ(0x112f4);
3482	Serge	3240	getrawmonotonic(&dev_priv->ips.last_time2);
3031	serge	3241
		3242	spin_unlock_irq(&mchdev_lock);
		3243	}
		3244
		3245	static void ironlake_disable_drps(struct drm_device *dev)
		3246	{
		3247	struct drm_i915_private *dev_priv = dev->dev_private;
		3248	u16 rgvswctl;
		3249
		3250	spin_lock_irq(&mchdev_lock);
		3251
		3252	rgvswctl = I915_READ16(MEMSWCTL);
		3253
		3254	/* Ack interrupts, disable EFC interrupt */
		3255	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		3256	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		3257	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		3258	I915_WRITE(DEIIR, DE_PCU_EVENT);
		3259	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		3260
		3261	/* Go back to the starting frequency */
		3262	ironlake_set_drps(dev, dev_priv->ips.fstart);
		3263	mdelay(1);
		3264	rgvswctl \|= MEMCTL_CMD_STS;
		3265	I915_WRITE(MEMSWCTL, rgvswctl);
		3266	mdelay(1);
		3267
		3268	spin_unlock_irq(&mchdev_lock);
		3269	}
		3270
		3271	/* There's a funny hw issue where the hw returns all 0 when reading from
		3272	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		3273	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		3274	* all limits and the gpu stuck at whatever frequency it is at atm).
		3275	*/
		3276	static u32 gen6_rps_limits(struct drm_i915_private dev_priv, u8 val)
		3277	{
		3278	u32 limits;
		3279
		3280	limits = 0;
		3281
		3282	if (*val >= dev_priv->rps.max_delay)
		3283	*val = dev_priv->rps.max_delay;
		3284	limits \|= dev_priv->rps.max_delay << 24;
		3285
		3286	/* Only set the down limit when we've reached the lowest level to avoid
		3287	* getting more interrupts, otherwise leave this clear. This prevents a
		3288	* race in the hw when coming out of rc6: There's a tiny window where
		3289	* the hw runs at the minimal clock before selecting the desired
		3290	* frequency, if the down threshold expires in that window we will not
		3291	* receive a down interrupt. */
		3292	if (*val <= dev_priv->rps.min_delay) {
		3293	*val = dev_priv->rps.min_delay;
		3294	limits \|= dev_priv->rps.min_delay << 16;
		3295	}
		3296
		3297	return limits;
		3298	}
		3299
		3300	void gen6_set_rps(struct drm_device *dev, u8 val)
		3301	{
		3302	struct drm_i915_private *dev_priv = dev->dev_private;
		3303	u32 limits = gen6_rps_limits(dev_priv, &val);
		3304
3243	Serge	3305	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3306	WARN_ON(val > dev_priv->rps.max_delay);
		3307	WARN_ON(val < dev_priv->rps.min_delay);
		3308
		3309	if (val == dev_priv->rps.cur_delay)
		3310	return;
		3311
3746	Serge	3312	if (IS_HASWELL(dev))
		3313	I915_WRITE(GEN6_RPNSWREQ,
		3314	HSW_FREQUENCY(val));
		3315	else
3031	serge	3316	I915_WRITE(GEN6_RPNSWREQ,
		3317	GEN6_FREQUENCY(val) \|
		3318	GEN6_OFFSET(0) \|
		3319	GEN6_AGGRESSIVE_TURBO);
		3320
		3321	/* Make sure we continue to get interrupts
		3322	* until we hit the minimum or maximum frequencies.
		3323	*/
		3324	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
		3325
		3326	POSTING_READ(GEN6_RPNSWREQ);
		3327
		3328	dev_priv->rps.cur_delay = val;
		3329
		3330	trace_intel_gpu_freq_change(val * 50);
		3331	}
		3332
4104	Serge	3333	/*
		3334	* Wait until the previous freq change has completed,
		3335	* or the timeout elapsed, and then update our notion
		3336	* of the current GPU frequency.
		3337	*/
		3338	static void vlv_update_rps_cur_delay(struct drm_i915_private *dev_priv)
3031	serge	3339	{
4104	Serge	3340	u32 pval;
		3341
		3342	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		3343
		3344	if (wait_for(((pval = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS)) & GENFREQSTATUS) == 0, 10))
		3345	DRM_DEBUG_DRIVER("timed out waiting for Punit\n");
		3346
		3347	pval >>= 8;
		3348
		3349	if (pval != dev_priv->rps.cur_delay)
		3350	DRM_DEBUG_DRIVER("Punit overrode GPU freq: %d MHz (%u) requested, but got %d Mhz (%u)\n",
		3351	vlv_gpu_freq(dev_priv->mem_freq, dev_priv->rps.cur_delay),
		3352	dev_priv->rps.cur_delay,
		3353	vlv_gpu_freq(dev_priv->mem_freq, pval), pval);
		3354
		3355	dev_priv->rps.cur_delay = pval;
		3356	}
		3357
		3358	void valleyview_set_rps(struct drm_device *dev, u8 val)
		3359	{
3031	serge	3360	struct drm_i915_private *dev_priv = dev->dev_private;
		3361
4104	Serge	3362	gen6_rps_limits(dev_priv, &val);
		3363
		3364	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		3365	WARN_ON(val > dev_priv->rps.max_delay);
		3366	WARN_ON(val < dev_priv->rps.min_delay);
		3367
		3368	vlv_update_rps_cur_delay(dev_priv);
		3369
		3370	DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
		3371	vlv_gpu_freq(dev_priv->mem_freq,
		3372	dev_priv->rps.cur_delay),
		3373	dev_priv->rps.cur_delay,
		3374	vlv_gpu_freq(dev_priv->mem_freq, val), val);
		3375
		3376	if (val == dev_priv->rps.cur_delay)
		3377	return;
		3378
		3379	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
		3380
		3381	dev_priv->rps.cur_delay = val;
		3382
		3383	trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv->mem_freq, val));
		3384	}
		3385
		3386	static void gen6_disable_rps_interrupts(struct drm_device *dev)
		3387	{
		3388	struct drm_i915_private *dev_priv = dev->dev_private;
		3389
3031	serge	3390	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
4104	Serge	3391	I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & ~GEN6_PM_RPS_EVENTS);
3031	serge	3392	/* Complete PM interrupt masking here doesn't race with the rps work
		3393	* item again unmasking PM interrupts because that is using a different
		3394	* register (PMIMR) to mask PM interrupts. The only risk is in leaving
		3395	* stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
		3396
4104	Serge	3397	spin_lock_irq(&dev_priv->irq_lock);
3031	serge	3398	dev_priv->rps.pm_iir = 0;
4104	Serge	3399	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	3400
4104	Serge	3401	I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
3031	serge	3402	}
		3403
4104	Serge	3404	static void gen6_disable_rps(struct drm_device *dev)
		3405	{
		3406	struct drm_i915_private *dev_priv = dev->dev_private;
		3407
		3408	I915_WRITE(GEN6_RC_CONTROL, 0);
		3409	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		3410
		3411	gen6_disable_rps_interrupts(dev);
		3412	}
		3413
		3414	static void valleyview_disable_rps(struct drm_device *dev)
		3415	{
		3416	struct drm_i915_private *dev_priv = dev->dev_private;
		3417
		3418	I915_WRITE(GEN6_RC_CONTROL, 0);
		3419
		3420	gen6_disable_rps_interrupts(dev);
		3421
		3422	if (dev_priv->vlv_pctx) {
		3423	drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
		3424	dev_priv->vlv_pctx = NULL;
		3425	}
		3426	}
		3427
3031	serge	3428	int intel_enable_rc6(const struct drm_device *dev)
		3429	{
4104	Serge	3430	/* No RC6 before Ironlake */
		3431	if (INTEL_INFO(dev)->gen < 5)
		3432	return 0;
		3433
3031	serge	3434	/* Respect the kernel parameter if it is set */
		3435	if (i915_enable_rc6 >= 0)
		3436	return i915_enable_rc6;
		3437
3120	serge	3438	/* Disable RC6 on Ironlake */
		3439	if (INTEL_INFO(dev)->gen == 5)
		3440	return 0;
3031	serge	3441
		3442	if (IS_HASWELL(dev)) {
		3443	DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
		3444	return INTEL_RC6_ENABLE;
		3445	}
		3446
		3447	/* snb/ivb have more than one rc6 state. */
		3448	if (INTEL_INFO(dev)->gen == 6) {
		3449	DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
		3450	return INTEL_RC6_ENABLE;
		3451	}
		3452
		3453	DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
		3454	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
		3455	}
		3456
4104	Serge	3457	static void gen6_enable_rps_interrupts(struct drm_device *dev)
		3458	{
		3459	struct drm_i915_private *dev_priv = dev->dev_private;
		3460	u32 enabled_intrs;
		3461
		3462	spin_lock_irq(&dev_priv->irq_lock);
		3463	WARN_ON(dev_priv->rps.pm_iir);
		3464	snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
		3465	I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
		3466	spin_unlock_irq(&dev_priv->irq_lock);
		3467
		3468	/* only unmask PM interrupts we need. Mask all others. */
		3469	enabled_intrs = GEN6_PM_RPS_EVENTS;
		3470
		3471	/* IVB and SNB hard hangs on looping batchbuffer
		3472	* if GEN6_PM_UP_EI_EXPIRED is masked.
		3473	*/
		3474	if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
		3475	enabled_intrs \|= GEN6_PM_RP_UP_EI_EXPIRED;
		3476
		3477	I915_WRITE(GEN6_PMINTRMSK, ~enabled_intrs);
		3478	}
		3479
3031	serge	3480	static void gen6_enable_rps(struct drm_device *dev)
		3481	{
		3482	struct drm_i915_private *dev_priv = dev->dev_private;
		3483	struct intel_ring_buffer *ring;
		3484	u32 rp_state_cap;
		3485	u32 gt_perf_status;
3243	Serge	3486	u32 rc6vids, pcu_mbox, rc6_mask = 0;
3031	serge	3487	u32 gtfifodbg;
		3488	int rc6_mode;
3243	Serge	3489	int i, ret;
3031	serge	3490
3243	Serge	3491	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3492
		3493	/* Here begins a magic sequence of register writes to enable
		3494	* auto-downclocking.
		3495	*
		3496	* Perhaps there might be some value in exposing these to
		3497	* userspace...
		3498	*/
		3499	I915_WRITE(GEN6_RC_STATE, 0);
		3500
		3501	/* Clear the DBG now so we don't confuse earlier errors */
		3502	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		3503	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		3504	I915_WRITE(GTFIFODBG, gtfifodbg);
		3505	}
		3506
		3507	gen6_gt_force_wake_get(dev_priv);
		3508
		3509	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		3510	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
		3511
3746	Serge	3512	/* In units of 50MHz */
		3513	dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff;
3031	serge	3514	dev_priv->rps.min_delay = (rp_state_cap & 0xff0000) >> 16;
		3515	dev_priv->rps.cur_delay = 0;
		3516
		3517	/* disable the counters and set deterministic thresholds */
		3518	I915_WRITE(GEN6_RC_CONTROL, 0);
		3519
		3520	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		3521	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		3522	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		3523	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		3524	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		3525
		3526	for_each_ring(ring, dev_priv, i)
		3527	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		3528
		3529	I915_WRITE(GEN6_RC_SLEEP, 0);
		3530	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4104	Serge	3531	if (INTEL_INFO(dev)->gen <= 6 \|\| IS_IVYBRIDGE(dev))
		3532	I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
		3533	else
3031	serge	3534	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	3535	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	3536	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		3537
		3538	/* Check if we are enabling RC6 */
		3539	rc6_mode = intel_enable_rc6(dev_priv->dev);
		3540	if (rc6_mode & INTEL_RC6_ENABLE)
		3541	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		3542
		3543	/* We don't use those on Haswell */
		3544	if (!IS_HASWELL(dev)) {
		3545	if (rc6_mode & INTEL_RC6p_ENABLE)
		3546	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		3547
		3548	if (rc6_mode & INTEL_RC6pp_ENABLE)
		3549	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		3550	}
		3551
		3552	DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
		3553	(rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		3554	(rc6_mask & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		3555	(rc6_mask & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
		3556
		3557	I915_WRITE(GEN6_RC_CONTROL,
		3558	rc6_mask \|
		3559	GEN6_RC_CTL_EI_MODE(1) \|
		3560	GEN6_RC_CTL_HW_ENABLE);
		3561
3746	Serge	3562	if (IS_HASWELL(dev)) {
		3563	I915_WRITE(GEN6_RPNSWREQ,
		3564	HSW_FREQUENCY(10));
		3565	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		3566	HSW_FREQUENCY(12));
		3567	} else {
3031	serge	3568	I915_WRITE(GEN6_RPNSWREQ,
		3569	GEN6_FREQUENCY(10) \|
		3570	GEN6_OFFSET(0) \|
		3571	GEN6_AGGRESSIVE_TURBO);
		3572	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		3573	GEN6_FREQUENCY(12));
3746	Serge	3574	}
3031	serge	3575
		3576	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
		3577	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
		3578	dev_priv->rps.max_delay << 24 \|
		3579	dev_priv->rps.min_delay << 16);
		3580
		3581	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		3582	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		3583	I915_WRITE(GEN6_RP_UP_EI, 66000);
		3584	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		3585
		3586	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		3587	I915_WRITE(GEN6_RP_CONTROL,
		3588	GEN6_RP_MEDIA_TURBO \|
		3589	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3590	GEN6_RP_MEDIA_IS_GFX \|
		3591	GEN6_RP_ENABLE \|
		3592	GEN6_RP_UP_BUSY_AVG \|
		3593	(IS_HASWELL(dev) ? GEN7_RP_DOWN_IDLE_AVG : GEN6_RP_DOWN_IDLE_CONT));
		3594
3243	Serge	3595	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
		3596	if (!ret) {
		3597	pcu_mbox = 0;
		3598	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
3746	Serge	3599	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		3600	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
		3601	(dev_priv->rps.max_delay & 0xff) * 50,
		3602	(pcu_mbox & 0xff) * 50);
		3603	dev_priv->rps.hw_max = pcu_mbox & 0xff;
3031	serge	3604	}
3243	Serge	3605	} else {
		3606	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		3607	}
3031	serge	3608
		3609	gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
		3610
4104	Serge	3611	gen6_enable_rps_interrupts(dev);
3031	serge	3612
3243	Serge	3613	rc6vids = 0;
		3614	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		3615	if (IS_GEN6(dev) && ret) {
		3616	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		3617	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		3618	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		3619	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		3620	rc6vids &= 0xffff00;
		3621	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		3622	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		3623	if (ret)
		3624	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		3625	}
		3626
3031	serge	3627	gen6_gt_force_wake_put(dev_priv);
		3628	}
		3629
4104	Serge	3630	void gen6_update_ring_freq(struct drm_device *dev)
3031	serge	3631	{
		3632	struct drm_i915_private *dev_priv = dev->dev_private;
		3633	int min_freq = 15;
3746	Serge	3634	unsigned int gpu_freq;
		3635	unsigned int max_ia_freq, min_ring_freq;
3031	serge	3636	int scaling_factor = 180;
		3637
3243	Serge	3638	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3639
		3640	max_ia_freq = cpufreq_quick_get_max(0);
		3641	/*
		3642	* Default to measured freq if none found, PCU will ensure we don't go
		3643	* over
		3644	*/
		3645	if (!max_ia_freq)
		3646	max_ia_freq = tsc_khz;
		3647
		3648	/* Convert from kHz to MHz */
		3649	max_ia_freq /= 1000;
		3650
3746	Serge	3651	min_ring_freq = I915_READ(MCHBAR_MIRROR_BASE_SNB + DCLK);
		3652	/* convert DDR frequency from units of 133.3MHz to bandwidth */
		3653	min_ring_freq = (2 * 4 * min_ring_freq + 2) / 3;
		3654
3031	serge	3655	/*
		3656	* For each potential GPU frequency, load a ring frequency we'd like
		3657	* to use for memory access. We do this by specifying the IA frequency
		3658	* the PCU should use as a reference to determine the ring frequency.
		3659	*/
		3660	for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
		3661	gpu_freq--) {
		3662	int diff = dev_priv->rps.max_delay - gpu_freq;
3746	Serge	3663	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	3664
3746	Serge	3665	if (IS_HASWELL(dev)) {
		3666	ring_freq = (gpu_freq * 5 + 3) / 4;
		3667	ring_freq = max(min_ring_freq, ring_freq);
		3668	/* leave ia_freq as the default, chosen by cpufreq */
		3669	} else {
		3670	/* On older processors, there is no separate ring
		3671	* clock domain, so in order to boost the bandwidth
		3672	* of the ring, we need to upclock the CPU (ia_freq).
		3673	*
		3674	* For GPU frequencies less than 750MHz,
		3675	* just use the lowest ring freq.
3031	serge	3676	*/
		3677	if (gpu_freq < min_freq)
		3678	ia_freq = 800;
		3679	else
		3680	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		3681	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	3682	}
3031	serge	3683
3243	Serge	3684	sandybridge_pcode_write(dev_priv,
		3685	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	3686	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		3687	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		3688	gpu_freq);
3031	serge	3689	}
		3690	}
		3691
4104	Serge	3692	int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
		3693	{
		3694	u32 val, rp0;
		3695
		3696	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		3697
		3698	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
		3699	/* Clamp to max */
		3700	rp0 = min_t(u32, rp0, 0xea);
		3701
		3702	return rp0;
		3703	}
		3704
		3705	static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		3706	{
		3707	u32 val, rpe;
		3708
		3709	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
		3710	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
		3711	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
		3712	rpe \|= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
		3713
		3714	return rpe;
		3715	}
		3716
		3717	int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
		3718	{
		3719	return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
		3720	}
		3721
		3722	static void vlv_rps_timer_work(struct work_struct *work)
		3723	{
		3724	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
		3725	rps.vlv_work.work);
		3726
		3727	/*
		3728	* Timer fired, we must be idle. Drop to min voltage state.
		3729	* Note: we use RPe here since it should match the
		3730	* Vmin we were shooting for. That should give us better
		3731	* perf when we come back out of RC6 than if we used the
		3732	* min freq available.
		3733	*/
		3734	mutex_lock(&dev_priv->rps.hw_lock);
		3735	if (dev_priv->rps.cur_delay > dev_priv->rps.rpe_delay)
		3736	valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay);
		3737	mutex_unlock(&dev_priv->rps.hw_lock);
		3738	}
		3739
		3740	static void valleyview_setup_pctx(struct drm_device *dev)
		3741	{
		3742	struct drm_i915_private *dev_priv = dev->dev_private;
		3743	struct drm_i915_gem_object *pctx;
		3744	unsigned long pctx_paddr;
		3745	u32 pcbr;
		3746	int pctx_size = 24*1024;
		3747
		3748	pcbr = I915_READ(VLV_PCBR);
		3749	if (pcbr) {
		3750	/* BIOS set it up already, grab the pre-alloc'd space */
		3751	int pcbr_offset;
		3752
		3753	pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
		3754	pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
		3755	pcbr_offset,
		3756	I915_GTT_OFFSET_NONE,
		3757	pctx_size);
		3758	goto out;
		3759	}
		3760
		3761	/*
		3762	* From the Gunit register HAS:
		3763	* The Gfx driver is expected to program this register and ensure
		3764	* proper allocation within Gfx stolen memory. For example, this
		3765	* register should be programmed such than the PCBR range does not
		3766	* overlap with other ranges, such as the frame buffer, protected
		3767	* memory, or any other relevant ranges.
		3768	*/
		3769	pctx = i915_gem_object_create_stolen(dev, pctx_size);
		3770	if (!pctx) {
		3771	DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
		3772	return;
		3773	}
		3774
		3775	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
		3776	I915_WRITE(VLV_PCBR, pctx_paddr);
		3777
		3778	out:
		3779	dev_priv->vlv_pctx = pctx;
		3780	}
		3781
		3782	static void valleyview_enable_rps(struct drm_device *dev)
		3783	{
		3784	struct drm_i915_private *dev_priv = dev->dev_private;
		3785	struct intel_ring_buffer *ring;
		3786	u32 gtfifodbg, val;
		3787	int i;
		3788
		3789	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		3790
		3791	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		3792	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		3793	I915_WRITE(GTFIFODBG, gtfifodbg);
		3794	}
		3795
		3796	valleyview_setup_pctx(dev);
		3797
		3798	gen6_gt_force_wake_get(dev_priv);
		3799
		3800	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		3801	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		3802	I915_WRITE(GEN6_RP_UP_EI, 66000);
		3803	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		3804
		3805	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		3806
		3807	I915_WRITE(GEN6_RP_CONTROL,
		3808	GEN6_RP_MEDIA_TURBO \|
		3809	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3810	GEN6_RP_MEDIA_IS_GFX \|
		3811	GEN6_RP_ENABLE \|
		3812	GEN6_RP_UP_BUSY_AVG \|
		3813	GEN6_RP_DOWN_IDLE_CONT);
		3814
		3815	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
		3816	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		3817	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		3818
		3819	for_each_ring(ring, dev_priv, i)
		3820	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		3821
		3822	I915_WRITE(GEN6_RC6_THRESHOLD, 0xc350);
		3823
		3824	/* allows RC6 residency counter to work */
		3825	I915_WRITE(0x138104, _MASKED_BIT_ENABLE(0x3));
		3826	I915_WRITE(GEN6_RC_CONTROL,
		3827	GEN7_RC_CTL_TO_MODE);
		3828
		3829	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		3830	switch ((val >> 6) & 3) {
		3831	case 0:
		3832	case 1:
		3833	dev_priv->mem_freq = 800;
		3834	break;
		3835	case 2:
		3836	dev_priv->mem_freq = 1066;
		3837	break;
		3838	case 3:
		3839	dev_priv->mem_freq = 1333;
		3840	break;
		3841	}
		3842	DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
		3843
		3844	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
		3845	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		3846
		3847	dev_priv->rps.cur_delay = (val >> 8) & 0xff;
		3848	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
		3849	vlv_gpu_freq(dev_priv->mem_freq,
		3850	dev_priv->rps.cur_delay),
		3851	dev_priv->rps.cur_delay);
		3852
		3853	dev_priv->rps.max_delay = valleyview_rps_max_freq(dev_priv);
		3854	dev_priv->rps.hw_max = dev_priv->rps.max_delay;
		3855	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
		3856	vlv_gpu_freq(dev_priv->mem_freq,
		3857	dev_priv->rps.max_delay),
		3858	dev_priv->rps.max_delay);
		3859
		3860	dev_priv->rps.rpe_delay = valleyview_rps_rpe_freq(dev_priv);
		3861	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
		3862	vlv_gpu_freq(dev_priv->mem_freq,
		3863	dev_priv->rps.rpe_delay),
		3864	dev_priv->rps.rpe_delay);
		3865
		3866	dev_priv->rps.min_delay = valleyview_rps_min_freq(dev_priv);
		3867	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
		3868	vlv_gpu_freq(dev_priv->mem_freq,
		3869	dev_priv->rps.min_delay),
		3870	dev_priv->rps.min_delay);
		3871
		3872	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
		3873	vlv_gpu_freq(dev_priv->mem_freq,
		3874	dev_priv->rps.rpe_delay),
		3875	dev_priv->rps.rpe_delay);
		3876
		3877	valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay);
		3878
		3879	gen6_enable_rps_interrupts(dev);
		3880
		3881	gen6_gt_force_wake_put(dev_priv);
		3882	}
		3883
3031	serge	3884	void ironlake_teardown_rc6(struct drm_device *dev)
		3885	{
		3886	struct drm_i915_private *dev_priv = dev->dev_private;
		3887
3243	Serge	3888	if (dev_priv->ips.renderctx) {
		3889	i915_gem_object_unpin(dev_priv->ips.renderctx);
		3890	drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
		3891	dev_priv->ips.renderctx = NULL;
3031	serge	3892	}
		3893
3243	Serge	3894	if (dev_priv->ips.pwrctx) {
		3895	i915_gem_object_unpin(dev_priv->ips.pwrctx);
		3896	drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
		3897	dev_priv->ips.pwrctx = NULL;
3031	serge	3898	}
		3899	}
		3900
		3901	static void ironlake_disable_rc6(struct drm_device *dev)
		3902	{
		3903	struct drm_i915_private *dev_priv = dev->dev_private;
		3904
		3905	if (I915_READ(PWRCTXA)) {
		3906	/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
		3907	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) \| RCX_SW_EXIT);
		3908	wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
		3909	50);
		3910
		3911	I915_WRITE(PWRCTXA, 0);
		3912	POSTING_READ(PWRCTXA);
		3913
		3914	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		3915	POSTING_READ(RSTDBYCTL);
		3916	}
		3917	}
		3918
		3919	static int ironlake_setup_rc6(struct drm_device *dev)
		3920	{
		3921	struct drm_i915_private *dev_priv = dev->dev_private;
		3922
3243	Serge	3923	if (dev_priv->ips.renderctx == NULL)
		3924	dev_priv->ips.renderctx = intel_alloc_context_page(dev);
		3925	if (!dev_priv->ips.renderctx)
3031	serge	3926	return -ENOMEM;
		3927
3243	Serge	3928	if (dev_priv->ips.pwrctx == NULL)
		3929	dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
		3930	if (!dev_priv->ips.pwrctx) {
3031	serge	3931	ironlake_teardown_rc6(dev);
		3932	return -ENOMEM;
		3933	}
		3934
		3935	return 0;
		3936	}
		3937
		3938	static void ironlake_enable_rc6(struct drm_device *dev)
		3939	{
		3940	struct drm_i915_private *dev_priv = dev->dev_private;
		3941	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
3243	Serge	3942	bool was_interruptible;
3031	serge	3943	int ret;
		3944
		3945	/* rc6 disabled by default due to repeated reports of hanging during
		3946	* boot and resume.
		3947	*/
		3948	if (!intel_enable_rc6(dev))
		3949	return;
		3950
		3951	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		3952
		3953	ret = ironlake_setup_rc6(dev);
		3954	if (ret)
		3955	return;
		3956
3243	Serge	3957	was_interruptible = dev_priv->mm.interruptible;
		3958	dev_priv->mm.interruptible = false;
		3959
3031	serge	3960	/*
		3961	* GPU can automatically power down the render unit if given a page
		3962	* to save state.
		3963	*/
		3964	ret = intel_ring_begin(ring, 6);
		3965	if (ret) {
		3966	ironlake_teardown_rc6(dev);
3243	Serge	3967	dev_priv->mm.interruptible = was_interruptible;
3031	serge	3968	return;
		3969	}
		3970
		3971	intel_ring_emit(ring, MI_SUSPEND_FLUSH \| MI_SUSPEND_FLUSH_EN);
		3972	intel_ring_emit(ring, MI_SET_CONTEXT);
4104	Serge	3973	intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) \|
3031	serge	3974	MI_MM_SPACE_GTT \|
		3975	MI_SAVE_EXT_STATE_EN \|
		3976	MI_RESTORE_EXT_STATE_EN \|
		3977	MI_RESTORE_INHIBIT);
		3978	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
		3979	intel_ring_emit(ring, MI_NOOP);
		3980	intel_ring_emit(ring, MI_FLUSH);
		3981	intel_ring_advance(ring);
		3982
		3983	/*
		3984	* Wait for the command parser to advance past MI_SET_CONTEXT. The HW
		3985	* does an implicit flush, combined with MI_FLUSH above, it should be
		3986	* safe to assume that renderctx is valid
		3987	*/
3243	Serge	3988	ret = intel_ring_idle(ring);
		3989	dev_priv->mm.interruptible = was_interruptible;
3031	serge	3990	if (ret) {
3746	Serge	3991	DRM_ERROR("failed to enable ironlake power savings\n");
3031	serge	3992	ironlake_teardown_rc6(dev);
		3993	return;
		3994	}
		3995
4104	Serge	3996	I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) \| PWRCTX_EN);
3031	serge	3997	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		3998	}
		3999
		4000	static unsigned long intel_pxfreq(u32 vidfreq)
		4001	{
		4002	unsigned long freq;
		4003	int div = (vidfreq & 0x3f0000) >> 16;
		4004	int post = (vidfreq & 0x3000) >> 12;
		4005	int pre = (vidfreq & 0x7);
		4006
		4007	if (!pre)
		4008	return 0;
		4009
		4010	freq = ((div * 133333) / ((1<
		4011
		4012	return freq;
		4013	}
		4014
		4015	static const struct cparams {
		4016	u16 i;
		4017	u16 t;
		4018	u16 m;
		4019	u16 c;
		4020	} cparams[] = {
		4021	{ 1, 1333, 301, 28664 },
		4022	{ 1, 1066, 294, 24460 },
		4023	{ 1, 800, 294, 25192 },
		4024	{ 0, 1333, 276, 27605 },
		4025	{ 0, 1066, 276, 27605 },
		4026	{ 0, 800, 231, 23784 },
		4027	};
		4028
		4029	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		4030	{
		4031	u64 total_count, diff, ret;
		4032	u32 count1, count2, count3, m = 0, c = 0;
		4033	unsigned long now = jiffies_to_msecs(GetTimerTicks()), diff1;
		4034	int i;
		4035
		4036	assert_spin_locked(&mchdev_lock);
		4037
		4038	diff1 = now - dev_priv->ips.last_time1;
		4039
		4040	/* Prevent division-by-zero if we are asking too fast.
		4041	* Also, we don't get interesting results if we are polling
		4042	* faster than once in 10ms, so just return the saved value
		4043	* in such cases.
		4044	*/
		4045	if (diff1 <= 10)
		4046	return dev_priv->ips.chipset_power;
		4047
		4048	count1 = I915_READ(DMIEC);
		4049	count2 = I915_READ(DDREC);
		4050	count3 = I915_READ(CSIEC);
		4051
		4052	total_count = count1 + count2 + count3;
		4053
		4054	/* FIXME: handle per-counter overflow */
		4055	if (total_count < dev_priv->ips.last_count1) {
		4056	diff = ~0UL - dev_priv->ips.last_count1;
		4057	diff += total_count;
		4058	} else {
		4059	diff = total_count - dev_priv->ips.last_count1;
		4060	}
		4061
		4062	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		4063	if (cparams[i].i == dev_priv->ips.c_m &&
		4064	cparams[i].t == dev_priv->ips.r_t) {
		4065	m = cparams[i].m;
		4066	c = cparams[i].c;
		4067	break;
		4068	}
		4069	}
		4070
		4071	diff = div_u64(diff, diff1);
		4072	ret = ((m * diff) + c);
		4073	ret = div_u64(ret, 10);
		4074
		4075	dev_priv->ips.last_count1 = total_count;
		4076	dev_priv->ips.last_time1 = now;
		4077
		4078	dev_priv->ips.chipset_power = ret;
		4079
		4080	return ret;
		4081	}
		4082
		4083	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		4084	{
		4085	unsigned long val;
		4086
		4087	if (dev_priv->info->gen != 5)
		4088	return 0;
		4089
		4090	spin_lock_irq(&mchdev_lock);
		4091
		4092	val = __i915_chipset_val(dev_priv);
		4093
		4094	spin_unlock_irq(&mchdev_lock);
		4095
		4096	return val;
		4097	}
		4098
		4099	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		4100	{
		4101	unsigned long m, x, b;
		4102	u32 tsfs;
		4103
		4104	tsfs = I915_READ(TSFS);
		4105
		4106	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		4107	x = I915_READ8(TR1);
		4108
		4109	b = tsfs & TSFS_INTR_MASK;
		4110
		4111	return ((m * x) / 127) - b;
		4112	}
		4113
		4114	static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		4115	{
		4116	static const struct v_table {
		4117	u16 vd; /* in .1 mil */
		4118	u16 vm; /* in .1 mil */
		4119	} v_table[] = {
		4120	{ 0, 0, },
		4121	{ 375, 0, },
		4122	{ 500, 0, },
		4123	{ 625, 0, },
		4124	{ 750, 0, },
		4125	{ 875, 0, },
		4126	{ 1000, 0, },
		4127	{ 1125, 0, },
		4128	{ 4125, 3000, },
		4129	{ 4125, 3000, },
		4130	{ 4125, 3000, },
		4131	{ 4125, 3000, },
		4132	{ 4125, 3000, },
		4133	{ 4125, 3000, },
		4134	{ 4125, 3000, },
		4135	{ 4125, 3000, },
		4136	{ 4125, 3000, },
		4137	{ 4125, 3000, },
		4138	{ 4125, 3000, },
		4139	{ 4125, 3000, },
		4140	{ 4125, 3000, },
		4141	{ 4125, 3000, },
		4142	{ 4125, 3000, },
		4143	{ 4125, 3000, },
		4144	{ 4125, 3000, },
		4145	{ 4125, 3000, },
		4146	{ 4125, 3000, },
		4147	{ 4125, 3000, },
		4148	{ 4125, 3000, },
		4149	{ 4125, 3000, },
		4150	{ 4125, 3000, },
		4151	{ 4125, 3000, },
		4152	{ 4250, 3125, },
		4153	{ 4375, 3250, },
		4154	{ 4500, 3375, },
		4155	{ 4625, 3500, },
		4156	{ 4750, 3625, },
		4157	{ 4875, 3750, },
		4158	{ 5000, 3875, },
		4159	{ 5125, 4000, },
		4160	{ 5250, 4125, },
		4161	{ 5375, 4250, },
		4162	{ 5500, 4375, },
		4163	{ 5625, 4500, },
		4164	{ 5750, 4625, },
		4165	{ 5875, 4750, },
		4166	{ 6000, 4875, },
		4167	{ 6125, 5000, },
		4168	{ 6250, 5125, },
		4169	{ 6375, 5250, },
		4170	{ 6500, 5375, },
		4171	{ 6625, 5500, },
		4172	{ 6750, 5625, },
		4173	{ 6875, 5750, },
		4174	{ 7000, 5875, },
		4175	{ 7125, 6000, },
		4176	{ 7250, 6125, },
		4177	{ 7375, 6250, },
		4178	{ 7500, 6375, },
		4179	{ 7625, 6500, },
		4180	{ 7750, 6625, },
		4181	{ 7875, 6750, },
		4182	{ 8000, 6875, },
		4183	{ 8125, 7000, },
		4184	{ 8250, 7125, },
		4185	{ 8375, 7250, },
		4186	{ 8500, 7375, },
		4187	{ 8625, 7500, },
		4188	{ 8750, 7625, },
		4189	{ 8875, 7750, },
		4190	{ 9000, 7875, },
		4191	{ 9125, 8000, },
		4192	{ 9250, 8125, },
		4193	{ 9375, 8250, },
		4194	{ 9500, 8375, },
		4195	{ 9625, 8500, },
		4196	{ 9750, 8625, },
		4197	{ 9875, 8750, },
		4198	{ 10000, 8875, },
		4199	{ 10125, 9000, },
		4200	{ 10250, 9125, },
		4201	{ 10375, 9250, },
		4202	{ 10500, 9375, },
		4203	{ 10625, 9500, },
		4204	{ 10750, 9625, },
		4205	{ 10875, 9750, },
		4206	{ 11000, 9875, },
		4207	{ 11125, 10000, },
		4208	{ 11250, 10125, },
		4209	{ 11375, 10250, },
		4210	{ 11500, 10375, },
		4211	{ 11625, 10500, },
		4212	{ 11750, 10625, },
		4213	{ 11875, 10750, },
		4214	{ 12000, 10875, },
		4215	{ 12125, 11000, },
		4216	{ 12250, 11125, },
		4217	{ 12375, 11250, },
		4218	{ 12500, 11375, },
		4219	{ 12625, 11500, },
		4220	{ 12750, 11625, },
		4221	{ 12875, 11750, },
		4222	{ 13000, 11875, },
		4223	{ 13125, 12000, },
		4224	{ 13250, 12125, },
		4225	{ 13375, 12250, },
		4226	{ 13500, 12375, },
		4227	{ 13625, 12500, },
		4228	{ 13750, 12625, },
		4229	{ 13875, 12750, },
		4230	{ 14000, 12875, },
		4231	{ 14125, 13000, },
		4232	{ 14250, 13125, },
		4233	{ 14375, 13250, },
		4234	{ 14500, 13375, },
		4235	{ 14625, 13500, },
		4236	{ 14750, 13625, },
		4237	{ 14875, 13750, },
		4238	{ 15000, 13875, },
		4239	{ 15125, 14000, },
		4240	{ 15250, 14125, },
		4241	{ 15375, 14250, },
		4242	{ 15500, 14375, },
		4243	{ 15625, 14500, },
		4244	{ 15750, 14625, },
		4245	{ 15875, 14750, },
		4246	{ 16000, 14875, },
		4247	{ 16125, 15000, },
		4248	};
		4249	if (dev_priv->info->is_mobile)
		4250	return v_table[pxvid].vm;
		4251	else
		4252	return v_table[pxvid].vd;
		4253	}
		4254
		4255	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		4256	{
		4257	struct timespec now, diff1;
		4258	u64 diff;
		4259	unsigned long diffms;
		4260	u32 count;
		4261
		4262	assert_spin_locked(&mchdev_lock);
		4263
		4264	getrawmonotonic(&now);
		4265	diff1 = timespec_sub(now, dev_priv->ips.last_time2);
		4266
		4267	/* Don't divide by 0 */
		4268	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
		4269	if (!diffms)
		4270	return;
		4271
		4272	count = I915_READ(GFXEC);
		4273
		4274	if (count < dev_priv->ips.last_count2) {
		4275	diff = ~0UL - dev_priv->ips.last_count2;
		4276	diff += count;
		4277	} else {
		4278	diff = count - dev_priv->ips.last_count2;
		4279	}
		4280
		4281	dev_priv->ips.last_count2 = count;
		4282	dev_priv->ips.last_time2 = now;
		4283
		4284	/* More magic constants... */
		4285	diff = diff * 1181;
		4286	diff = div_u64(diff, diffms * 10);
		4287	dev_priv->ips.gfx_power = diff;
		4288	}
		4289
		4290	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		4291	{
		4292	if (dev_priv->info->gen != 5)
		4293	return;
		4294
		4295	spin_lock_irq(&mchdev_lock);
		4296
		4297	__i915_update_gfx_val(dev_priv);
		4298
		4299	spin_unlock_irq(&mchdev_lock);
		4300	}
		4301
		4302	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		4303	{
		4304	unsigned long t, corr, state1, corr2, state2;
		4305	u32 pxvid, ext_v;
		4306
		4307	assert_spin_locked(&mchdev_lock);
		4308
		4309	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
		4310	pxvid = (pxvid >> 24) & 0x7f;
		4311	ext_v = pvid_to_extvid(dev_priv, pxvid);
		4312
		4313	state1 = ext_v;
		4314
		4315	t = i915_mch_val(dev_priv);
		4316
		4317	/* Revel in the empirically derived constants */
		4318
		4319	/* Correction factor in 1/100000 units */
		4320	if (t > 80)
		4321	corr = ((t * 2349) + 135940);
		4322	else if (t >= 50)
		4323	corr = ((t * 964) + 29317);
		4324	else /* < 50 */
		4325	corr = ((t * 301) + 1004);
		4326
		4327	corr = corr * ((150142 * state1) / 10000 - 78642);
		4328	corr /= 100000;
		4329	corr2 = (corr * dev_priv->ips.corr);
		4330
		4331	state2 = (corr2 * state1) / 10000;
		4332	state2 /= 100; /* convert to mW */
		4333
		4334	__i915_update_gfx_val(dev_priv);
		4335
		4336	return dev_priv->ips.gfx_power + state2;
		4337	}
		4338
		4339	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		4340	{
		4341	unsigned long val;
		4342
		4343	if (dev_priv->info->gen != 5)
		4344	return 0;
		4345
		4346	spin_lock_irq(&mchdev_lock);
		4347
		4348	val = __i915_gfx_val(dev_priv);
		4349
		4350	spin_unlock_irq(&mchdev_lock);
		4351
		4352	return val;
		4353	}
		4354
		4355	/**
		4356	* i915_read_mch_val - return value for IPS use
		4357	*
		4358	* Calculate and return a value for the IPS driver to use when deciding whether
		4359	* we have thermal and power headroom to increase CPU or GPU power budget.
		4360	*/
		4361	unsigned long i915_read_mch_val(void)
		4362	{
		4363	struct drm_i915_private *dev_priv;
		4364	unsigned long chipset_val, graphics_val, ret = 0;
		4365
		4366	spin_lock_irq(&mchdev_lock);
		4367	if (!i915_mch_dev)
		4368	goto out_unlock;
		4369	dev_priv = i915_mch_dev;
		4370
		4371	chipset_val = __i915_chipset_val(dev_priv);
		4372	graphics_val = __i915_gfx_val(dev_priv);
		4373
		4374	ret = chipset_val + graphics_val;
		4375
		4376	out_unlock:
		4377	spin_unlock_irq(&mchdev_lock);
		4378
		4379	return ret;
		4380	}
		4381	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		4382
		4383	/**
		4384	* i915_gpu_raise - raise GPU frequency limit
		4385	*
		4386	* Raise the limit; IPS indicates we have thermal headroom.
		4387	*/
		4388	bool i915_gpu_raise(void)
		4389	{
		4390	struct drm_i915_private *dev_priv;
		4391	bool ret = true;
		4392
		4393	spin_lock_irq(&mchdev_lock);
		4394	if (!i915_mch_dev) {
		4395	ret = false;
		4396	goto out_unlock;
		4397	}
		4398	dev_priv = i915_mch_dev;
		4399
		4400	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		4401	dev_priv->ips.max_delay--;
		4402
		4403	out_unlock:
		4404	spin_unlock_irq(&mchdev_lock);
		4405
		4406	return ret;
		4407	}
		4408	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		4409
		4410	/**
		4411	* i915_gpu_lower - lower GPU frequency limit
		4412	*
		4413	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		4414	* frequency maximum.
		4415	*/
		4416	bool i915_gpu_lower(void)
		4417	{
		4418	struct drm_i915_private *dev_priv;
		4419	bool ret = true;
		4420
		4421	spin_lock_irq(&mchdev_lock);
		4422	if (!i915_mch_dev) {
		4423	ret = false;
		4424	goto out_unlock;
		4425	}
		4426	dev_priv = i915_mch_dev;
		4427
		4428	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		4429	dev_priv->ips.max_delay++;
		4430
		4431	out_unlock:
		4432	spin_unlock_irq(&mchdev_lock);
		4433
		4434	return ret;
		4435	}
		4436	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		4437
		4438	/**
		4439	* i915_gpu_busy - indicate GPU business to IPS
		4440	*
		4441	* Tell the IPS driver whether or not the GPU is busy.
		4442	*/
		4443	bool i915_gpu_busy(void)
		4444	{
		4445	struct drm_i915_private *dev_priv;
		4446	struct intel_ring_buffer *ring;
		4447	bool ret = false;
		4448	int i;
		4449
		4450	spin_lock_irq(&mchdev_lock);
		4451	if (!i915_mch_dev)
		4452	goto out_unlock;
		4453	dev_priv = i915_mch_dev;
		4454
		4455	for_each_ring(ring, dev_priv, i)
		4456	ret \|= !list_empty(&ring->request_list);
		4457
		4458	out_unlock:
		4459	spin_unlock_irq(&mchdev_lock);
		4460
		4461	return ret;
		4462	}
		4463	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		4464
		4465	/**
		4466	* i915_gpu_turbo_disable - disable graphics turbo
		4467	*
		4468	* Disable graphics turbo by resetting the max frequency and setting the
		4469	* current frequency to the default.
		4470	*/
		4471	bool i915_gpu_turbo_disable(void)
		4472	{
		4473	struct drm_i915_private *dev_priv;
		4474	bool ret = true;
		4475
		4476	spin_lock_irq(&mchdev_lock);
		4477	if (!i915_mch_dev) {
		4478	ret = false;
		4479	goto out_unlock;
		4480	}
		4481	dev_priv = i915_mch_dev;
		4482
		4483	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		4484
		4485	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		4486	ret = false;
		4487
		4488	out_unlock:
		4489	spin_unlock_irq(&mchdev_lock);
		4490
		4491	return ret;
		4492	}
		4493	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		4494
		4495	/**
		4496	* Tells the intel_ips driver that the i915 driver is now loaded, if
		4497	* IPS got loaded first.
		4498	*
		4499	* This awkward dance is so that neither module has to depend on the
		4500	* other in order for IPS to do the appropriate communication of
		4501	* GPU turbo limits to i915.
		4502	*/
		4503	static void
		4504	ips_ping_for_i915_load(void)
		4505	{
		4506	void (*link)(void);
		4507
		4508	// link = symbol_get(ips_link_to_i915_driver);
		4509	// if (link) {
		4510	// link();
		4511	// symbol_put(ips_link_to_i915_driver);
		4512	// }
		4513	}
		4514
		4515	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		4516	{
		4517	/* We only register the i915 ips part with intel-ips once everything is
		4518	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		4519	spin_lock_irq(&mchdev_lock);
		4520	i915_mch_dev = dev_priv;
		4521	spin_unlock_irq(&mchdev_lock);
		4522
		4523	ips_ping_for_i915_load();
		4524	}
		4525
		4526	void intel_gpu_ips_teardown(void)
		4527	{
		4528	spin_lock_irq(&mchdev_lock);
		4529	i915_mch_dev = NULL;
		4530	spin_unlock_irq(&mchdev_lock);
		4531	}
		4532	static void intel_init_emon(struct drm_device *dev)
		4533	{
		4534	struct drm_i915_private *dev_priv = dev->dev_private;
		4535	u32 lcfuse;
		4536	u8 pxw[16];
		4537	int i;
		4538
		4539	/* Disable to program */
		4540	I915_WRITE(ECR, 0);
		4541	POSTING_READ(ECR);
		4542
		4543	/* Program energy weights for various events */
		4544	I915_WRITE(SDEW, 0x15040d00);
		4545	I915_WRITE(CSIEW0, 0x007f0000);
		4546	I915_WRITE(CSIEW1, 0x1e220004);
		4547	I915_WRITE(CSIEW2, 0x04000004);
		4548
		4549	for (i = 0; i < 5; i++)
		4550	I915_WRITE(PEW + (i * 4), 0);
		4551	for (i = 0; i < 3; i++)
		4552	I915_WRITE(DEW + (i * 4), 0);
		4553
		4554	/* Program P-state weights to account for frequency power adjustment */
		4555	for (i = 0; i < 16; i++) {
		4556	u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
		4557	unsigned long freq = intel_pxfreq(pxvidfreq);
		4558	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		4559	PXVFREQ_PX_SHIFT;
		4560	unsigned long val;
		4561
		4562	val = vid * vid;
		4563	val *= (freq / 1000);
		4564	val *= 255;
		4565	val /= (127127900);
		4566	if (val > 0xff)
		4567	DRM_ERROR("bad pxval: %ld\n", val);
		4568	pxw[i] = val;
		4569	}
		4570	/* Render standby states get 0 weight */
		4571	pxw[14] = 0;
		4572	pxw[15] = 0;
		4573
		4574	for (i = 0; i < 4; i++) {
		4575	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		4576	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
		4577	I915_WRITE(PXW + (i * 4), val);
		4578	}
		4579
		4580	/* Adjust magic regs to magic values (more experimental results) */
		4581	I915_WRITE(OGW0, 0);
		4582	I915_WRITE(OGW1, 0);
		4583	I915_WRITE(EG0, 0x00007f00);
		4584	I915_WRITE(EG1, 0x0000000e);
		4585	I915_WRITE(EG2, 0x000e0000);
		4586	I915_WRITE(EG3, 0x68000300);
		4587	I915_WRITE(EG4, 0x42000000);
		4588	I915_WRITE(EG5, 0x00140031);
		4589	I915_WRITE(EG6, 0);
		4590	I915_WRITE(EG7, 0);
		4591
		4592	for (i = 0; i < 8; i++)
		4593	I915_WRITE(PXWL + (i * 4), 0);
		4594
		4595	/* Enable PMON + select events */
		4596	I915_WRITE(ECR, 0x80000019);
		4597
		4598	lcfuse = I915_READ(LCFUSE02);
		4599
		4600	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		4601	}
		4602
		4603	void intel_disable_gt_powersave(struct drm_device *dev)
		4604	{
3243	Serge	4605	struct drm_i915_private *dev_priv = dev->dev_private;
		4606
4104	Serge	4607	/* Interrupts should be disabled already to avoid re-arming. */
		4608	WARN_ON(dev->irq_enabled);
		4609
3031	serge	4610	if (IS_IRONLAKE_M(dev)) {
		4611	ironlake_disable_drps(dev);
		4612	ironlake_disable_rc6(dev);
		4613	} else if (INTEL_INFO(dev)->gen >= 6 && !IS_VALLEYVIEW(dev)) {
3482	Serge	4614	// cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
		4615	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	4616	if (IS_VALLEYVIEW(dev))
		4617	valleyview_disable_rps(dev);
		4618	else
3031	serge	4619	gen6_disable_rps(dev);
3480	Serge	4620	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	4621	}
		4622	}
		4623
3482	Serge	4624	static void intel_gen6_powersave_work(struct work_struct *work)
		4625	{
		4626	struct drm_i915_private *dev_priv =
		4627	container_of(work, struct drm_i915_private,
		4628	rps.delayed_resume_work.work);
		4629	struct drm_device *dev = dev_priv->dev;
		4630
		4631	ENTER();
		4632
		4633	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	4634
		4635	if (IS_VALLEYVIEW(dev)) {
		4636	valleyview_enable_rps(dev);
		4637	} else {
3482	Serge	4638	gen6_enable_rps(dev);
		4639	gen6_update_ring_freq(dev);
4104	Serge	4640	}
3482	Serge	4641	mutex_unlock(&dev_priv->rps.hw_lock);
		4642
		4643	LEAVE();
		4644	}
		4645
3031	serge	4646	void intel_enable_gt_powersave(struct drm_device *dev)
		4647	{
3243	Serge	4648	struct drm_i915_private *dev_priv = dev->dev_private;
		4649
3031	serge	4650	if (IS_IRONLAKE_M(dev)) {
		4651	ironlake_enable_drps(dev);
		4652	ironlake_enable_rc6(dev);
		4653	intel_init_emon(dev);
4104	Serge	4654	} else if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {
3243	Serge	4655	/*
		4656	* PCU communication is slow and this doesn't need to be
		4657	* done at any specific time, so do this out of our fast path
		4658	* to make resume and init faster.
		4659	*/
3482	Serge	4660	schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		4661	round_jiffies_up_relative(HZ));
3031	serge	4662	}
		4663	}
		4664
3243	Serge	4665	static void ibx_init_clock_gating(struct drm_device *dev)
		4666	{
		4667	struct drm_i915_private *dev_priv = dev->dev_private;
		4668
		4669	/*
		4670	* On Ibex Peak and Cougar Point, we need to disable clock
		4671	* gating for the panel power sequencer or it will fail to
		4672	* start up when no ports are active.
		4673	*/
		4674	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		4675	}
		4676
4104	Serge	4677	static void g4x_disable_trickle_feed(struct drm_device *dev)
		4678	{
		4679	struct drm_i915_private *dev_priv = dev->dev_private;
		4680	int pipe;
		4681
		4682	for_each_pipe(pipe) {
		4683	I915_WRITE(DSPCNTR(pipe),
		4684	I915_READ(DSPCNTR(pipe)) \|
		4685	DISPPLANE_TRICKLE_FEED_DISABLE);
		4686	intel_flush_display_plane(dev_priv, pipe);
		4687	}
		4688	}
		4689
3031	serge	4690	static void ironlake_init_clock_gating(struct drm_device *dev)
		4691	{
		4692	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	4693	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	4694
4104	Serge	4695	/*
		4696	* Required for FBC
		4697	* WaFbcDisableDpfcClockGating:ilk
		4698	*/
3243	Serge	4699	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		4700	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		4701	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	4702
		4703	I915_WRITE(PCH_3DCGDIS0,
		4704	MARIUNIT_CLOCK_GATE_DISABLE \|
		4705	SVSMUNIT_CLOCK_GATE_DISABLE);
		4706	I915_WRITE(PCH_3DCGDIS1,
		4707	VFMUNIT_CLOCK_GATE_DISABLE);
		4708
		4709	/*
		4710	* According to the spec the following bits should be set in
		4711	* order to enable memory self-refresh
		4712	* The bit 22/21 of 0x42004
		4713	* The bit 5 of 0x42020
		4714	* The bit 15 of 0x45000
		4715	*/
		4716	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4717	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4718	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	4719	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	4720	I915_WRITE(DISP_ARB_CTL,
		4721	(I915_READ(DISP_ARB_CTL) \|
		4722	DISP_FBC_WM_DIS));
		4723	I915_WRITE(WM3_LP_ILK, 0);
		4724	I915_WRITE(WM2_LP_ILK, 0);
		4725	I915_WRITE(WM1_LP_ILK, 0);
		4726
		4727	/*
		4728	* Based on the document from hardware guys the following bits
		4729	* should be set unconditionally in order to enable FBC.
		4730	* The bit 22 of 0x42000
		4731	* The bit 22 of 0x42004
		4732	* The bit 7,8,9 of 0x42020.
		4733	*/
		4734	if (IS_IRONLAKE_M(dev)) {
4104	Serge	4735	/* WaFbcAsynchFlipDisableFbcQueue:ilk */
3031	serge	4736	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		4737	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		4738	ILK_FBCQ_DIS);
		4739	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4740	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4741	ILK_DPARB_GATE);
		4742	}
		4743
3243	Serge	4744	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		4745
3031	serge	4746	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4747	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4748	ILK_ELPIN_409_SELECT);
		4749	I915_WRITE(_3D_CHICKEN2,
		4750	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		4751	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	4752
4104	Serge	4753	/* WaDisableRenderCachePipelinedFlush:ilk */
3243	Serge	4754	I915_WRITE(CACHE_MODE_0,
		4755	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		4756
4104	Serge	4757	g4x_disable_trickle_feed(dev);
		4758
3243	Serge	4759	ibx_init_clock_gating(dev);
3031	serge	4760	}
		4761
3243	Serge	4762	static void cpt_init_clock_gating(struct drm_device *dev)
		4763	{
		4764	struct drm_i915_private *dev_priv = dev->dev_private;
		4765	int pipe;
3746	Serge	4766	uint32_t val;
3243	Serge	4767
		4768	/*
		4769	* On Ibex Peak and Cougar Point, we need to disable clock
		4770	* gating for the panel power sequencer or it will fail to
		4771	* start up when no ports are active.
		4772	*/
		4773	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		4774	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		4775	DPLS_EDP_PPS_FIX_DIS);
		4776	/* The below fixes the weird display corruption, a few pixels shifted
		4777	* downward, on (only) LVDS of some HP laptops with IVY.
		4778	*/
3746	Serge	4779	for_each_pipe(pipe) {
		4780	val = I915_READ(TRANS_CHICKEN2(pipe));
		4781	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		4782	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4104	Serge	4783	if (dev_priv->vbt.fdi_rx_polarity_inverted)
3746	Serge	4784	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		4785	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		4786	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		4787	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		4788	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		4789	}
3243	Serge	4790	/* WADP0ClockGatingDisable */
		4791	for_each_pipe(pipe) {
		4792	I915_WRITE(TRANS_CHICKEN1(pipe),
		4793	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		4794	}
		4795	}
		4796
3480	Serge	4797	static void gen6_check_mch_setup(struct drm_device *dev)
		4798	{
		4799	struct drm_i915_private *dev_priv = dev->dev_private;
		4800	uint32_t tmp;
		4801
		4802	tmp = I915_READ(MCH_SSKPD);
		4803	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) {
		4804	DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp);
		4805	DRM_INFO("This can cause pipe underruns and display issues.\n");
		4806	DRM_INFO("Please upgrade your BIOS to fix this.\n");
		4807	}
		4808	}
		4809
3031	serge	4810	static void gen6_init_clock_gating(struct drm_device *dev)
		4811	{
		4812	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	4813	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	4814
3243	Serge	4815	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	4816
		4817	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4818	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4819	ILK_ELPIN_409_SELECT);
		4820
4104	Serge	4821	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3243	Serge	4822	I915_WRITE(_3D_CHICKEN,
		4823	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		4824
4104	Serge	4825	/* WaSetupGtModeTdRowDispatch:snb */
3243	Serge	4826	if (IS_SNB_GT1(dev))
		4827	I915_WRITE(GEN6_GT_MODE,
		4828	_MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
		4829
3031	serge	4830	I915_WRITE(WM3_LP_ILK, 0);
		4831	I915_WRITE(WM2_LP_ILK, 0);
		4832	I915_WRITE(WM1_LP_ILK, 0);
		4833
		4834	I915_WRITE(CACHE_MODE_0,
		4835	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		4836
		4837	I915_WRITE(GEN6_UCGCTL1,
		4838	I915_READ(GEN6_UCGCTL1) \|
		4839	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		4840	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		4841
		4842	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		4843	* gating disable must be set. Failure to set it results in
		4844	* flickering pixels due to Z write ordering failures after
		4845	* some amount of runtime in the Mesa "fire" demo, and Unigine
		4846	* Sanctuary and Tropics, and apparently anything else with
		4847	* alpha test or pixel discard.
		4848	*
		4849	* According to the spec, bit 11 (RCCUNIT) must also be set,
		4850	* but we didn't debug actual testcases to find it out.
		4851	*
4104	Serge	4852	* Also apply WaDisableVDSUnitClockGating:snb and
		4853	* WaDisableRCPBUnitClockGating:snb.
3031	serge	4854	*/
		4855	I915_WRITE(GEN6_UCGCTL2,
		4856	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		4857	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		4858	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		4859
		4860	/* Bspec says we need to always set all mask bits. */
		4861	I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) \|
		4862	_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
		4863
		4864	/*
		4865	* According to the spec the following bits should be
		4866	* set in order to enable memory self-refresh and fbc:
		4867	* The bit21 and bit22 of 0x42000
		4868	* The bit21 and bit22 of 0x42004
		4869	* The bit5 and bit7 of 0x42020
		4870	* The bit14 of 0x70180
		4871	* The bit14 of 0x71180
4104	Serge	4872	*
		4873	* WaFbcAsynchFlipDisableFbcQueue:snb
3031	serge	4874	*/
		4875	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		4876	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		4877	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		4878	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4879	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4880	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	4881	I915_WRITE(ILK_DSPCLK_GATE_D,
		4882	I915_READ(ILK_DSPCLK_GATE_D) \|
		4883	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		4884	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	4885
4104	Serge	4886	g4x_disable_trickle_feed(dev);
3031	serge	4887
		4888	/* The default value should be 0x200 according to docs, but the two
		4889	* platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
		4890	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
		4891	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
3243	Serge	4892
		4893	cpt_init_clock_gating(dev);
3480	Serge	4894
		4895	gen6_check_mch_setup(dev);
3031	serge	4896	}
		4897
		4898	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		4899	{
		4900	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		4901
		4902	reg &= ~GEN7_FF_SCHED_MASK;
		4903	reg \|= GEN7_FF_TS_SCHED_HW;
		4904	reg \|= GEN7_FF_VS_SCHED_HW;
		4905	reg \|= GEN7_FF_DS_SCHED_HW;
		4906
3480	Serge	4907	if (IS_HASWELL(dev_priv->dev))
		4908	reg &= ~GEN7_FF_VS_REF_CNT_FFME;
		4909
3031	serge	4910	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		4911	}
		4912
3243	Serge	4913	static void lpt_init_clock_gating(struct drm_device *dev)
		4914	{
		4915	struct drm_i915_private *dev_priv = dev->dev_private;
		4916
		4917	/*
		4918	* TODO: this bit should only be enabled when really needed, then
		4919	* disabled when not needed anymore in order to save power.
		4920	*/
		4921	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
		4922	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		4923	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		4924	PCH_LP_PARTITION_LEVEL_DISABLE);
4104	Serge	4925
		4926	/* WADPOClockGatingDisable:hsw */
		4927	I915_WRITE(_TRANSA_CHICKEN1,
		4928	I915_READ(_TRANSA_CHICKEN1) \|
		4929	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3243	Serge	4930	}
		4931
4104	Serge	4932	static void lpt_suspend_hw(struct drm_device *dev)
		4933	{
		4934	struct drm_i915_private *dev_priv = dev->dev_private;
		4935
		4936	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
		4937	uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
		4938
		4939	val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		4940	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
		4941	}
		4942	}
		4943
3031	serge	4944	static void haswell_init_clock_gating(struct drm_device *dev)
		4945	{
		4946	struct drm_i915_private *dev_priv = dev->dev_private;
		4947
		4948	I915_WRITE(WM3_LP_ILK, 0);
		4949	I915_WRITE(WM2_LP_ILK, 0);
		4950	I915_WRITE(WM1_LP_ILK, 0);
		4951
		4952	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	4953	* This implements the WaDisableRCZUnitClockGating:hsw workaround.
3031	serge	4954	*/
		4955	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
		4956
4104	Serge	4957	/* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */
3031	serge	4958	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		4959	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		4960
4104	Serge	4961	/* WaApplyL3ControlAndL3ChickenMode:hsw */
3031	serge	4962	I915_WRITE(GEN7_L3CNTLREG1,
		4963	GEN7_WA_FOR_GEN7_L3_CONTROL);
		4964	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		4965	GEN7_WA_L3_CHICKEN_MODE);
		4966
4104	Serge	4967	/* L3 caching of data atomics doesn't work -- disable it. */
		4968	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
		4969	I915_WRITE(HSW_ROW_CHICKEN3,
		4970	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
		4971
		4972	/* This is required by WaCatErrorRejectionIssue:hsw */
3031	serge	4973	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		4974	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		4975	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		4976
4104	Serge	4977	/* WaVSRefCountFullforceMissDisable:hsw */
3031	serge	4978	gen7_setup_fixed_func_scheduler(dev_priv);
		4979
4104	Serge	4980	/* WaDisable4x2SubspanOptimization:hsw */
3031	serge	4981	I915_WRITE(CACHE_MODE_1,
		4982	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		4983
4104	Serge	4984	/* WaSwitchSolVfFArbitrationPriority:hsw */
3746	Serge	4985	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		4986
4104	Serge	4987	/* WaRsPkgCStateDisplayPMReq:hsw */
		4988	I915_WRITE(CHICKEN_PAR1_1,
		4989	I915_READ(CHICKEN_PAR1_1) \| FORCE_ARB_IDLE_PLANES);
3031	serge	4990
3243	Serge	4991	lpt_init_clock_gating(dev);
3031	serge	4992	}
		4993
		4994	static void ivybridge_init_clock_gating(struct drm_device *dev)
		4995	{
		4996	struct drm_i915_private *dev_priv = dev->dev_private;
		4997	uint32_t snpcr;
		4998
		4999	I915_WRITE(WM3_LP_ILK, 0);
		5000	I915_WRITE(WM2_LP_ILK, 0);
		5001	I915_WRITE(WM1_LP_ILK, 0);
		5002
3243	Serge	5003	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	5004
4104	Serge	5005	/* WaDisableEarlyCull:ivb */
3243	Serge	5006	I915_WRITE(_3D_CHICKEN3,
		5007	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		5008
4104	Serge	5009	/* WaDisableBackToBackFlipFix:ivb */
3031	serge	5010	I915_WRITE(IVB_CHICKEN3,
		5011	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		5012	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		5013
4104	Serge	5014	/* WaDisablePSDDualDispatchEnable:ivb */
3243	Serge	5015	if (IS_IVB_GT1(dev))
		5016	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		5017	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		5018	else
		5019	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
		5020	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		5021
4104	Serge	5022	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
3031	serge	5023	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		5024	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		5025
4104	Serge	5026	/* WaApplyL3ControlAndL3ChickenMode:ivb */
3031	serge	5027	I915_WRITE(GEN7_L3CNTLREG1,
		5028	GEN7_WA_FOR_GEN7_L3_CONTROL);
		5029	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		5030	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	5031	if (IS_IVB_GT1(dev))
		5032	I915_WRITE(GEN7_ROW_CHICKEN2,
		5033	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		5034	else
		5035	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		5036	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3031	serge	5037
3243	Serge	5038
4104	Serge	5039	/* WaForceL3Serialization:ivb */
3243	Serge	5040	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		5041	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		5042
3031	serge	5043	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		5044	* gating disable must be set. Failure to set it results in
		5045	* flickering pixels due to Z write ordering failures after
		5046	* some amount of runtime in the Mesa "fire" demo, and Unigine
		5047	* Sanctuary and Tropics, and apparently anything else with
		5048	* alpha test or pixel discard.
		5049	*
		5050	* According to the spec, bit 11 (RCCUNIT) must also be set,
		5051	* but we didn't debug actual testcases to find it out.
		5052	*
		5053	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	5054	* This implements the WaDisableRCZUnitClockGating:ivb workaround.
3031	serge	5055	*/
		5056	I915_WRITE(GEN6_UCGCTL2,
		5057	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		5058	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		5059
4104	Serge	5060	/* This is required by WaCatErrorRejectionIssue:ivb */
3031	serge	5061	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		5062	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		5063	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		5064
4104	Serge	5065	g4x_disable_trickle_feed(dev);
3031	serge	5066
4104	Serge	5067	/* WaVSRefCountFullforceMissDisable:ivb */
3031	serge	5068	gen7_setup_fixed_func_scheduler(dev_priv);
		5069
4104	Serge	5070	/* WaDisable4x2SubspanOptimization:ivb */
3031	serge	5071	I915_WRITE(CACHE_MODE_1,
		5072	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		5073
		5074	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		5075	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		5076	snpcr \|= GEN6_MBC_SNPCR_MED;
		5077	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	5078
3746	Serge	5079	if (!HAS_PCH_NOP(dev))
3243	Serge	5080	cpt_init_clock_gating(dev);
3480	Serge	5081
		5082	gen6_check_mch_setup(dev);
3031	serge	5083	}
		5084
		5085	static void valleyview_init_clock_gating(struct drm_device *dev)
		5086	{
		5087	struct drm_i915_private *dev_priv = dev->dev_private;
		5088
4104	Serge	5089	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	5090
4104	Serge	5091	/* WaDisableEarlyCull:vlv */
3243	Serge	5092	I915_WRITE(_3D_CHICKEN3,
		5093	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		5094
4104	Serge	5095	/* WaDisableBackToBackFlipFix:vlv */
3031	serge	5096	I915_WRITE(IVB_CHICKEN3,
		5097	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		5098	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		5099
4104	Serge	5100	/* WaDisablePSDDualDispatchEnable:vlv */
3243	Serge	5101	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	5102	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		5103	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	5104
4104	Serge	5105	/* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */
3031	serge	5106	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		5107	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		5108
4104	Serge	5109	/* WaApplyL3ControlAndL3ChickenMode:vlv */
3243	Serge	5110	I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) \| GEN7_L3AGDIS);
3031	serge	5111	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
		5112
4104	Serge	5113	/* WaForceL3Serialization:vlv */
3243	Serge	5114	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		5115	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		5116
4104	Serge	5117	/* WaDisableDopClockGating:vlv */
3243	Serge	5118	I915_WRITE(GEN7_ROW_CHICKEN2,
		5119	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		5120
4104	Serge	5121	/* This is required by WaCatErrorRejectionIssue:vlv */
3031	serge	5122	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		5123	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		5124	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		5125
		5126	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		5127	* gating disable must be set. Failure to set it results in
		5128	* flickering pixels due to Z write ordering failures after
		5129	* some amount of runtime in the Mesa "fire" demo, and Unigine
		5130	* Sanctuary and Tropics, and apparently anything else with
		5131	* alpha test or pixel discard.
		5132	*
		5133	* According to the spec, bit 11 (RCCUNIT) must also be set,
		5134	* but we didn't debug actual testcases to find it out.
		5135	*
		5136	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	5137	* This implements the WaDisableRCZUnitClockGating:vlv workaround.
3031	serge	5138	*
4104	Serge	5139	* Also apply WaDisableVDSUnitClockGating:vlv and
		5140	* WaDisableRCPBUnitClockGating:vlv.
3031	serge	5141	*/
		5142	I915_WRITE(GEN6_UCGCTL2,
		5143	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		5144	GEN7_TDLUNIT_CLOCK_GATE_DISABLE \|
		5145	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		5146	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		5147	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		5148
		5149	I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
		5150
4104	Serge	5151	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
3031	serge	5152
		5153	I915_WRITE(CACHE_MODE_1,
		5154	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		5155
		5156	/*
4104	Serge	5157	* WaDisableVLVClockGating_VBIIssue:vlv
3243	Serge	5158	* Disable clock gating on th GCFG unit to prevent a delay
		5159	* in the reporting of vblank events.
		5160	*/
3746	Serge	5161	I915_WRITE(VLV_GUNIT_CLOCK_GATE, 0xffffffff);
		5162
		5163	/* Conservative clock gating settings for now */
		5164	I915_WRITE(0x9400, 0xffffffff);
		5165	I915_WRITE(0x9404, 0xffffffff);
		5166	I915_WRITE(0x9408, 0xffffffff);
		5167	I915_WRITE(0x940c, 0xffffffff);
		5168	I915_WRITE(0x9410, 0xffffffff);
		5169	I915_WRITE(0x9414, 0xffffffff);
		5170	I915_WRITE(0x9418, 0xffffffff);
3031	serge	5171	}
		5172
		5173	static void g4x_init_clock_gating(struct drm_device *dev)
		5174	{
		5175	struct drm_i915_private *dev_priv = dev->dev_private;
		5176	uint32_t dspclk_gate;
		5177
		5178	I915_WRITE(RENCLK_GATE_D1, 0);
		5179	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		5180	GS_UNIT_CLOCK_GATE_DISABLE \|
		5181	CL_UNIT_CLOCK_GATE_DISABLE);
		5182	I915_WRITE(RAMCLK_GATE_D, 0);
		5183	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		5184	OVRUNIT_CLOCK_GATE_DISABLE \|
		5185	OVCUNIT_CLOCK_GATE_DISABLE;
		5186	if (IS_GM45(dev))
		5187	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		5188	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	5189
		5190	/* WaDisableRenderCachePipelinedFlush */
		5191	I915_WRITE(CACHE_MODE_0,
		5192	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4104	Serge	5193
		5194	g4x_disable_trickle_feed(dev);
3031	serge	5195	}
		5196
		5197	static void crestline_init_clock_gating(struct drm_device *dev)
		5198	{
		5199	struct drm_i915_private *dev_priv = dev->dev_private;
		5200
		5201	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		5202	I915_WRITE(RENCLK_GATE_D2, 0);
		5203	I915_WRITE(DSPCLK_GATE_D, 0);
		5204	I915_WRITE(RAMCLK_GATE_D, 0);
		5205	I915_WRITE16(DEUC, 0);
4104	Serge	5206	I915_WRITE(MI_ARB_STATE,
		5207	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	5208	}
		5209
		5210	static void broadwater_init_clock_gating(struct drm_device *dev)
		5211	{
		5212	struct drm_i915_private *dev_priv = dev->dev_private;
		5213
		5214	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		5215	I965_RCC_CLOCK_GATE_DISABLE \|
		5216	I965_RCPB_CLOCK_GATE_DISABLE \|
		5217	I965_ISC_CLOCK_GATE_DISABLE \|
		5218	I965_FBC_CLOCK_GATE_DISABLE);
		5219	I915_WRITE(RENCLK_GATE_D2, 0);
4104	Serge	5220	I915_WRITE(MI_ARB_STATE,
		5221	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	5222	}
		5223
		5224	static void gen3_init_clock_gating(struct drm_device *dev)
		5225	{
		5226	struct drm_i915_private *dev_priv = dev->dev_private;
		5227	u32 dstate = I915_READ(D_STATE);
		5228
		5229	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		5230	DSTATE_DOT_CLOCK_GATING;
		5231	I915_WRITE(D_STATE, dstate);
		5232
		5233	if (IS_PINEVIEW(dev))
		5234	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		5235
		5236	/* IIR "flip pending" means done if this bit is set */
		5237	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
		5238	}
		5239
		5240	static void i85x_init_clock_gating(struct drm_device *dev)
		5241	{
		5242	struct drm_i915_private *dev_priv = dev->dev_private;
		5243
		5244	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
		5245	}
		5246
		5247	static void i830_init_clock_gating(struct drm_device *dev)
		5248	{
		5249	struct drm_i915_private *dev_priv = dev->dev_private;
		5250
		5251	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
		5252	}
		5253
		5254	void intel_init_clock_gating(struct drm_device *dev)
		5255	{
		5256	struct drm_i915_private *dev_priv = dev->dev_private;
		5257
		5258	dev_priv->display.init_clock_gating(dev);
		5259	}
		5260
4104	Serge	5261	void intel_suspend_hw(struct drm_device *dev)
		5262	{
		5263	if (HAS_PCH_LPT(dev))
		5264	lpt_suspend_hw(dev);
		5265	}
		5266
3746	Serge	5267	/**
		5268	* We should only use the power well if we explicitly asked the hardware to
		5269	* enable it, so check if it's enabled and also check if we've requested it to
		5270	* be enabled.
		5271	*/
4104	Serge	5272	bool intel_display_power_enabled(struct drm_device *dev,
		5273	enum intel_display_power_domain domain)
3746	Serge	5274	{
		5275	struct drm_i915_private *dev_priv = dev->dev_private;
		5276
4104	Serge	5277	if (!HAS_POWER_WELL(dev))
		5278	return true;
		5279
		5280	switch (domain) {
		5281	case POWER_DOMAIN_PIPE_A:
		5282	case POWER_DOMAIN_TRANSCODER_EDP:
		5283	return true;
		5284	case POWER_DOMAIN_PIPE_B:
		5285	case POWER_DOMAIN_PIPE_C:
		5286	case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
		5287	case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
		5288	case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
		5289	case POWER_DOMAIN_TRANSCODER_A:
		5290	case POWER_DOMAIN_TRANSCODER_B:
		5291	case POWER_DOMAIN_TRANSCODER_C:
3746	Serge	5292	return I915_READ(HSW_PWR_WELL_DRIVER) ==
4104	Serge	5293	(HSW_PWR_WELL_ENABLE_REQUEST \| HSW_PWR_WELL_STATE_ENABLED);
		5294	default:
		5295	BUG();
		5296	}
3746	Serge	5297	}
		5298
4104	Serge	5299	static void __intel_set_power_well(struct drm_device *dev, bool enable)
3031	serge	5300	{
		5301	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	5302	bool is_enabled, enable_requested;
		5303	uint32_t tmp;
3031	serge	5304
3480	Serge	5305	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
4104	Serge	5306	is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
		5307	enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
3031	serge	5308
3480	Serge	5309	if (enable) {
		5310	if (!enable_requested)
4104	Serge	5311	I915_WRITE(HSW_PWR_WELL_DRIVER,
		5312	HSW_PWR_WELL_ENABLE_REQUEST);
3031	serge	5313
3480	Serge	5314	if (!is_enabled) {
		5315	DRM_DEBUG_KMS("Enabling power well\n");
		5316	if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
4104	Serge	5317	HSW_PWR_WELL_STATE_ENABLED), 20))
3480	Serge	5318	DRM_ERROR("Timeout enabling power well\n");
3031	serge	5319	}
3480	Serge	5320	} else {
		5321	if (enable_requested) {
4104	Serge	5322	unsigned long irqflags;
		5323	enum pipe p;
		5324
3480	Serge	5325	I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
4104	Serge	5326	POSTING_READ(HSW_PWR_WELL_DRIVER);
3480	Serge	5327	DRM_DEBUG_KMS("Requesting to disable the power well\n");
4104	Serge	5328
		5329	/*
		5330	* After this, the registers on the pipes that are part
		5331	* of the power well will become zero, so we have to
		5332	* adjust our counters according to that.
		5333	*
		5334	* FIXME: Should we do this in general in
		5335	* drm_vblank_post_modeset?
		5336	*/
		5337	spin_lock_irqsave(&dev->vbl_lock, irqflags);
		5338	for_each_pipe(p)
		5339	if (p != PIPE_A)
		5340	dev->last_vblank[p] = 0;
		5341	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
		5342	}
		5343	}
		5344	}
		5345
		5346	static struct i915_power_well *hsw_pwr;
		5347
		5348	/* Display audio driver power well request */
		5349	void i915_request_power_well(void)
		5350	{
		5351	if (WARN_ON(!hsw_pwr))
		5352	return;
		5353
		5354	spin_lock_irq(&hsw_pwr->lock);
		5355	if (!hsw_pwr->count++ &&
		5356	!hsw_pwr->i915_request)
		5357	__intel_set_power_well(hsw_pwr->device, true);
		5358	spin_unlock_irq(&hsw_pwr->lock);
		5359	}
		5360	EXPORT_SYMBOL_GPL(i915_request_power_well);
		5361
		5362	/* Display audio driver power well release */
		5363	void i915_release_power_well(void)
		5364	{
		5365	if (WARN_ON(!hsw_pwr))
		5366	return;
		5367
		5368	spin_lock_irq(&hsw_pwr->lock);
		5369	WARN_ON(!hsw_pwr->count);
		5370	if (!--hsw_pwr->count &&
		5371	!hsw_pwr->i915_request)
		5372	__intel_set_power_well(hsw_pwr->device, false);
		5373	spin_unlock_irq(&hsw_pwr->lock);
		5374	}
		5375	EXPORT_SYMBOL_GPL(i915_release_power_well);
		5376
		5377	int i915_init_power_well(struct drm_device *dev)
		5378	{
		5379	struct drm_i915_private *dev_priv = dev->dev_private;
		5380
		5381	hsw_pwr = &dev_priv->power_well;
		5382
		5383	hsw_pwr->device = dev;
		5384	spin_lock_init(&hsw_pwr->lock);
		5385	hsw_pwr->count = 0;
		5386
		5387	return 0;
		5388	}
		5389
		5390	void i915_remove_power_well(struct drm_device *dev)
		5391	{
		5392	hsw_pwr = NULL;
		5393	}
		5394
		5395	void intel_set_power_well(struct drm_device *dev, bool enable)
		5396	{
		5397	struct drm_i915_private *dev_priv = dev->dev_private;
		5398	struct i915_power_well *power_well = &dev_priv->power_well;
		5399
		5400	if (!HAS_POWER_WELL(dev))
		5401	return;
		5402
		5403	if (!i915_disable_power_well && !enable)
		5404	return;
		5405
		5406	spin_lock_irq(&power_well->lock);
		5407	power_well->i915_request = enable;
		5408
		5409	/* only reject "disable" power well request */
		5410	if (power_well->count && !enable) {
		5411	spin_unlock_irq(&power_well->lock);
		5412	return;
3031	serge	5413	}
4104	Serge	5414
		5415	__intel_set_power_well(dev, enable);
		5416	spin_unlock_irq(&power_well->lock);
3480	Serge	5417	}
3031	serge	5418
3480	Serge	5419	/*
		5420	* Starting with Haswell, we have a "Power Down Well" that can be turned off
		5421	* when not needed anymore. We have 4 registers that can request the power well
		5422	* to be enabled, and it will only be disabled if none of the registers is
		5423	* requesting it to be enabled.
		5424	*/
		5425	void intel_init_power_well(struct drm_device *dev)
		5426	{
		5427	struct drm_i915_private *dev_priv = dev->dev_private;
		5428
3746	Serge	5429	if (!HAS_POWER_WELL(dev))
3480	Serge	5430	return;
		5431
		5432	/* For now, we need the power well to be always enabled. */
		5433	intel_set_power_well(dev, true);
		5434
		5435	/* We're taking over the BIOS, so clear any requests made by it since
		5436	* the driver is in charge now. */
4104	Serge	5437	if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
3480	Serge	5438	I915_WRITE(HSW_PWR_WELL_BIOS, 0);
3031	serge	5439	}
		5440
4104	Serge	5441	/* Disables PC8 so we can use the GMBUS and DP AUX interrupts. */
		5442	void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
		5443	{
		5444	hsw_disable_package_c8(dev_priv);
		5445	}
		5446
		5447	void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
		5448	{
		5449	hsw_enable_package_c8(dev_priv);
		5450	}
		5451
3031	serge	5452	/* Set up chip specific power management-related functions */
		5453	void intel_init_pm(struct drm_device *dev)
		5454	{
		5455	struct drm_i915_private *dev_priv = dev->dev_private;
		5456
		5457	if (I915_HAS_FBC(dev)) {
		5458	if (HAS_PCH_SPLIT(dev)) {
		5459	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
4104	Serge	5460	if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev))
		5461	dev_priv->display.enable_fbc =
		5462	gen7_enable_fbc;
		5463	else
		5464	dev_priv->display.enable_fbc =
		5465	ironlake_enable_fbc;
3031	serge	5466	dev_priv->display.disable_fbc = ironlake_disable_fbc;
		5467	} else if (IS_GM45(dev)) {
		5468	dev_priv->display.fbc_enabled = g4x_fbc_enabled;
		5469	dev_priv->display.enable_fbc = g4x_enable_fbc;
		5470	dev_priv->display.disable_fbc = g4x_disable_fbc;
		5471	} else if (IS_CRESTLINE(dev)) {
		5472	dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
		5473	dev_priv->display.enable_fbc = i8xx_enable_fbc;
		5474	dev_priv->display.disable_fbc = i8xx_disable_fbc;
		5475	}
		5476	/* 855GM needs testing */
		5477	}
		5478
		5479	/* For cxsr */
		5480	if (IS_PINEVIEW(dev))
		5481	i915_pineview_get_mem_freq(dev);
		5482	else if (IS_GEN5(dev))
		5483	i915_ironlake_get_mem_freq(dev);
		5484
		5485	/* For FIFO watermark updates */
		5486	if (HAS_PCH_SPLIT(dev)) {
4104	Serge	5487	intel_setup_wm_latency(dev);
		5488
3031	serge	5489	if (IS_GEN5(dev)) {
4104	Serge	5490	if (dev_priv->wm.pri_latency[1] &&
		5491	dev_priv->wm.spr_latency[1] &&
		5492	dev_priv->wm.cur_latency[1])
3031	serge	5493	dev_priv->display.update_wm = ironlake_update_wm;
		5494	else {
		5495	DRM_DEBUG_KMS("Failed to get proper latency. "
		5496	"Disable CxSR\n");
		5497	dev_priv->display.update_wm = NULL;
		5498	}
		5499	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		5500	} else if (IS_GEN6(dev)) {
4104	Serge	5501	if (dev_priv->wm.pri_latency[0] &&
		5502	dev_priv->wm.spr_latency[0] &&
		5503	dev_priv->wm.cur_latency[0]) {
3031	serge	5504	dev_priv->display.update_wm = sandybridge_update_wm;
		5505	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		5506	} else {
		5507	DRM_DEBUG_KMS("Failed to read display plane latency. "
		5508	"Disable CxSR\n");
		5509	dev_priv->display.update_wm = NULL;
		5510	}
		5511	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
		5512	} else if (IS_IVYBRIDGE(dev)) {
4104	Serge	5513	if (dev_priv->wm.pri_latency[0] &&
		5514	dev_priv->wm.spr_latency[0] &&
		5515	dev_priv->wm.cur_latency[0]) {
3243	Serge	5516	dev_priv->display.update_wm = ivybridge_update_wm;
3031	serge	5517	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		5518	} else {
		5519	DRM_DEBUG_KMS("Failed to read display plane latency. "
		5520	"Disable CxSR\n");
		5521	dev_priv->display.update_wm = NULL;
		5522	}
		5523	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
		5524	} else if (IS_HASWELL(dev)) {
4104	Serge	5525	if (dev_priv->wm.pri_latency[0] &&
		5526	dev_priv->wm.spr_latency[0] &&
		5527	dev_priv->wm.cur_latency[0]) {
		5528	dev_priv->display.update_wm = haswell_update_wm;
		5529	dev_priv->display.update_sprite_wm =
		5530	haswell_update_sprite_wm;
3031	serge	5531	} else {
		5532	DRM_DEBUG_KMS("Failed to read display plane latency. "
		5533	"Disable CxSR\n");
		5534	dev_priv->display.update_wm = NULL;
		5535	}
		5536	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
		5537	} else
		5538	dev_priv->display.update_wm = NULL;
		5539	} else if (IS_VALLEYVIEW(dev)) {
		5540	dev_priv->display.update_wm = valleyview_update_wm;
		5541	dev_priv->display.init_clock_gating =
		5542	valleyview_init_clock_gating;
		5543	} else if (IS_PINEVIEW(dev)) {
		5544	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		5545	dev_priv->is_ddr3,
		5546	dev_priv->fsb_freq,
		5547	dev_priv->mem_freq)) {
		5548	DRM_INFO("failed to find known CxSR latency "
		5549	"(found ddr%s fsb freq %d, mem freq %d), "
		5550	"disabling CxSR\n",
		5551	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		5552	dev_priv->fsb_freq, dev_priv->mem_freq);
		5553	/* Disable CxSR and never update its watermark again */
		5554	pineview_disable_cxsr(dev);
		5555	dev_priv->display.update_wm = NULL;
		5556	} else
		5557	dev_priv->display.update_wm = pineview_update_wm;
		5558	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		5559	} else if (IS_G4X(dev)) {
		5560	dev_priv->display.update_wm = g4x_update_wm;
		5561	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		5562	} else if (IS_GEN4(dev)) {
		5563	dev_priv->display.update_wm = i965_update_wm;
		5564	if (IS_CRESTLINE(dev))
		5565	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		5566	else if (IS_BROADWATER(dev))
		5567	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		5568	} else if (IS_GEN3(dev)) {
		5569	dev_priv->display.update_wm = i9xx_update_wm;
		5570	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		5571	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		5572	} else if (IS_I865G(dev)) {
		5573	dev_priv->display.update_wm = i830_update_wm;
		5574	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		5575	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		5576	} else if (IS_I85X(dev)) {
		5577	dev_priv->display.update_wm = i9xx_update_wm;
		5578	dev_priv->display.get_fifo_size = i85x_get_fifo_size;
		5579	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		5580	} else {
		5581	dev_priv->display.update_wm = i830_update_wm;
		5582	dev_priv->display.init_clock_gating = i830_init_clock_gating;
		5583	if (IS_845G(dev))
		5584	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		5585	else
		5586	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		5587	}
		5588	}
		5589
3243	Serge	5590	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u8 mbox, u32 val)
		5591	{
		5592	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	5593
3243	Serge	5594	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		5595	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		5596	return -EAGAIN;
		5597	}
3031	serge	5598
3243	Serge	5599	I915_WRITE(GEN6_PCODE_DATA, *val);
		5600	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		5601
		5602	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		5603	500)) {
		5604	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		5605	return -ETIMEDOUT;
3031	serge	5606	}
3243	Serge	5607
		5608	*val = I915_READ(GEN6_PCODE_DATA);
		5609	I915_WRITE(GEN6_PCODE_DATA, 0);
		5610
		5611	return 0;
		5612	}
		5613
		5614	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
		5615	{
		5616	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5617
		5618	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		5619	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		5620	return -EAGAIN;
3031	serge	5621	}
3243	Serge	5622
		5623	I915_WRITE(GEN6_PCODE_DATA, val);
		5624	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		5625
		5626	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		5627	500)) {
		5628	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		5629	return -ETIMEDOUT;
3031	serge	5630	}
3243	Serge	5631
		5632	I915_WRITE(GEN6_PCODE_DATA, 0);
		5633
		5634	return 0;
3031	serge	5635	}
3746	Serge	5636
4104	Serge	5637	int vlv_gpu_freq(int ddr_freq, int val)
3746	Serge	5638	{
4104	Serge	5639	int mult, base;
3746	Serge	5640
4104	Serge	5641	switch (ddr_freq) {
		5642	case 800:
		5643	mult = 20;
		5644	base = 120;
		5645	break;
		5646	case 1066:
		5647	mult = 22;
		5648	base = 133;
		5649	break;
		5650	case 1333:
		5651	mult = 21;
		5652	base = 125;
		5653	break;
		5654	default:
		5655	return -1;
		5656	}
3746	Serge	5657
4104	Serge	5658	return ((val - 0xbd) * mult) + base;
		5659	}
3746	Serge	5660
4104	Serge	5661	int vlv_freq_opcode(int ddr_freq, int val)
		5662	{
		5663	int mult, base;
3746	Serge	5664
4104	Serge	5665	switch (ddr_freq) {
		5666	case 800:
		5667	mult = 20;
		5668	base = 120;
		5669	break;
		5670	case 1066:
		5671	mult = 22;
		5672	base = 133;
		5673	break;
		5674	case 1333:
		5675	mult = 21;
		5676	base = 125;
		5677	break;
		5678	default:
		5679	return -1;
3746	Serge	5680	}
		5681
4104	Serge	5682	val /= mult;
		5683	val -= base / mult;
		5684	val += 0xbd;
3746	Serge	5685
4104	Serge	5686	if (val > 0xea)
		5687	val = 0xea;
3746	Serge	5688
4104	Serge	5689	return val;
3746	Serge	5690	}
		5691
4104	Serge	5692	void intel_pm_init(struct drm_device *dev)
3746	Serge	5693	{
4104	Serge	5694	struct drm_i915_private *dev_priv = dev->dev_private;
		5695
		5696	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		5697	intel_gen6_powersave_work);
		5698
		5699	INIT_DELAYED_WORK(&dev_priv->rps.vlv_work, vlv_rps_timer_work);
3746	Serge	5700	}
		5701

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 4104