WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	#include
		32	//#include "../../../platform/x86/intel_ips.h"
		33	#include
		34
		35	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		36
		37	#define assert_spin_locked(x)
		38
		39	void getrawmonotonic(struct timespec *ts);
		40
		41
		42
		43	/* FBC, or Frame Buffer Compression, is a technique employed to compress the
		44	* framebuffer contents in-memory, aiming at reducing the required bandwidth
		45	* during in-memory transfers and, therefore, reduce the power packet.
		46	*
		47	* The benefits of FBC are mostly visible with solid backgrounds and
		48	* variation-less patterns.
		49	*
		50	* FBC-related functionality can be enabled by the means of the
		51	* i915.i915_enable_fbc parameter
		52	*/
		53
3243	Serge	54	static bool intel_crtc_active(struct drm_crtc *crtc)
		55	{
		56	/* Be paranoid as we can arrive here with only partial
		57	* state retrieved from the hardware during setup.
		58	*/
		59	return to_intel_crtc(crtc)->active && crtc->fb && crtc->mode.clock;
		60	}
		61
3031	serge	62	static void i8xx_disable_fbc(struct drm_device *dev)
		63	{
		64	struct drm_i915_private *dev_priv = dev->dev_private;
		65	u32 fbc_ctl;
		66
		67	/* Disable compression */
		68	fbc_ctl = I915_READ(FBC_CONTROL);
		69	if ((fbc_ctl & FBC_CTL_EN) == 0)
		70	return;
		71
		72	fbc_ctl &= ~FBC_CTL_EN;
		73	I915_WRITE(FBC_CONTROL, fbc_ctl);
		74
		75	/* Wait for compressing bit to clear */
		76	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
		77	DRM_DEBUG_KMS("FBC idle timed out\n");
		78	return;
		79	}
		80
		81	DRM_DEBUG_KMS("disabled FBC\n");
		82	}
		83
		84	static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		85	{
		86	struct drm_device *dev = crtc->dev;
		87	struct drm_i915_private *dev_priv = dev->dev_private;
		88	struct drm_framebuffer *fb = crtc->fb;
		89	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		90	struct drm_i915_gem_object *obj = intel_fb->obj;
		91	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		92	int cfb_pitch;
		93	int plane, i;
		94	u32 fbc_ctl, fbc_ctl2;
		95
4104	Serge	96	cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE;
3031	serge	97	if (fb->pitches[0] < cfb_pitch)
		98	cfb_pitch = fb->pitches[0];
		99
		100	/* FBC_CTL wants 64B units */
		101	cfb_pitch = (cfb_pitch / 64) - 1;
		102	plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
		103
		104	/* Clear old tags */
		105	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
		106	I915_WRITE(FBC_TAG + (i * 4), 0);
		107
		108	/* Set it up... */
		109	fbc_ctl2 = FBC_CTL_FENCE_DBL \| FBC_CTL_IDLE_IMM \| FBC_CTL_CPU_FENCE;
		110	fbc_ctl2 \|= plane;
		111	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
		112	I915_WRITE(FBC_FENCE_OFF, crtc->y);
		113
		114	/* enable it... */
		115	fbc_ctl = FBC_CTL_EN \| FBC_CTL_PERIODIC;
		116	if (IS_I945GM(dev))
		117	fbc_ctl \|= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
		118	fbc_ctl \|= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
		119	fbc_ctl \|= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
		120	fbc_ctl \|= obj->fence_reg;
		121	I915_WRITE(FBC_CONTROL, fbc_ctl);
		122
4104	Serge	123	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ",
		124	cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
3031	serge	125	}
		126
		127	static bool i8xx_fbc_enabled(struct drm_device *dev)
		128	{
		129	struct drm_i915_private *dev_priv = dev->dev_private;
		130
		131	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
		132	}
		133
		134	static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		135	{
		136	struct drm_device *dev = crtc->dev;
		137	struct drm_i915_private *dev_priv = dev->dev_private;
		138	struct drm_framebuffer *fb = crtc->fb;
		139	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		140	struct drm_i915_gem_object *obj = intel_fb->obj;
		141	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		142	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		143	unsigned long stall_watermark = 200;
		144	u32 dpfc_ctl;
		145
		146	dpfc_ctl = plane \| DPFC_SR_EN \| DPFC_CTL_LIMIT_1X;
		147	dpfc_ctl \|= DPFC_CTL_FENCE_EN \| obj->fence_reg;
		148	I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
		149
		150	I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		151	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		152	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		153	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
		154
		155	/* enable it... */
		156	I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) \| DPFC_CTL_EN);
		157
4104	Serge	158	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
3031	serge	159	}
		160
		161	static void g4x_disable_fbc(struct drm_device *dev)
		162	{
		163	struct drm_i915_private *dev_priv = dev->dev_private;
		164	u32 dpfc_ctl;
		165
		166	/* Disable compression */
		167	dpfc_ctl = I915_READ(DPFC_CONTROL);
		168	if (dpfc_ctl & DPFC_CTL_EN) {
		169	dpfc_ctl &= ~DPFC_CTL_EN;
		170	I915_WRITE(DPFC_CONTROL, dpfc_ctl);
		171
		172	DRM_DEBUG_KMS("disabled FBC\n");
		173	}
		174	}
		175
		176	static bool g4x_fbc_enabled(struct drm_device *dev)
		177	{
		178	struct drm_i915_private *dev_priv = dev->dev_private;
		179
		180	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
		181	}
		182
		183	static void sandybridge_blit_fbc_update(struct drm_device *dev)
		184	{
		185	struct drm_i915_private *dev_priv = dev->dev_private;
		186	u32 blt_ecoskpd;
		187
		188	/* Make sure blitter notifies FBC of writes */
		189	gen6_gt_force_wake_get(dev_priv);
		190	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
		191	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY <<
		192	GEN6_BLITTER_LOCK_SHIFT;
		193	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		194	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY;
		195	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		196	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
		197	GEN6_BLITTER_LOCK_SHIFT);
		198	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		199	POSTING_READ(GEN6_BLITTER_ECOSKPD);
		200	gen6_gt_force_wake_put(dev_priv);
		201	}
		202
		203	static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		204	{
		205	struct drm_device *dev = crtc->dev;
		206	struct drm_i915_private *dev_priv = dev->dev_private;
		207	struct drm_framebuffer *fb = crtc->fb;
		208	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		209	struct drm_i915_gem_object *obj = intel_fb->obj;
		210	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		211	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		212	unsigned long stall_watermark = 200;
		213	u32 dpfc_ctl;
		214
		215	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		216	dpfc_ctl &= DPFC_RESERVED;
		217	dpfc_ctl \|= (plane \| DPFC_CTL_LIMIT_1X);
		218	/* Set persistent mode for front-buffer rendering, ala X. */
		219	dpfc_ctl \|= DPFC_CTL_PERSISTENT_MODE;
		220	dpfc_ctl \|= (DPFC_CTL_FENCE_EN \| obj->fence_reg);
		221	I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
		222
		223	I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		224	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		225	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		226	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
4104	Serge	227	I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) \| ILK_FBC_RT_VALID);
3031	serge	228	/* enable it... */
		229	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		230
		231	if (IS_GEN6(dev)) {
		232	I915_WRITE(SNB_DPFC_CTL_SA,
		233	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		234	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		235	sandybridge_blit_fbc_update(dev);
		236	}
		237
4104	Serge	238	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
3031	serge	239	}
		240
		241	static void ironlake_disable_fbc(struct drm_device *dev)
		242	{
		243	struct drm_i915_private *dev_priv = dev->dev_private;
		244	u32 dpfc_ctl;
		245
		246	/* Disable compression */
		247	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		248	if (dpfc_ctl & DPFC_CTL_EN) {
		249	dpfc_ctl &= ~DPFC_CTL_EN;
		250	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
		251
4104	Serge	252	if (IS_IVYBRIDGE(dev))
		253	/* WaFbcDisableDpfcClockGating:ivb */
		254	I915_WRITE(ILK_DSPCLK_GATE_D,
		255	I915_READ(ILK_DSPCLK_GATE_D) &
		256	~ILK_DPFCUNIT_CLOCK_GATE_DISABLE);
		257
		258	if (IS_HASWELL(dev))
		259	/* WaFbcDisableDpfcClockGating:hsw */
		260	I915_WRITE(HSW_CLKGATE_DISABLE_PART_1,
		261	I915_READ(HSW_CLKGATE_DISABLE_PART_1) &
		262	~HSW_DPFC_GATING_DISABLE);
		263
3031	serge	264	DRM_DEBUG_KMS("disabled FBC\n");
		265	}
		266	}
		267
		268	static bool ironlake_fbc_enabled(struct drm_device *dev)
		269	{
		270	struct drm_i915_private *dev_priv = dev->dev_private;
		271
		272	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
		273	}
		274
4104	Serge	275	static void gen7_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		276	{
		277	struct drm_device *dev = crtc->dev;
		278	struct drm_i915_private *dev_priv = dev->dev_private;
		279	struct drm_framebuffer *fb = crtc->fb;
		280	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		281	struct drm_i915_gem_object *obj = intel_fb->obj;
		282	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		283
		284	I915_WRITE(IVB_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj));
		285
		286	I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN \| DPFC_CTL_LIMIT_1X \|
		287	IVB_DPFC_CTL_FENCE_EN \|
		288	intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT);
		289
		290	if (IS_IVYBRIDGE(dev)) {
		291	/* WaFbcAsynchFlipDisableFbcQueue:ivb */
		292	I915_WRITE(ILK_DISPLAY_CHICKEN1, ILK_FBCQ_DIS);
		293	/* WaFbcDisableDpfcClockGating:ivb */
		294	I915_WRITE(ILK_DSPCLK_GATE_D,
		295	I915_READ(ILK_DSPCLK_GATE_D) \|
		296	ILK_DPFCUNIT_CLOCK_GATE_DISABLE);
		297	} else {
		298	/* WaFbcAsynchFlipDisableFbcQueue:hsw */
		299	I915_WRITE(HSW_PIPE_SLICE_CHICKEN_1(intel_crtc->pipe),
		300	HSW_BYPASS_FBC_QUEUE);
		301	/* WaFbcDisableDpfcClockGating:hsw */
		302	I915_WRITE(HSW_CLKGATE_DISABLE_PART_1,
		303	I915_READ(HSW_CLKGATE_DISABLE_PART_1) \|
		304	HSW_DPFC_GATING_DISABLE);
		305	}
		306
		307	I915_WRITE(SNB_DPFC_CTL_SA,
		308	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		309	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		310
		311	sandybridge_blit_fbc_update(dev);
		312
		313	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		314	}
		315
3031	serge	316	bool intel_fbc_enabled(struct drm_device *dev)
		317	{
		318	struct drm_i915_private *dev_priv = dev->dev_private;
		319
		320	if (!dev_priv->display.fbc_enabled)
		321	return false;
		322
		323	return dev_priv->display.fbc_enabled(dev);
		324	}
		325
		326	static void intel_fbc_work_fn(struct work_struct *__work)
		327	{
		328	struct intel_fbc_work *work =
		329	container_of(to_delayed_work(__work),
		330	struct intel_fbc_work, work);
		331	struct drm_device *dev = work->crtc->dev;
		332	struct drm_i915_private *dev_priv = dev->dev_private;
		333
		334	mutex_lock(&dev->struct_mutex);
4104	Serge	335	if (work == dev_priv->fbc.fbc_work) {
3031	serge	336	/* Double check that we haven't switched fb without cancelling
		337	* the prior work.
		338	*/
		339	if (work->crtc->fb == work->fb) {
		340	dev_priv->display.enable_fbc(work->crtc,
		341	work->interval);
		342
4104	Serge	343	dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane;
		344	dev_priv->fbc.fb_id = work->crtc->fb->base.id;
		345	dev_priv->fbc.y = work->crtc->y;
3031	serge	346	}
		347
4104	Serge	348	dev_priv->fbc.fbc_work = NULL;
3031	serge	349	}
		350	mutex_unlock(&dev->struct_mutex);
		351
		352	kfree(work);
		353	}
		354
		355	static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
		356	{
4104	Serge	357	if (dev_priv->fbc.fbc_work == NULL)
3031	serge	358	return;
		359
		360	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
		361
		362	/* Synchronisation is provided by struct_mutex and checking of
4104	Serge	363	* dev_priv->fbc.fbc_work, so we can perform the cancellation
3031	serge	364	* entirely asynchronously.
		365	*/
4293	Serge	366	if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work))
3031	serge	367	/* tasklet was killed before being run, clean up */
4293	Serge	368	kfree(dev_priv->fbc.fbc_work);
3031	serge	369
		370	/* Mark the work as no longer wanted so that if it does
		371	* wake-up (because the work was already running and waiting
		372	* for our mutex), it will discover that is no longer
		373	* necessary to run.
		374	*/
4104	Serge	375	dev_priv->fbc.fbc_work = NULL;
3031	serge	376	}
		377
4104	Serge	378	static void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
3031	serge	379	{
		380	struct intel_fbc_work *work;
		381	struct drm_device *dev = crtc->dev;
		382	struct drm_i915_private *dev_priv = dev->dev_private;
		383
3482	Serge	384	if (!dev_priv->display.enable_fbc)
3031	serge	385	return;
3482	Serge	386
3031	serge	387	intel_cancel_fbc_work(dev_priv);
		388
		389	work = kzalloc(sizeof *work, GFP_KERNEL);
		390	if (work == NULL) {
4104	Serge	391	DRM_ERROR("Failed to allocate FBC work structure\n");
3031	serge	392	dev_priv->display.enable_fbc(crtc, interval);
		393	return;
		394	}
		395
		396	work->crtc = crtc;
		397	work->fb = crtc->fb;
		398	work->interval = interval;
		399	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
		400
4104	Serge	401	dev_priv->fbc.fbc_work = work;
3031	serge	402
		403	/* Delay the actual enabling to let pageflipping cease and the
		404	* display to settle before starting the compression. Note that
		405	* this delay also serves a second purpose: it allows for a
		406	* vblank to pass after disabling the FBC before we attempt
		407	* to modify the control registers.
		408	*
		409	* A more complicated solution would involve tracking vblanks
		410	* following the termination of the page-flipping sequence
		411	* and indeed performing the enable as a co-routine and not
		412	* waiting synchronously upon the vblank.
4104	Serge	413	*
		414	* WaFbcWaitForVBlankBeforeEnable:ilk,snb
3031	serge	415	*/
		416	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
		417	}
		418
		419	void intel_disable_fbc(struct drm_device *dev)
		420	{
		421	struct drm_i915_private *dev_priv = dev->dev_private;
		422
3482	Serge	423	intel_cancel_fbc_work(dev_priv);
3031	serge	424
3482	Serge	425	if (!dev_priv->display.disable_fbc)
		426	return;
3031	serge	427
3482	Serge	428	dev_priv->display.disable_fbc(dev);
4104	Serge	429	dev_priv->fbc.plane = -1;
3031	serge	430	}
		431
4104	Serge	432	static bool set_no_fbc_reason(struct drm_i915_private *dev_priv,
		433	enum no_fbc_reason reason)
		434	{
		435	if (dev_priv->fbc.no_fbc_reason == reason)
		436	return false;
		437
		438	dev_priv->fbc.no_fbc_reason = reason;
		439	return true;
		440	}
		441
3031	serge	442	/**
		443	* intel_update_fbc - enable/disable FBC as needed
		444	* @dev: the drm_device
		445	*
		446	* Set up the framebuffer compression hardware at mode set time. We
		447	* enable it if possible:
		448	* - plane A only (on pre-965)
		449	* - no pixel mulitply/line duplication
		450	* - no alpha buffer discard
		451	* - no dual wide
4104	Serge	452	* - framebuffer <= max_hdisplay in width, max_vdisplay in height
3031	serge	453	*
		454	* We can't assume that any compression will take place (worst case),
		455	* so the compressed buffer has to be the same size as the uncompressed
		456	* one. It also must reside (along with the line length buffer) in
		457	* stolen memory.
		458	*
		459	* We need to enable/disable FBC on a global basis.
		460	*/
		461	void intel_update_fbc(struct drm_device *dev)
		462	{
		463	struct drm_i915_private *dev_priv = dev->dev_private;
		464	struct drm_crtc crtc = NULL, tmp_crtc;
		465	struct intel_crtc *intel_crtc;
		466	struct drm_framebuffer *fb;
		467	struct intel_framebuffer *intel_fb;
		468	struct drm_i915_gem_object *obj;
4104	Serge	469	unsigned int max_hdisplay, max_vdisplay;
3031	serge	470
4104	Serge	471	if (!I915_HAS_FBC(dev)) {
		472	set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED);
3031	serge	473	return;
4104	Serge	474	}
3031	serge	475
4104	Serge	476	if (!i915_powersave) {
		477	if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
		478	DRM_DEBUG_KMS("fbc disabled per module param\n");
3031	serge	479	return;
4104	Serge	480	}
3031	serge	481
		482	/*
		483	* If FBC is already on, we just have to verify that we can
		484	* keep it that way...
		485	* Need to disable if:
		486	* - more than one pipe is active
		487	* - changing FBC params (stride, fence, mode)
		488	* - new fb is too large to fit in compressed buffer
		489	* - going to an unsupported config (interlace, pixel multiply, etc.)
		490	*/
		491	list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	492	if (intel_crtc_active(tmp_crtc) &&
		493	!to_intel_crtc(tmp_crtc)->primary_disabled) {
3031	serge	494	if (crtc) {
4104	Serge	495	if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES))
3031	serge	496	DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
		497	goto out_disable;
		498	}
		499	crtc = tmp_crtc;
		500	}
		501	}
		502
		503	if (!crtc \|\| crtc->fb == NULL) {
4104	Serge	504	if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT))
3031	serge	505	DRM_DEBUG_KMS("no output, disabling\n");
		506	goto out_disable;
		507	}
		508
		509	intel_crtc = to_intel_crtc(crtc);
		510	fb = crtc->fb;
		511	intel_fb = to_intel_framebuffer(fb);
		512	obj = intel_fb->obj;
		513
4104	Serge	514	if (i915_enable_fbc < 0 &&
		515	INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) {
		516	if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT))
		517	DRM_DEBUG_KMS("disabled per chip default\n");
		518	goto out_disable;
3031	serge	519	}
4104	Serge	520	if (!i915_enable_fbc) {
		521	if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
3031	serge	522	DRM_DEBUG_KMS("fbc disabled per module param\n");
		523	goto out_disable;
		524	}
		525	if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) \|\|
		526	(crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
4104	Serge	527	if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
3031	serge	528	DRM_DEBUG_KMS("mode incompatible with compression, "
		529	"disabling\n");
		530	goto out_disable;
		531	}
4104	Serge	532
		533	if (IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
		534	max_hdisplay = 4096;
		535	max_vdisplay = 2048;
		536	} else {
		537	max_hdisplay = 2048;
		538	max_vdisplay = 1536;
		539	}
		540	if ((crtc->mode.hdisplay > max_hdisplay) \|\|
		541	(crtc->mode.vdisplay > max_vdisplay)) {
		542	if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE))
3031	serge	543	DRM_DEBUG_KMS("mode too large for compression, disabling\n");
		544	goto out_disable;
		545	}
4104	Serge	546	if ((IS_I915GM(dev) \|\| IS_I945GM(dev) \|\| IS_HASWELL(dev)) &&
		547	intel_crtc->plane != 0) {
		548	if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE))
3031	serge	549	DRM_DEBUG_KMS("plane not 0, disabling compression\n");
		550	goto out_disable;
		551	}
		552
		553	/* The use of a CPU fence is mandatory in order to detect writes
		554	* by the CPU to the scanout and trigger updates to the FBC.
		555	*/
		556	if (obj->tiling_mode != I915_TILING_X \|\|
		557	obj->fence_reg == I915_FENCE_REG_NONE) {
4104	Serge	558	if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED))
3031	serge	559	DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
		560	goto out_disable;
		561	}
		562
		563	/* If the kernel debugger is active, always disable compression */
		564	if (in_dbg_master())
		565	goto out_disable;
		566
3480	Serge	567	if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
4104	Serge	568	if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL))
3480	Serge	569	DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
		570	goto out_disable;
		571	}
		572
3031	serge	573	/* If the scanout has not changed, don't modify the FBC settings.
		574	* Note that we make the fundamental assumption that the fb->obj
		575	* cannot be unpinned (and have its GTT offset and fence revoked)
		576	* without first being decoupled from the scanout and FBC disabled.
		577	*/
4104	Serge	578	if (dev_priv->fbc.plane == intel_crtc->plane &&
		579	dev_priv->fbc.fb_id == fb->base.id &&
		580	dev_priv->fbc.y == crtc->y)
3031	serge	581	return;
		582
		583	if (intel_fbc_enabled(dev)) {
		584	/* We update FBC along two paths, after changing fb/crtc
		585	* configuration (modeswitching) and after page-flipping
		586	* finishes. For the latter, we know that not only did
		587	* we disable the FBC at the start of the page-flip
		588	* sequence, but also more than one vblank has passed.
		589	*
		590	* For the former case of modeswitching, it is possible
		591	* to switch between two FBC valid configurations
		592	* instantaneously so we do need to disable the FBC
		593	* before we can modify its control registers. We also
		594	* have to wait for the next vblank for that to take
		595	* effect. However, since we delay enabling FBC we can
		596	* assume that a vblank has passed since disabling and
		597	* that we can safely alter the registers in the deferred
		598	* callback.
		599	*
		600	* In the scenario that we go from a valid to invalid
		601	* and then back to valid FBC configuration we have
		602	* no strict enforcement that a vblank occurred since
		603	* disabling the FBC. However, along all current pipe
		604	* disabling paths we do need to wait for a vblank at
		605	* some point. And we wait before enabling FBC anyway.
		606	*/
		607	DRM_DEBUG_KMS("disabling active FBC for update\n");
		608	intel_disable_fbc(dev);
		609	}
		610
		611	intel_enable_fbc(crtc, 500);
4104	Serge	612	dev_priv->fbc.no_fbc_reason = FBC_OK;
3031	serge	613	return;
		614
		615	out_disable:
		616	/* Multiple disables should be harmless */
		617	if (intel_fbc_enabled(dev)) {
		618	DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
		619	intel_disable_fbc(dev);
		620	}
3480	Serge	621	i915_gem_stolen_cleanup_compression(dev);
3031	serge	622	}
		623
		624	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		625	{
		626	drm_i915_private_t *dev_priv = dev->dev_private;
		627	u32 tmp;
		628
		629	tmp = I915_READ(CLKCFG);
		630
		631	switch (tmp & CLKCFG_FSB_MASK) {
		632	case CLKCFG_FSB_533:
		633	dev_priv->fsb_freq = 533; /* 1334 /
		634	break;
		635	case CLKCFG_FSB_800:
		636	dev_priv->fsb_freq = 800; /* 2004 /
		637	break;
		638	case CLKCFG_FSB_667:
		639	dev_priv->fsb_freq = 667; /* 1674 /
		640	break;
		641	case CLKCFG_FSB_400:
		642	dev_priv->fsb_freq = 400; /* 1004 /
		643	break;
		644	}
		645
		646	switch (tmp & CLKCFG_MEM_MASK) {
		647	case CLKCFG_MEM_533:
		648	dev_priv->mem_freq = 533;
		649	break;
		650	case CLKCFG_MEM_667:
		651	dev_priv->mem_freq = 667;
		652	break;
		653	case CLKCFG_MEM_800:
		654	dev_priv->mem_freq = 800;
		655	break;
		656	}
		657
		658	/* detect pineview DDR3 setting */
		659	tmp = I915_READ(CSHRDDR3CTL);
		660	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		661	}
		662
		663	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		664	{
		665	drm_i915_private_t *dev_priv = dev->dev_private;
		666	u16 ddrpll, csipll;
		667
		668	ddrpll = I915_READ16(DDRMPLL1);
		669	csipll = I915_READ16(CSIPLL0);
		670
		671	switch (ddrpll & 0xff) {
		672	case 0xc:
		673	dev_priv->mem_freq = 800;
		674	break;
		675	case 0x10:
		676	dev_priv->mem_freq = 1066;
		677	break;
		678	case 0x14:
		679	dev_priv->mem_freq = 1333;
		680	break;
		681	case 0x18:
		682	dev_priv->mem_freq = 1600;
		683	break;
		684	default:
		685	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		686	ddrpll & 0xff);
		687	dev_priv->mem_freq = 0;
		688	break;
		689	}
		690
		691	dev_priv->ips.r_t = dev_priv->mem_freq;
		692
		693	switch (csipll & 0x3ff) {
		694	case 0x00c:
		695	dev_priv->fsb_freq = 3200;
		696	break;
		697	case 0x00e:
		698	dev_priv->fsb_freq = 3733;
		699	break;
		700	case 0x010:
		701	dev_priv->fsb_freq = 4266;
		702	break;
		703	case 0x012:
		704	dev_priv->fsb_freq = 4800;
		705	break;
		706	case 0x014:
		707	dev_priv->fsb_freq = 5333;
		708	break;
		709	case 0x016:
		710	dev_priv->fsb_freq = 5866;
		711	break;
		712	case 0x018:
		713	dev_priv->fsb_freq = 6400;
		714	break;
		715	default:
		716	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		717	csipll & 0x3ff);
		718	dev_priv->fsb_freq = 0;
		719	break;
		720	}
		721
		722	if (dev_priv->fsb_freq == 3200) {
		723	dev_priv->ips.c_m = 0;
		724	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		725	dev_priv->ips.c_m = 1;
		726	} else {
		727	dev_priv->ips.c_m = 2;
		728	}
		729	}
		730
		731	static const struct cxsr_latency cxsr_latency_table[] = {
		732	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		733	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		734	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		735	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		736	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		737
		738	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		739	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		740	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		741	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		742	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		743
		744	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		745	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		746	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		747	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		748	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		749
		750	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		751	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		752	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		753	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		754	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		755
		756	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		757	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		758	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		759	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		760	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		761
		762	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		763	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		764	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		765	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		766	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		767	};
		768
		769	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		770	int is_ddr3,
		771	int fsb,
		772	int mem)
		773	{
		774	const struct cxsr_latency *latency;
		775	int i;
		776
		777	if (fsb == 0 \|\| mem == 0)
		778	return NULL;
		779
		780	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		781	latency = &cxsr_latency_table[i];
		782	if (is_desktop == latency->is_desktop &&
		783	is_ddr3 == latency->is_ddr3 &&
		784	fsb == latency->fsb_freq && mem == latency->mem_freq)
		785	return latency;
		786	}
		787
		788	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		789
		790	return NULL;
		791	}
		792
		793	static void pineview_disable_cxsr(struct drm_device *dev)
		794	{
		795	struct drm_i915_private *dev_priv = dev->dev_private;
		796
		797	/* deactivate cxsr */
		798	I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
		799	}
		800
		801	/*
		802	* Latency for FIFO fetches is dependent on several factors:
		803	* - memory configuration (speed, channels)
		804	* - chipset
		805	* - current MCH state
		806	* It can be fairly high in some situations, so here we assume a fairly
		807	* pessimal value. It's a tradeoff between extra memory fetches (if we
		808	* set this value too high, the FIFO will fetch frequently to stay full)
		809	* and power consumption (set it too low to save power and we might see
		810	* FIFO underruns and display "flicker").
		811	*
		812	* A value of 5us seems to be a good balance; safe for very low end
		813	* platforms but not overly aggressive on lower latency configs.
		814	*/
		815	static const int latency_ns = 5000;
		816
		817	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		818	{
		819	struct drm_i915_private *dev_priv = dev->dev_private;
		820	uint32_t dsparb = I915_READ(DSPARB);
		821	int size;
		822
		823	size = dsparb & 0x7f;
		824	if (plane)
		825	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		826
		827	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		828	plane ? "B" : "A", size);
		829
		830	return size;
		831	}
		832
		833	static int i85x_get_fifo_size(struct drm_device *dev, int plane)
		834	{
		835	struct drm_i915_private *dev_priv = dev->dev_private;
		836	uint32_t dsparb = I915_READ(DSPARB);
		837	int size;
		838
		839	size = dsparb & 0x1ff;
		840	if (plane)
		841	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		842	size >>= 1; /* Convert to cachelines */
		843
		844	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		845	plane ? "B" : "A", size);
		846
		847	return size;
		848	}
		849
		850	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		851	{
		852	struct drm_i915_private *dev_priv = dev->dev_private;
		853	uint32_t dsparb = I915_READ(DSPARB);
		854	int size;
		855
		856	size = dsparb & 0x7f;
		857	size >>= 2; /* Convert to cachelines */
		858
		859	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		860	plane ? "B" : "A",
		861	size);
		862
		863	return size;
		864	}
		865
		866	static int i830_get_fifo_size(struct drm_device *dev, int plane)
		867	{
		868	struct drm_i915_private *dev_priv = dev->dev_private;
		869	uint32_t dsparb = I915_READ(DSPARB);
		870	int size;
		871
		872	size = dsparb & 0x7f;
		873	size >>= 1; /* Convert to cachelines */
		874
		875	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		876	plane ? "B" : "A", size);
		877
		878	return size;
		879	}
		880
		881	/* Pineview has different values for various configs */
		882	static const struct intel_watermark_params pineview_display_wm = {
		883	PINEVIEW_DISPLAY_FIFO,
		884	PINEVIEW_MAX_WM,
		885	PINEVIEW_DFT_WM,
		886	PINEVIEW_GUARD_WM,
		887	PINEVIEW_FIFO_LINE_SIZE
		888	};
		889	static const struct intel_watermark_params pineview_display_hplloff_wm = {
		890	PINEVIEW_DISPLAY_FIFO,
		891	PINEVIEW_MAX_WM,
		892	PINEVIEW_DFT_HPLLOFF_WM,
		893	PINEVIEW_GUARD_WM,
		894	PINEVIEW_FIFO_LINE_SIZE
		895	};
		896	static const struct intel_watermark_params pineview_cursor_wm = {
		897	PINEVIEW_CURSOR_FIFO,
		898	PINEVIEW_CURSOR_MAX_WM,
		899	PINEVIEW_CURSOR_DFT_WM,
		900	PINEVIEW_CURSOR_GUARD_WM,
		901	PINEVIEW_FIFO_LINE_SIZE,
		902	};
		903	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
		904	PINEVIEW_CURSOR_FIFO,
		905	PINEVIEW_CURSOR_MAX_WM,
		906	PINEVIEW_CURSOR_DFT_WM,
		907	PINEVIEW_CURSOR_GUARD_WM,
		908	PINEVIEW_FIFO_LINE_SIZE
		909	};
		910	static const struct intel_watermark_params g4x_wm_info = {
		911	G4X_FIFO_SIZE,
		912	G4X_MAX_WM,
		913	G4X_MAX_WM,
		914	2,
		915	G4X_FIFO_LINE_SIZE,
		916	};
		917	static const struct intel_watermark_params g4x_cursor_wm_info = {
		918	I965_CURSOR_FIFO,
		919	I965_CURSOR_MAX_WM,
		920	I965_CURSOR_DFT_WM,
		921	2,
		922	G4X_FIFO_LINE_SIZE,
		923	};
		924	static const struct intel_watermark_params valleyview_wm_info = {
		925	VALLEYVIEW_FIFO_SIZE,
		926	VALLEYVIEW_MAX_WM,
		927	VALLEYVIEW_MAX_WM,
		928	2,
		929	G4X_FIFO_LINE_SIZE,
		930	};
		931	static const struct intel_watermark_params valleyview_cursor_wm_info = {
		932	I965_CURSOR_FIFO,
		933	VALLEYVIEW_CURSOR_MAX_WM,
		934	I965_CURSOR_DFT_WM,
		935	2,
		936	G4X_FIFO_LINE_SIZE,
		937	};
		938	static const struct intel_watermark_params i965_cursor_wm_info = {
		939	I965_CURSOR_FIFO,
		940	I965_CURSOR_MAX_WM,
		941	I965_CURSOR_DFT_WM,
		942	2,
		943	I915_FIFO_LINE_SIZE,
		944	};
		945	static const struct intel_watermark_params i945_wm_info = {
		946	I945_FIFO_SIZE,
		947	I915_MAX_WM,
		948	1,
		949	2,
		950	I915_FIFO_LINE_SIZE
		951	};
		952	static const struct intel_watermark_params i915_wm_info = {
		953	I915_FIFO_SIZE,
		954	I915_MAX_WM,
		955	1,
		956	2,
		957	I915_FIFO_LINE_SIZE
		958	};
		959	static const struct intel_watermark_params i855_wm_info = {
		960	I855GM_FIFO_SIZE,
		961	I915_MAX_WM,
		962	1,
		963	2,
		964	I830_FIFO_LINE_SIZE
		965	};
		966	static const struct intel_watermark_params i830_wm_info = {
		967	I830_FIFO_SIZE,
		968	I915_MAX_WM,
		969	1,
		970	2,
		971	I830_FIFO_LINE_SIZE
		972	};
		973
		974	static const struct intel_watermark_params ironlake_display_wm_info = {
		975	ILK_DISPLAY_FIFO,
		976	ILK_DISPLAY_MAXWM,
		977	ILK_DISPLAY_DFTWM,
		978	2,
		979	ILK_FIFO_LINE_SIZE
		980	};
		981	static const struct intel_watermark_params ironlake_cursor_wm_info = {
		982	ILK_CURSOR_FIFO,
		983	ILK_CURSOR_MAXWM,
		984	ILK_CURSOR_DFTWM,
		985	2,
		986	ILK_FIFO_LINE_SIZE
		987	};
		988	static const struct intel_watermark_params ironlake_display_srwm_info = {
		989	ILK_DISPLAY_SR_FIFO,
		990	ILK_DISPLAY_MAX_SRWM,
		991	ILK_DISPLAY_DFT_SRWM,
		992	2,
		993	ILK_FIFO_LINE_SIZE
		994	};
		995	static const struct intel_watermark_params ironlake_cursor_srwm_info = {
		996	ILK_CURSOR_SR_FIFO,
		997	ILK_CURSOR_MAX_SRWM,
		998	ILK_CURSOR_DFT_SRWM,
		999	2,
		1000	ILK_FIFO_LINE_SIZE
		1001	};
		1002
		1003	static const struct intel_watermark_params sandybridge_display_wm_info = {
		1004	SNB_DISPLAY_FIFO,
		1005	SNB_DISPLAY_MAXWM,
		1006	SNB_DISPLAY_DFTWM,
		1007	2,
		1008	SNB_FIFO_LINE_SIZE
		1009	};
		1010	static const struct intel_watermark_params sandybridge_cursor_wm_info = {
		1011	SNB_CURSOR_FIFO,
		1012	SNB_CURSOR_MAXWM,
		1013	SNB_CURSOR_DFTWM,
		1014	2,
		1015	SNB_FIFO_LINE_SIZE
		1016	};
		1017	static const struct intel_watermark_params sandybridge_display_srwm_info = {
		1018	SNB_DISPLAY_SR_FIFO,
		1019	SNB_DISPLAY_MAX_SRWM,
		1020	SNB_DISPLAY_DFT_SRWM,
		1021	2,
		1022	SNB_FIFO_LINE_SIZE
		1023	};
		1024	static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
		1025	SNB_CURSOR_SR_FIFO,
		1026	SNB_CURSOR_MAX_SRWM,
		1027	SNB_CURSOR_DFT_SRWM,
		1028	2,
		1029	SNB_FIFO_LINE_SIZE
		1030	};
		1031
		1032
		1033	/**
		1034	* intel_calculate_wm - calculate watermark level
		1035	* @clock_in_khz: pixel clock
		1036	* @wm: chip FIFO params
		1037	* @pixel_size: display pixel size
		1038	* @latency_ns: memory latency for the platform
		1039	*
		1040	* Calculate the watermark level (the level at which the display plane will
		1041	* start fetching from memory again). Each chip has a different display
		1042	* FIFO size and allocation, so the caller needs to figure that out and pass
		1043	* in the correct intel_watermark_params structure.
		1044	*
		1045	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		1046	* on the pixel size. When it reaches the watermark level, it'll start
		1047	* fetching FIFO line sized based chunks from memory until the FIFO fills
		1048	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		1049	* will occur, and a display engine hang could result.
		1050	*/
		1051	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		1052	const struct intel_watermark_params *wm,
		1053	int fifo_size,
		1054	int pixel_size,
		1055	unsigned long latency_ns)
		1056	{
		1057	long entries_required, wm_size;
		1058
		1059	/*
		1060	* Note: we need to make sure we don't overflow for various clock &
		1061	* latency values.
		1062	* clocks go from a few thousand to several hundred thousand.
		1063	* latency is usually a few thousand
		1064	*/
		1065	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		1066	1000;
		1067	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		1068
		1069	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		1070
		1071	wm_size = fifo_size - (entries_required + wm->guard_size);
		1072
		1073	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		1074
		1075	/* Don't promote wm_size to unsigned... */
		1076	if (wm_size > (long)wm->max_wm)
		1077	wm_size = wm->max_wm;
		1078	if (wm_size <= 0)
		1079	wm_size = wm->default_wm;
		1080	return wm_size;
		1081	}
		1082
		1083	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		1084	{
		1085	struct drm_crtc crtc, enabled = NULL;
		1086
		1087	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	1088	if (intel_crtc_active(crtc)) {
3031	serge	1089	if (enabled)
		1090	return NULL;
		1091	enabled = crtc;
		1092	}
		1093	}
		1094
		1095	return enabled;
		1096	}
		1097
		1098	static void pineview_update_wm(struct drm_device *dev)
		1099	{
		1100	struct drm_i915_private *dev_priv = dev->dev_private;
		1101	struct drm_crtc *crtc;
		1102	const struct cxsr_latency *latency;
		1103	u32 reg;
		1104	unsigned long wm;
		1105
		1106	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		1107	dev_priv->fsb_freq, dev_priv->mem_freq);
		1108	if (!latency) {
		1109	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		1110	pineview_disable_cxsr(dev);
		1111	return;
		1112	}
		1113
		1114	crtc = single_enabled_crtc(dev);
		1115	if (crtc) {
		1116	int clock = crtc->mode.clock;
		1117	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1118
		1119	/* Display SR */
		1120	wm = intel_calculate_wm(clock, &pineview_display_wm,
		1121	pineview_display_wm.fifo_size,
		1122	pixel_size, latency->display_sr);
		1123	reg = I915_READ(DSPFW1);
		1124	reg &= ~DSPFW_SR_MASK;
		1125	reg \|= wm << DSPFW_SR_SHIFT;
		1126	I915_WRITE(DSPFW1, reg);
		1127	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		1128
		1129	/* cursor SR */
		1130	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		1131	pineview_display_wm.fifo_size,
		1132	pixel_size, latency->cursor_sr);
		1133	reg = I915_READ(DSPFW3);
		1134	reg &= ~DSPFW_CURSOR_SR_MASK;
		1135	reg \|= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
		1136	I915_WRITE(DSPFW3, reg);
		1137
		1138	/* Display HPLL off SR */
		1139	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		1140	pineview_display_hplloff_wm.fifo_size,
		1141	pixel_size, latency->display_hpll_disable);
		1142	reg = I915_READ(DSPFW3);
		1143	reg &= ~DSPFW_HPLL_SR_MASK;
		1144	reg \|= wm & DSPFW_HPLL_SR_MASK;
		1145	I915_WRITE(DSPFW3, reg);
		1146
		1147	/* cursor HPLL off SR */
		1148	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		1149	pineview_display_hplloff_wm.fifo_size,
		1150	pixel_size, latency->cursor_hpll_disable);
		1151	reg = I915_READ(DSPFW3);
		1152	reg &= ~DSPFW_HPLL_CURSOR_MASK;
		1153	reg \|= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
		1154	I915_WRITE(DSPFW3, reg);
		1155	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		1156
		1157	/* activate cxsr */
		1158	I915_WRITE(DSPFW3,
		1159	I915_READ(DSPFW3) \| PINEVIEW_SELF_REFRESH_EN);
		1160	DRM_DEBUG_KMS("Self-refresh is enabled\n");
		1161	} else {
		1162	pineview_disable_cxsr(dev);
		1163	DRM_DEBUG_KMS("Self-refresh is disabled\n");
		1164	}
		1165	}
		1166
		1167	static bool g4x_compute_wm0(struct drm_device *dev,
		1168	int plane,
		1169	const struct intel_watermark_params *display,
		1170	int display_latency_ns,
		1171	const struct intel_watermark_params *cursor,
		1172	int cursor_latency_ns,
		1173	int *plane_wm,
		1174	int *cursor_wm)
		1175	{
		1176	struct drm_crtc *crtc;
		1177	int htotal, hdisplay, clock, pixel_size;
		1178	int line_time_us, line_count;
		1179	int entries, tlb_miss;
		1180
		1181	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1182	if (!intel_crtc_active(crtc)) {
3031	serge	1183	*cursor_wm = cursor->guard_size;
		1184	*plane_wm = display->guard_size;
		1185	return false;
		1186	}
		1187
		1188	htotal = crtc->mode.htotal;
		1189	hdisplay = crtc->mode.hdisplay;
		1190	clock = crtc->mode.clock;
		1191	pixel_size = crtc->fb->bits_per_pixel / 8;
		1192
		1193	/* Use the small buffer method to calculate plane watermark */
		1194	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1195	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		1196	if (tlb_miss > 0)
		1197	entries += tlb_miss;
		1198	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1199	*plane_wm = entries + display->guard_size;
		1200	if (*plane_wm > (int)display->max_wm)
		1201	*plane_wm = display->max_wm;
		1202
		1203	/* Use the large buffer method to calculate cursor watermark */
		1204	line_time_us = ((htotal * 1000) / clock);
		1205	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
		1206	entries = line_count * 64 * pixel_size;
		1207	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		1208	if (tlb_miss > 0)
		1209	entries += tlb_miss;
		1210	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1211	*cursor_wm = entries + cursor->guard_size;
		1212	if (*cursor_wm > (int)cursor->max_wm)
		1213	*cursor_wm = (int)cursor->max_wm;
		1214
		1215	return true;
		1216	}
		1217
		1218	/*
		1219	* Check the wm result.
		1220	*
		1221	* If any calculated watermark values is larger than the maximum value that
		1222	* can be programmed into the associated watermark register, that watermark
		1223	* must be disabled.
		1224	*/
		1225	static bool g4x_check_srwm(struct drm_device *dev,
		1226	int display_wm, int cursor_wm,
		1227	const struct intel_watermark_params *display,
		1228	const struct intel_watermark_params *cursor)
		1229	{
		1230	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		1231	display_wm, cursor_wm);
		1232
		1233	if (display_wm > display->max_wm) {
		1234	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		1235	display_wm, display->max_wm);
		1236	return false;
		1237	}
		1238
		1239	if (cursor_wm > cursor->max_wm) {
		1240	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		1241	cursor_wm, cursor->max_wm);
		1242	return false;
		1243	}
		1244
		1245	if (!(display_wm \|\| cursor_wm)) {
		1246	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		1247	return false;
		1248	}
		1249
		1250	return true;
		1251	}
		1252
		1253	static bool g4x_compute_srwm(struct drm_device *dev,
		1254	int plane,
		1255	int latency_ns,
		1256	const struct intel_watermark_params *display,
		1257	const struct intel_watermark_params *cursor,
		1258	int display_wm, int cursor_wm)
		1259	{
		1260	struct drm_crtc *crtc;
		1261	int hdisplay, htotal, pixel_size, clock;
		1262	unsigned long line_time_us;
		1263	int line_count, line_size;
		1264	int small, large;
		1265	int entries;
		1266
		1267	if (!latency_ns) {
		1268	display_wm = cursor_wm = 0;
		1269	return false;
		1270	}
		1271
		1272	crtc = intel_get_crtc_for_plane(dev, plane);
		1273	hdisplay = crtc->mode.hdisplay;
		1274	htotal = crtc->mode.htotal;
		1275	clock = crtc->mode.clock;
		1276	pixel_size = crtc->fb->bits_per_pixel / 8;
		1277
		1278	line_time_us = (htotal * 1000) / clock;
		1279	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1280	line_size = hdisplay * pixel_size;
		1281
		1282	/* Use the minimum of the small and large buffer method for primary */
		1283	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1284	large = line_count * line_size;
		1285
		1286	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1287	*display_wm = entries + display->guard_size;
		1288
		1289	/* calculate the self-refresh watermark for display cursor */
		1290	entries = line_count * pixel_size * 64;
		1291	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1292	*cursor_wm = entries + cursor->guard_size;
		1293
		1294	return g4x_check_srwm(dev,
		1295	display_wm, cursor_wm,
		1296	display, cursor);
		1297	}
		1298
		1299	static bool vlv_compute_drain_latency(struct drm_device *dev,
		1300	int plane,
		1301	int *plane_prec_mult,
		1302	int *plane_dl,
		1303	int *cursor_prec_mult,
		1304	int *cursor_dl)
		1305	{
		1306	struct drm_crtc *crtc;
		1307	int clock, pixel_size;
		1308	int entries;
		1309
		1310	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1311	if (!intel_crtc_active(crtc))
3031	serge	1312	return false;
		1313
		1314	clock = crtc->mode.clock; /* VESA DOT Clock */
		1315	pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */
		1316
		1317	entries = (clock / 1000) * pixel_size;
		1318	*plane_prec_mult = (entries > 256) ?
		1319	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1320	plane_dl = (64 (plane_prec_mult) 4) / ((clock / 1000) *
		1321	pixel_size);
		1322
		1323	entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
		1324	*cursor_prec_mult = (entries > 256) ?
		1325	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1326	cursor_dl = (64 (cursor_prec_mult) 4) / ((clock / 1000) * 4);
		1327
		1328	return true;
		1329	}
		1330
		1331	/*
		1332	* Update drain latency registers of memory arbiter
		1333	*
		1334	* Valleyview SoC has a new memory arbiter and needs drain latency registers
		1335	* to be programmed. Each plane has a drain latency multiplier and a drain
		1336	* latency value.
		1337	*/
		1338
		1339	static void vlv_update_drain_latency(struct drm_device *dev)
		1340	{
		1341	struct drm_i915_private *dev_priv = dev->dev_private;
		1342	int planea_prec, planea_dl, planeb_prec, planeb_dl;
		1343	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
		1344	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
		1345	either 16 or 32 */
		1346
		1347	/* For plane A, Cursor A */
		1348	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
		1349	&cursor_prec_mult, &cursora_dl)) {
		1350	cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1351	DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
		1352	planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1353	DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
		1354
		1355	I915_WRITE(VLV_DDL1, cursora_prec \|
		1356	(cursora_dl << DDL_CURSORA_SHIFT) \|
		1357	planea_prec \| planea_dl);
		1358	}
		1359
		1360	/* For plane B, Cursor B */
		1361	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
		1362	&cursor_prec_mult, &cursorb_dl)) {
		1363	cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1364	DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
		1365	planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1366	DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
		1367
		1368	I915_WRITE(VLV_DDL2, cursorb_prec \|
		1369	(cursorb_dl << DDL_CURSORB_SHIFT) \|
		1370	planeb_prec \| planeb_dl);
		1371	}
		1372	}
		1373
		1374	#define single_plane_enabled(mask) is_power_of_2(mask)
		1375
		1376	static void valleyview_update_wm(struct drm_device *dev)
		1377	{
		1378	static const int sr_latency_ns = 12000;
		1379	struct drm_i915_private *dev_priv = dev->dev_private;
		1380	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1381	int plane_sr, cursor_sr;
3243	Serge	1382	int ignore_plane_sr, ignore_cursor_sr;
3031	serge	1383	unsigned int enabled = 0;
		1384
		1385	vlv_update_drain_latency(dev);
		1386
3746	Serge	1387	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1388	&valleyview_wm_info, latency_ns,
		1389	&valleyview_cursor_wm_info, latency_ns,
		1390	&planea_wm, &cursora_wm))
3746	Serge	1391	enabled \|= 1 << PIPE_A;
3031	serge	1392
3746	Serge	1393	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1394	&valleyview_wm_info, latency_ns,
		1395	&valleyview_cursor_wm_info, latency_ns,
		1396	&planeb_wm, &cursorb_wm))
3746	Serge	1397	enabled \|= 1 << PIPE_B;
3031	serge	1398
		1399	if (single_plane_enabled(enabled) &&
		1400	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1401	sr_latency_ns,
		1402	&valleyview_wm_info,
		1403	&valleyview_cursor_wm_info,
3243	Serge	1404	&plane_sr, &ignore_cursor_sr) &&
		1405	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1406	2*sr_latency_ns,
		1407	&valleyview_wm_info,
		1408	&valleyview_cursor_wm_info,
		1409	&ignore_plane_sr, &cursor_sr)) {
3031	serge	1410	I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
3243	Serge	1411	} else {
3031	serge	1412	I915_WRITE(FW_BLC_SELF_VLV,
		1413	I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
3243	Serge	1414	plane_sr = cursor_sr = 0;
		1415	}
3031	serge	1416
		1417	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1418	planea_wm, cursora_wm,
		1419	planeb_wm, cursorb_wm,
		1420	plane_sr, cursor_sr);
		1421
		1422	I915_WRITE(DSPFW1,
		1423	(plane_sr << DSPFW_SR_SHIFT) \|
		1424	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1425	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1426	planea_wm);
		1427	I915_WRITE(DSPFW2,
3243	Serge	1428	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1429	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1430	I915_WRITE(DSPFW3,
3243	Serge	1431	(I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) \|
		1432	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
3031	serge	1433	}
		1434
		1435	static void g4x_update_wm(struct drm_device *dev)
		1436	{
		1437	static const int sr_latency_ns = 12000;
		1438	struct drm_i915_private *dev_priv = dev->dev_private;
		1439	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1440	int plane_sr, cursor_sr;
		1441	unsigned int enabled = 0;
		1442
3746	Serge	1443	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1444	&g4x_wm_info, latency_ns,
		1445	&g4x_cursor_wm_info, latency_ns,
		1446	&planea_wm, &cursora_wm))
3746	Serge	1447	enabled \|= 1 << PIPE_A;
3031	serge	1448
3746	Serge	1449	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1450	&g4x_wm_info, latency_ns,
		1451	&g4x_cursor_wm_info, latency_ns,
		1452	&planeb_wm, &cursorb_wm))
3746	Serge	1453	enabled \|= 1 << PIPE_B;
3031	serge	1454
		1455	if (single_plane_enabled(enabled) &&
		1456	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1457	sr_latency_ns,
		1458	&g4x_wm_info,
		1459	&g4x_cursor_wm_info,
3243	Serge	1460	&plane_sr, &cursor_sr)) {
3031	serge	1461	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
3243	Serge	1462	} else {
3031	serge	1463	I915_WRITE(FW_BLC_SELF,
		1464	I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
3243	Serge	1465	plane_sr = cursor_sr = 0;
		1466	}
3031	serge	1467
		1468	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1469	planea_wm, cursora_wm,
		1470	planeb_wm, cursorb_wm,
		1471	plane_sr, cursor_sr);
		1472
		1473	I915_WRITE(DSPFW1,
		1474	(plane_sr << DSPFW_SR_SHIFT) \|
		1475	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1476	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1477	planea_wm);
		1478	I915_WRITE(DSPFW2,
3243	Serge	1479	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1480	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1481	/* HPLL off in SR has some issues on G4x... disable it */
		1482	I915_WRITE(DSPFW3,
3243	Serge	1483	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
3031	serge	1484	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1485	}
		1486
		1487	static void i965_update_wm(struct drm_device *dev)
		1488	{
		1489	struct drm_i915_private *dev_priv = dev->dev_private;
		1490	struct drm_crtc *crtc;
		1491	int srwm = 1;
		1492	int cursor_sr = 16;
		1493
		1494	/* Calc sr entries for one plane configs */
		1495	crtc = single_enabled_crtc(dev);
		1496	if (crtc) {
		1497	/* self-refresh has much higher latency */
		1498	static const int sr_latency_ns = 12000;
		1499	int clock = crtc->mode.clock;
		1500	int htotal = crtc->mode.htotal;
		1501	int hdisplay = crtc->mode.hdisplay;
		1502	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1503	unsigned long line_time_us;
		1504	int entries;
		1505
		1506	line_time_us = ((htotal * 1000) / clock);
		1507
		1508	/* Use ns/us then divide to preserve precision */
		1509	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1510	pixel_size * hdisplay;
		1511	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1512	srwm = I965_FIFO_SIZE - entries;
		1513	if (srwm < 0)
		1514	srwm = 1;
		1515	srwm &= 0x1ff;
		1516	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1517	entries, srwm);
		1518
		1519	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1520	pixel_size * 64;
		1521	entries = DIV_ROUND_UP(entries,
		1522	i965_cursor_wm_info.cacheline_size);
		1523	cursor_sr = i965_cursor_wm_info.fifo_size -
		1524	(entries + i965_cursor_wm_info.guard_size);
		1525
		1526	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1527	cursor_sr = i965_cursor_wm_info.max_wm;
		1528
		1529	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1530	"cursor %d\n", srwm, cursor_sr);
		1531
		1532	if (IS_CRESTLINE(dev))
		1533	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
		1534	} else {
		1535	/* Turn off self refresh if both pipes are enabled */
		1536	if (IS_CRESTLINE(dev))
		1537	I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
		1538	& ~FW_BLC_SELF_EN);
		1539	}
		1540
		1541	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1542	srwm);
		1543
		1544	/* 965 has limitations... */
		1545	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) \|
		1546	(8 << 16) \| (8 << 8) \| (8 << 0));
		1547	I915_WRITE(DSPFW2, (8 << 8) \| (8 << 0));
		1548	/* update cursor SR watermark */
		1549	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1550	}
		1551
		1552	static void i9xx_update_wm(struct drm_device *dev)
		1553	{
		1554	struct drm_i915_private *dev_priv = dev->dev_private;
		1555	const struct intel_watermark_params *wm_info;
		1556	uint32_t fwater_lo;
		1557	uint32_t fwater_hi;
		1558	int cwm, srwm = 1;
		1559	int fifo_size;
		1560	int planea_wm, planeb_wm;
		1561	struct drm_crtc crtc, enabled = NULL;
		1562
		1563	if (IS_I945GM(dev))
		1564	wm_info = &i945_wm_info;
		1565	else if (!IS_GEN2(dev))
		1566	wm_info = &i915_wm_info;
		1567	else
		1568	wm_info = &i855_wm_info;
		1569
		1570	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1571	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1572	if (intel_crtc_active(crtc)) {
		1573	int cpp = crtc->fb->bits_per_pixel / 8;
		1574	if (IS_GEN2(dev))
		1575	cpp = 4;
		1576
3031	serge	1577	planea_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1578	wm_info, fifo_size, cpp,
3031	serge	1579	latency_ns);
		1580	enabled = crtc;
		1581	} else
		1582	planea_wm = fifo_size - wm_info->guard_size;
		1583
		1584	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1585	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1586	if (intel_crtc_active(crtc)) {
		1587	int cpp = crtc->fb->bits_per_pixel / 8;
		1588	if (IS_GEN2(dev))
		1589	cpp = 4;
		1590
3031	serge	1591	planeb_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1592	wm_info, fifo_size, cpp,
3031	serge	1593	latency_ns);
		1594	if (enabled == NULL)
		1595	enabled = crtc;
		1596	else
		1597	enabled = NULL;
		1598	} else
		1599	planeb_wm = fifo_size - wm_info->guard_size;
		1600
		1601	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1602
		1603	/*
		1604	* Overlay gets an aggressive default since video jitter is bad.
		1605	*/
		1606	cwm = 2;
		1607
		1608	/* Play safe and disable self-refresh before adjusting watermarks. */
		1609	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1610	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK \| 0);
		1611	else if (IS_I915GM(dev))
		1612	I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
		1613
		1614	/* Calc sr entries for one plane configs */
		1615	if (HAS_FW_BLC(dev) && enabled) {
		1616	/* self-refresh has much higher latency */
		1617	static const int sr_latency_ns = 6000;
		1618	int clock = enabled->mode.clock;
		1619	int htotal = enabled->mode.htotal;
		1620	int hdisplay = enabled->mode.hdisplay;
		1621	int pixel_size = enabled->fb->bits_per_pixel / 8;
		1622	unsigned long line_time_us;
		1623	int entries;
		1624
		1625	line_time_us = (htotal * 1000) / clock;
		1626
		1627	/* Use ns/us then divide to preserve precision */
		1628	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1629	pixel_size * hdisplay;
		1630	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1631	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1632	srwm = wm_info->fifo_size - entries;
		1633	if (srwm < 0)
		1634	srwm = 1;
		1635
		1636	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1637	I915_WRITE(FW_BLC_SELF,
		1638	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1639	else if (IS_I915GM(dev))
		1640	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1641	}
		1642
		1643	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1644	planea_wm, planeb_wm, cwm, srwm);
		1645
		1646	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1647	fwater_hi = (cwm & 0x1f);
		1648
		1649	/* Set request length to 8 cachelines per fetch */
		1650	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1651	fwater_hi = fwater_hi \| (1 << 8);
		1652
		1653	I915_WRITE(FW_BLC, fwater_lo);
		1654	I915_WRITE(FW_BLC2, fwater_hi);
		1655
		1656	if (HAS_FW_BLC(dev)) {
		1657	if (enabled) {
		1658	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1659	I915_WRITE(FW_BLC_SELF,
		1660	FW_BLC_SELF_EN_MASK \| FW_BLC_SELF_EN);
		1661	else if (IS_I915GM(dev))
		1662	I915_WRITE(INSTPM, I915_READ(INSTPM) \| INSTPM_SELF_EN);
		1663	DRM_DEBUG_KMS("memory self refresh enabled\n");
		1664	} else
		1665	DRM_DEBUG_KMS("memory self refresh disabled\n");
		1666	}
		1667	}
		1668
		1669	static void i830_update_wm(struct drm_device *dev)
		1670	{
		1671	struct drm_i915_private *dev_priv = dev->dev_private;
		1672	struct drm_crtc *crtc;
		1673	uint32_t fwater_lo;
		1674	int planea_wm;
		1675
		1676	crtc = single_enabled_crtc(dev);
		1677	if (crtc == NULL)
		1678	return;
		1679
		1680	planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
		1681	dev_priv->display.get_fifo_size(dev, 0),
3243	Serge	1682	4, latency_ns);
3031	serge	1683	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1684	fwater_lo \|= (3<<8) \| planea_wm;
		1685
		1686	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1687
		1688	I915_WRITE(FW_BLC, fwater_lo);
		1689	}
		1690
		1691	/*
		1692	* Check the wm result.
		1693	*
		1694	* If any calculated watermark values is larger than the maximum value that
		1695	* can be programmed into the associated watermark register, that watermark
		1696	* must be disabled.
		1697	*/
		1698	static bool ironlake_check_srwm(struct drm_device *dev, int level,
		1699	int fbc_wm, int display_wm, int cursor_wm,
		1700	const struct intel_watermark_params *display,
		1701	const struct intel_watermark_params *cursor)
		1702	{
		1703	struct drm_i915_private *dev_priv = dev->dev_private;
		1704
		1705	DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
		1706	" cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
		1707
		1708	if (fbc_wm > SNB_FBC_MAX_SRWM) {
		1709	DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
		1710	fbc_wm, SNB_FBC_MAX_SRWM, level);
		1711
		1712	/* fbc has it's own way to disable FBC WM */
		1713	I915_WRITE(DISP_ARB_CTL,
		1714	I915_READ(DISP_ARB_CTL) \| DISP_FBC_WM_DIS);
		1715	return false;
4104	Serge	1716	} else if (INTEL_INFO(dev)->gen >= 6) {
		1717	/* enable FBC WM (except on ILK, where it must remain off) */
		1718	I915_WRITE(DISP_ARB_CTL,
		1719	I915_READ(DISP_ARB_CTL) & ~DISP_FBC_WM_DIS);
3031	serge	1720	}
		1721
		1722	if (display_wm > display->max_wm) {
		1723	DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
		1724	display_wm, SNB_DISPLAY_MAX_SRWM, level);
		1725	return false;
		1726	}
		1727
		1728	if (cursor_wm > cursor->max_wm) {
		1729	DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
		1730	cursor_wm, SNB_CURSOR_MAX_SRWM, level);
		1731	return false;
		1732	}
		1733
		1734	if (!(fbc_wm \|\| display_wm \|\| cursor_wm)) {
		1735	DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
		1736	return false;
		1737	}
		1738
		1739	return true;
		1740	}
		1741
		1742	/*
		1743	* Compute watermark values of WM[1-3],
		1744	*/
		1745	static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
		1746	int latency_ns,
		1747	const struct intel_watermark_params *display,
		1748	const struct intel_watermark_params *cursor,
		1749	int fbc_wm, int display_wm, int *cursor_wm)
		1750	{
		1751	struct drm_crtc *crtc;
		1752	unsigned long line_time_us;
		1753	int hdisplay, htotal, pixel_size, clock;
		1754	int line_count, line_size;
		1755	int small, large;
		1756	int entries;
		1757
		1758	if (!latency_ns) {
		1759	fbc_wm = display_wm = *cursor_wm = 0;
		1760	return false;
		1761	}
		1762
		1763	crtc = intel_get_crtc_for_plane(dev, plane);
		1764	hdisplay = crtc->mode.hdisplay;
		1765	htotal = crtc->mode.htotal;
		1766	clock = crtc->mode.clock;
		1767	pixel_size = crtc->fb->bits_per_pixel / 8;
		1768
		1769	line_time_us = (htotal * 1000) / clock;
		1770	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1771	line_size = hdisplay * pixel_size;
		1772
		1773	/* Use the minimum of the small and large buffer method for primary */
		1774	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1775	large = line_count * line_size;
		1776
		1777	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1778	*display_wm = entries + display->guard_size;
		1779
		1780	/*
		1781	* Spec says:
		1782	* FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
		1783	*/
		1784	fbc_wm = DIV_ROUND_UP(display_wm * 64, line_size) + 2;
		1785
		1786	/* calculate the self-refresh watermark for display cursor */
		1787	entries = line_count * pixel_size * 64;
		1788	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1789	*cursor_wm = entries + cursor->guard_size;
		1790
		1791	return ironlake_check_srwm(dev, level,
		1792	fbc_wm, display_wm, *cursor_wm,
		1793	display, cursor);
		1794	}
		1795
		1796	static void ironlake_update_wm(struct drm_device *dev)
		1797	{
		1798	struct drm_i915_private *dev_priv = dev->dev_private;
		1799	int fbc_wm, plane_wm, cursor_wm;
		1800	unsigned int enabled;
		1801
		1802	enabled = 0;
3746	Serge	1803	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1804	&ironlake_display_wm_info,
4104	Serge	1805	dev_priv->wm.pri_latency[0] * 100,
3031	serge	1806	&ironlake_cursor_wm_info,
4104	Serge	1807	dev_priv->wm.cur_latency[0] * 100,
3031	serge	1808	&plane_wm, &cursor_wm)) {
		1809	I915_WRITE(WM0_PIPEA_ILK,
		1810	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1811	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1812	" plane %d, " "cursor: %d\n",
		1813	plane_wm, cursor_wm);
3746	Serge	1814	enabled \|= 1 << PIPE_A;
3031	serge	1815	}
		1816
3746	Serge	1817	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1818	&ironlake_display_wm_info,
4104	Serge	1819	dev_priv->wm.pri_latency[0] * 100,
3031	serge	1820	&ironlake_cursor_wm_info,
4104	Serge	1821	dev_priv->wm.cur_latency[0] * 100,
3031	serge	1822	&plane_wm, &cursor_wm)) {
		1823	I915_WRITE(WM0_PIPEB_ILK,
		1824	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1825	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1826	" plane %d, cursor: %d\n",
		1827	plane_wm, cursor_wm);
3746	Serge	1828	enabled \|= 1 << PIPE_B;
3031	serge	1829	}
		1830
		1831	/*
		1832	* Calculate and update the self-refresh watermark only when one
		1833	* display plane is used.
		1834	*/
		1835	I915_WRITE(WM3_LP_ILK, 0);
		1836	I915_WRITE(WM2_LP_ILK, 0);
		1837	I915_WRITE(WM1_LP_ILK, 0);
		1838
		1839	if (!single_plane_enabled(enabled))
		1840	return;
		1841	enabled = ffs(enabled) - 1;
		1842
		1843	/* WM1 */
		1844	if (!ironlake_compute_srwm(dev, 1, enabled,
4104	Serge	1845	dev_priv->wm.pri_latency[1] * 500,
3031	serge	1846	&ironlake_display_srwm_info,
		1847	&ironlake_cursor_srwm_info,
		1848	&fbc_wm, &plane_wm, &cursor_wm))
		1849	return;
		1850
		1851	I915_WRITE(WM1_LP_ILK,
		1852	WM1_LP_SR_EN \|
4104	Serge	1853	(dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) \|
3031	serge	1854	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1855	(plane_wm << WM1_LP_SR_SHIFT) \|
		1856	cursor_wm);
		1857
		1858	/* WM2 */
		1859	if (!ironlake_compute_srwm(dev, 2, enabled,
4104	Serge	1860	dev_priv->wm.pri_latency[2] * 500,
3031	serge	1861	&ironlake_display_srwm_info,
		1862	&ironlake_cursor_srwm_info,
		1863	&fbc_wm, &plane_wm, &cursor_wm))
		1864	return;
		1865
		1866	I915_WRITE(WM2_LP_ILK,
		1867	WM2_LP_EN \|
4104	Serge	1868	(dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) \|
3031	serge	1869	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1870	(plane_wm << WM1_LP_SR_SHIFT) \|
		1871	cursor_wm);
		1872
		1873	/*
		1874	* WM3 is unsupported on ILK, probably because we don't have latency
		1875	* data for that power state
		1876	*/
		1877	}
		1878
		1879	static void sandybridge_update_wm(struct drm_device *dev)
		1880	{
		1881	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	1882	int latency = dev_priv->wm.pri_latency[0] * 100; /* In unit 0.1us */
3031	serge	1883	u32 val;
		1884	int fbc_wm, plane_wm, cursor_wm;
		1885	unsigned int enabled;
		1886
		1887	enabled = 0;
3746	Serge	1888	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1889	&sandybridge_display_wm_info, latency,
		1890	&sandybridge_cursor_wm_info, latency,
		1891	&plane_wm, &cursor_wm)) {
		1892	val = I915_READ(WM0_PIPEA_ILK);
		1893	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1894	I915_WRITE(WM0_PIPEA_ILK, val \|
		1895	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1896	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1897	" plane %d, " "cursor: %d\n",
		1898	plane_wm, cursor_wm);
3746	Serge	1899	enabled \|= 1 << PIPE_A;
3031	serge	1900	}
		1901
3746	Serge	1902	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1903	&sandybridge_display_wm_info, latency,
		1904	&sandybridge_cursor_wm_info, latency,
		1905	&plane_wm, &cursor_wm)) {
		1906	val = I915_READ(WM0_PIPEB_ILK);
		1907	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1908	I915_WRITE(WM0_PIPEB_ILK, val \|
		1909	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1910	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1911	" plane %d, cursor: %d\n",
		1912	plane_wm, cursor_wm);
3746	Serge	1913	enabled \|= 1 << PIPE_B;
3031	serge	1914	}
		1915
3243	Serge	1916	/*
		1917	* Calculate and update the self-refresh watermark only when one
		1918	* display plane is used.
		1919	*
		1920	* SNB support 3 levels of watermark.
		1921	*
		1922	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1923	* and disabled in the descending order
		1924	*
		1925	*/
		1926	I915_WRITE(WM3_LP_ILK, 0);
		1927	I915_WRITE(WM2_LP_ILK, 0);
		1928	I915_WRITE(WM1_LP_ILK, 0);
		1929
		1930	if (!single_plane_enabled(enabled) \|\|
		1931	dev_priv->sprite_scaling_enabled)
		1932	return;
		1933	enabled = ffs(enabled) - 1;
		1934
		1935	/* WM1 */
		1936	if (!ironlake_compute_srwm(dev, 1, enabled,
4104	Serge	1937	dev_priv->wm.pri_latency[1] * 500,
3243	Serge	1938	&sandybridge_display_srwm_info,
		1939	&sandybridge_cursor_srwm_info,
		1940	&fbc_wm, &plane_wm, &cursor_wm))
		1941	return;
		1942
		1943	I915_WRITE(WM1_LP_ILK,
		1944	WM1_LP_SR_EN \|
4104	Serge	1945	(dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) \|
3243	Serge	1946	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1947	(plane_wm << WM1_LP_SR_SHIFT) \|
		1948	cursor_wm);
		1949
		1950	/* WM2 */
		1951	if (!ironlake_compute_srwm(dev, 2, enabled,
4104	Serge	1952	dev_priv->wm.pri_latency[2] * 500,
3243	Serge	1953	&sandybridge_display_srwm_info,
		1954	&sandybridge_cursor_srwm_info,
		1955	&fbc_wm, &plane_wm, &cursor_wm))
		1956	return;
		1957
		1958	I915_WRITE(WM2_LP_ILK,
		1959	WM2_LP_EN \|
4104	Serge	1960	(dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) \|
3243	Serge	1961	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1962	(plane_wm << WM1_LP_SR_SHIFT) \|
		1963	cursor_wm);
		1964
		1965	/* WM3 */
		1966	if (!ironlake_compute_srwm(dev, 3, enabled,
4104	Serge	1967	dev_priv->wm.pri_latency[3] * 500,
3243	Serge	1968	&sandybridge_display_srwm_info,
		1969	&sandybridge_cursor_srwm_info,
		1970	&fbc_wm, &plane_wm, &cursor_wm))
		1971	return;
		1972
		1973	I915_WRITE(WM3_LP_ILK,
		1974	WM3_LP_EN \|
4104	Serge	1975	(dev_priv->wm.pri_latency[3] << WM1_LP_LATENCY_SHIFT) \|
3243	Serge	1976	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1977	(plane_wm << WM1_LP_SR_SHIFT) \|
		1978	cursor_wm);
		1979	}
		1980
		1981	static void ivybridge_update_wm(struct drm_device *dev)
		1982	{
		1983	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	1984	int latency = dev_priv->wm.pri_latency[0] * 100; /* In unit 0.1us */
3243	Serge	1985	u32 val;
		1986	int fbc_wm, plane_wm, cursor_wm;
		1987	int ignore_fbc_wm, ignore_plane_wm, ignore_cursor_wm;
		1988	unsigned int enabled;
		1989
		1990	enabled = 0;
3746	Serge	1991	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1992	&sandybridge_display_wm_info, latency,
		1993	&sandybridge_cursor_wm_info, latency,
		1994	&plane_wm, &cursor_wm)) {
3243	Serge	1995	val = I915_READ(WM0_PIPEA_ILK);
		1996	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1997	I915_WRITE(WM0_PIPEA_ILK, val \|
		1998	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1999	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		2000	" plane %d, " "cursor: %d\n",
		2001	plane_wm, cursor_wm);
3746	Serge	2002	enabled \|= 1 << PIPE_A;
3243	Serge	2003	}
		2004
3746	Serge	2005	if (g4x_compute_wm0(dev, PIPE_B,
3243	Serge	2006	&sandybridge_display_wm_info, latency,
		2007	&sandybridge_cursor_wm_info, latency,
		2008	&plane_wm, &cursor_wm)) {
		2009	val = I915_READ(WM0_PIPEB_ILK);
		2010	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		2011	I915_WRITE(WM0_PIPEB_ILK, val \|
		2012	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		2013	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		2014	" plane %d, cursor: %d\n",
		2015	plane_wm, cursor_wm);
3746	Serge	2016	enabled \|= 1 << PIPE_B;
3243	Serge	2017	}
		2018
3746	Serge	2019	if (g4x_compute_wm0(dev, PIPE_C,
3243	Serge	2020	&sandybridge_display_wm_info, latency,
		2021	&sandybridge_cursor_wm_info, latency,
		2022	&plane_wm, &cursor_wm)) {
3031	serge	2023	val = I915_READ(WM0_PIPEC_IVB);
		2024	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		2025	I915_WRITE(WM0_PIPEC_IVB, val \|
		2026	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		2027	DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
		2028	" plane %d, cursor: %d\n",
		2029	plane_wm, cursor_wm);
3746	Serge	2030	enabled \|= 1 << PIPE_C;
3031	serge	2031	}
		2032
		2033	/*
		2034	* Calculate and update the self-refresh watermark only when one
		2035	* display plane is used.
		2036	*
		2037	* SNB support 3 levels of watermark.
		2038	*
		2039	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		2040	* and disabled in the descending order
		2041	*
		2042	*/
		2043	I915_WRITE(WM3_LP_ILK, 0);
		2044	I915_WRITE(WM2_LP_ILK, 0);
		2045	I915_WRITE(WM1_LP_ILK, 0);
		2046
		2047	if (!single_plane_enabled(enabled) \|\|
		2048	dev_priv->sprite_scaling_enabled)
		2049	return;
		2050	enabled = ffs(enabled) - 1;
		2051
		2052	/* WM1 */
		2053	if (!ironlake_compute_srwm(dev, 1, enabled,
4104	Serge	2054	dev_priv->wm.pri_latency[1] * 500,
3031	serge	2055	&sandybridge_display_srwm_info,
		2056	&sandybridge_cursor_srwm_info,
		2057	&fbc_wm, &plane_wm, &cursor_wm))
		2058	return;
		2059
		2060	I915_WRITE(WM1_LP_ILK,
		2061	WM1_LP_SR_EN \|
4104	Serge	2062	(dev_priv->wm.pri_latency[1] << WM1_LP_LATENCY_SHIFT) \|
3031	serge	2063	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2064	(plane_wm << WM1_LP_SR_SHIFT) \|
		2065	cursor_wm);
		2066
		2067	/* WM2 */
		2068	if (!ironlake_compute_srwm(dev, 2, enabled,
4104	Serge	2069	dev_priv->wm.pri_latency[2] * 500,
3031	serge	2070	&sandybridge_display_srwm_info,
		2071	&sandybridge_cursor_srwm_info,
		2072	&fbc_wm, &plane_wm, &cursor_wm))
		2073	return;
		2074
		2075	I915_WRITE(WM2_LP_ILK,
		2076	WM2_LP_EN \|
4104	Serge	2077	(dev_priv->wm.pri_latency[2] << WM1_LP_LATENCY_SHIFT) \|
3031	serge	2078	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2079	(plane_wm << WM1_LP_SR_SHIFT) \|
		2080	cursor_wm);
		2081
3243	Serge	2082	/* WM3, note we have to correct the cursor latency */
3031	serge	2083	if (!ironlake_compute_srwm(dev, 3, enabled,
4104	Serge	2084	dev_priv->wm.pri_latency[3] * 500,
3031	serge	2085	&sandybridge_display_srwm_info,
		2086	&sandybridge_cursor_srwm_info,
3243	Serge	2087	&fbc_wm, &plane_wm, &ignore_cursor_wm) \|\|
		2088	!ironlake_compute_srwm(dev, 3, enabled,
4104	Serge	2089	dev_priv->wm.cur_latency[3] * 500,
3243	Serge	2090	&sandybridge_display_srwm_info,
		2091	&sandybridge_cursor_srwm_info,
		2092	&ignore_fbc_wm, &ignore_plane_wm, &cursor_wm))
3031	serge	2093	return;
		2094
		2095	I915_WRITE(WM3_LP_ILK,
		2096	WM3_LP_EN \|
4104	Serge	2097	(dev_priv->wm.pri_latency[3] << WM1_LP_LATENCY_SHIFT) \|
3031	serge	2098	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2099	(plane_wm << WM1_LP_SR_SHIFT) \|
		2100	cursor_wm);
		2101	}
		2102
4104	Serge	2103	static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
		2104	struct drm_crtc *crtc)
3031	serge	2105	{
4104	Serge	2106	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2107	uint32_t pixel_rate;
		2108
		2109	pixel_rate = intel_crtc->config.adjusted_mode.clock;
		2110
		2111	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
		2112	* adjust the pixel_rate here. */
		2113
		2114	if (intel_crtc->config.pch_pfit.enabled) {
		2115	uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
		2116	uint32_t pfit_size = intel_crtc->config.pch_pfit.size;
		2117
		2118	pipe_w = intel_crtc->config.requested_mode.hdisplay;
		2119	pipe_h = intel_crtc->config.requested_mode.vdisplay;
		2120	pfit_w = (pfit_size >> 16) & 0xFFFF;
		2121	pfit_h = pfit_size & 0xFFFF;
		2122	if (pipe_w < pfit_w)
		2123	pipe_w = pfit_w;
		2124	if (pipe_h < pfit_h)
		2125	pipe_h = pfit_h;
		2126
		2127	pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
		2128	pfit_w * pfit_h);
		2129	}
		2130
		2131	return pixel_rate;
		2132	}
		2133
		2134	/* latency must be in 0.1us units. */
		2135	static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		2136	uint32_t latency)
		2137	{
		2138	uint64_t ret;
		2139
		2140	if (WARN(latency == 0, "Latency value missing\n"))
		2141	return UINT_MAX;
		2142
		2143	ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
		2144	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
		2145
		2146	return ret;
		2147	}
		2148
		2149	/* latency must be in 0.1us units. */
		2150	static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		2151	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
		2152	uint32_t latency)
		2153	{
		2154	uint32_t ret;
		2155
		2156	if (WARN(latency == 0, "Latency value missing\n"))
		2157	return UINT_MAX;
		2158
		2159	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		2160	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		2161	ret = DIV_ROUND_UP(ret, 64) + 2;
		2162	return ret;
		2163	}
		2164
		2165	static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
		2166	uint8_t bytes_per_pixel)
		2167	{
		2168	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
		2169	}
		2170
		2171	struct hsw_pipe_wm_parameters {
		2172	bool active;
		2173	uint32_t pipe_htotal;
		2174	uint32_t pixel_rate;
		2175	struct intel_plane_wm_parameters pri;
		2176	struct intel_plane_wm_parameters spr;
		2177	struct intel_plane_wm_parameters cur;
		2178	};
		2179
		2180	struct hsw_wm_maximums {
		2181	uint16_t pri;
		2182	uint16_t spr;
		2183	uint16_t cur;
		2184	uint16_t fbc;
		2185	};
		2186
		2187	struct hsw_wm_values {
		2188	uint32_t wm_pipe[3];
		2189	uint32_t wm_lp[3];
		2190	uint32_t wm_lp_spr[3];
		2191	uint32_t wm_linetime[3];
		2192	bool enable_fbc_wm;
		2193	};
		2194
		2195	/* used in computing the new watermarks state */
		2196	struct intel_wm_config {
		2197	unsigned int num_pipes_active;
		2198	bool sprites_enabled;
		2199	bool sprites_scaled;
		2200	bool fbc_wm_enabled;
		2201	};
		2202
		2203	/*
		2204	* For both WM_PIPE and WM_LP.
		2205	* mem_value must be in 0.1us units.
		2206	*/
		2207	static uint32_t ilk_compute_pri_wm(struct hsw_pipe_wm_parameters *params,
		2208	uint32_t mem_value,
		2209	bool is_lp)
		2210	{
		2211	uint32_t method1, method2;
		2212
		2213	if (!params->active \|\| !params->pri.enabled)
		2214	return 0;
		2215
		2216	method1 = ilk_wm_method1(params->pixel_rate,
		2217	params->pri.bytes_per_pixel,
		2218	mem_value);
		2219
		2220	if (!is_lp)
		2221	return method1;
		2222
		2223	method2 = ilk_wm_method2(params->pixel_rate,
		2224	params->pipe_htotal,
		2225	params->pri.horiz_pixels,
		2226	params->pri.bytes_per_pixel,
		2227	mem_value);
		2228
		2229	return min(method1, method2);
		2230	}
		2231
		2232	/*
		2233	* For both WM_PIPE and WM_LP.
		2234	* mem_value must be in 0.1us units.
		2235	*/
		2236	static uint32_t ilk_compute_spr_wm(struct hsw_pipe_wm_parameters *params,
		2237	uint32_t mem_value)
		2238	{
		2239	uint32_t method1, method2;
		2240
		2241	if (!params->active \|\| !params->spr.enabled)
		2242	return 0;
		2243
		2244	method1 = ilk_wm_method1(params->pixel_rate,
		2245	params->spr.bytes_per_pixel,
		2246	mem_value);
		2247	method2 = ilk_wm_method2(params->pixel_rate,
		2248	params->pipe_htotal,
		2249	params->spr.horiz_pixels,
		2250	params->spr.bytes_per_pixel,
		2251	mem_value);
		2252	return min(method1, method2);
		2253	}
		2254
		2255	/*
		2256	* For both WM_PIPE and WM_LP.
		2257	* mem_value must be in 0.1us units.
		2258	*/
		2259	static uint32_t ilk_compute_cur_wm(struct hsw_pipe_wm_parameters *params,
		2260	uint32_t mem_value)
		2261	{
		2262	if (!params->active \|\| !params->cur.enabled)
		2263	return 0;
		2264
		2265	return ilk_wm_method2(params->pixel_rate,
		2266	params->pipe_htotal,
		2267	params->cur.horiz_pixels,
		2268	params->cur.bytes_per_pixel,
		2269	mem_value);
		2270	}
		2271
		2272	/* Only for WM_LP. */
		2273	static uint32_t ilk_compute_fbc_wm(struct hsw_pipe_wm_parameters *params,
		2274	uint32_t pri_val)
		2275	{
		2276	if (!params->active \|\| !params->pri.enabled)
		2277	return 0;
		2278
		2279	return ilk_wm_fbc(pri_val,
		2280	params->pri.horiz_pixels,
		2281	params->pri.bytes_per_pixel);
		2282	}
		2283
		2284	static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
		2285	{
		2286	if (INTEL_INFO(dev)->gen >= 7)
		2287	return 768;
		2288	else
		2289	return 512;
		2290	}
		2291
		2292	/* Calculate the maximum primary/sprite plane watermark */
		2293	static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
		2294	int level,
		2295	const struct intel_wm_config *config,
		2296	enum intel_ddb_partitioning ddb_partitioning,
		2297	bool is_sprite)
		2298	{
		2299	unsigned int fifo_size = ilk_display_fifo_size(dev);
		2300	unsigned int max;
		2301
		2302	/* if sprites aren't enabled, sprites get nothing */
		2303	if (is_sprite && !config->sprites_enabled)
		2304	return 0;
		2305
		2306	/* HSW allows LP1+ watermarks even with multiple pipes */
		2307	if (level == 0 \|\| config->num_pipes_active > 1) {
		2308	fifo_size /= INTEL_INFO(dev)->num_pipes;
		2309
		2310	/*
		2311	* For some reason the non self refresh
		2312	* FIFO size is only half of the self
		2313	* refresh FIFO size on ILK/SNB.
		2314	*/
		2315	if (INTEL_INFO(dev)->gen <= 6)
		2316	fifo_size /= 2;
		2317	}
		2318
		2319	if (config->sprites_enabled) {
		2320	/* level 0 is always calculated with 1:1 split */
		2321	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
		2322	if (is_sprite)
		2323	fifo_size *= 5;
		2324	fifo_size /= 6;
		2325	} else {
		2326	fifo_size /= 2;
		2327	}
		2328	}
		2329
		2330	/* clamp to max that the registers can hold */
		2331	if (INTEL_INFO(dev)->gen >= 7)
		2332	/* IVB/HSW primary/sprite plane watermarks */
		2333	max = level == 0 ? 127 : 1023;
		2334	else if (!is_sprite)
		2335	/* ILK/SNB primary plane watermarks */
		2336	max = level == 0 ? 127 : 511;
		2337	else
		2338	/* ILK/SNB sprite plane watermarks */
		2339	max = level == 0 ? 63 : 255;
		2340
		2341	return min(fifo_size, max);
		2342	}
		2343
		2344	/* Calculate the maximum cursor plane watermark */
		2345	static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
		2346	int level,
		2347	const struct intel_wm_config *config)
		2348	{
		2349	/* HSW LP1+ watermarks w/ multiple pipes */
		2350	if (level > 0 && config->num_pipes_active > 1)
		2351	return 64;
		2352
		2353	/* otherwise just report max that registers can hold */
		2354	if (INTEL_INFO(dev)->gen >= 7)
		2355	return level == 0 ? 63 : 255;
		2356	else
		2357	return level == 0 ? 31 : 63;
4539	Serge	2358	}
4104	Serge	2359
		2360	/* Calculate the maximum FBC watermark */
		2361	static unsigned int ilk_fbc_wm_max(void)
		2362	{
		2363	/* max that registers can hold */
		2364	return 15;
		2365	}
		2366
		2367	static void ilk_wm_max(struct drm_device *dev,
		2368	int level,
		2369	const struct intel_wm_config *config,
		2370	enum intel_ddb_partitioning ddb_partitioning,
		2371	struct hsw_wm_maximums *max)
		2372	{
		2373	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
		2374	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
		2375	max->cur = ilk_cursor_wm_max(dev, level, config);
		2376	max->fbc = ilk_fbc_wm_max();
4539	Serge	2377	}
4104	Serge	2378
		2379	static bool ilk_check_wm(int level,
		2380	const struct hsw_wm_maximums *max,
		2381	struct intel_wm_level *result)
		2382	{
		2383	bool ret;
		2384
		2385	/* already determined to be invalid? */
		2386	if (!result->enable)
		2387	return false;
		2388
		2389	result->enable = result->pri_val <= max->pri &&
		2390	result->spr_val <= max->spr &&
		2391	result->cur_val <= max->cur;
		2392
		2393	ret = result->enable;
		2394
		2395	/*
		2396	* HACK until we can pre-compute everything,
		2397	* and thus fail gracefully if LP0 watermarks
		2398	* are exceeded...
		2399	*/
		2400	if (level == 0 && !result->enable) {
		2401	if (result->pri_val > max->pri)
		2402	DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
		2403	level, result->pri_val, max->pri);
		2404	if (result->spr_val > max->spr)
		2405	DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
		2406	level, result->spr_val, max->spr);
		2407	if (result->cur_val > max->cur)
		2408	DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
		2409	level, result->cur_val, max->cur);
		2410
		2411	result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
		2412	result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
		2413	result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
		2414	result->enable = true;
		2415	}
		2416
		2417	DRM_DEBUG_KMS("WM%d: %sabled\n", level, result->enable ? "en" : "dis");
		2418
		2419	return ret;
		2420	}
		2421
		2422	static void ilk_compute_wm_level(struct drm_i915_private *dev_priv,
		2423	int level,
		2424	struct hsw_pipe_wm_parameters *p,
		2425	struct intel_wm_level *result)
		2426	{
		2427	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
		2428	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
		2429	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
		2430
		2431	/* WM1+ latency values stored in 0.5us units */
		2432	if (level > 0) {
		2433	pri_latency *= 5;
		2434	spr_latency *= 5;
		2435	cur_latency *= 5;
		2436	}
		2437
		2438	result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
		2439	result->spr_val = ilk_compute_spr_wm(p, spr_latency);
		2440	result->cur_val = ilk_compute_cur_wm(p, cur_latency);
		2441	result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
		2442	result->enable = true;
		2443	}
		2444
		2445	static bool hsw_compute_lp_wm(struct drm_i915_private *dev_priv,
		2446	int level, struct hsw_wm_maximums *max,
		2447	struct hsw_pipe_wm_parameters *params,
		2448	struct intel_wm_level *result)
		2449	{
		2450	enum pipe pipe;
		2451	struct intel_wm_level res[3];
		2452
		2453	for (pipe = PIPE_A; pipe <= PIPE_C; pipe++)
		2454	ilk_compute_wm_level(dev_priv, level, ¶ms[pipe], &res[pipe]);
		2455
		2456	result->pri_val = max3(res[0].pri_val, res[1].pri_val, res[2].pri_val);
		2457	result->spr_val = max3(res[0].spr_val, res[1].spr_val, res[2].spr_val);
		2458	result->cur_val = max3(res[0].cur_val, res[1].cur_val, res[2].cur_val);
		2459	result->fbc_val = max3(res[0].fbc_val, res[1].fbc_val, res[2].fbc_val);
		2460	result->enable = true;
		2461
		2462	return ilk_check_wm(level, max, result);
		2463	}
		2464
		2465	static uint32_t hsw_compute_wm_pipe(struct drm_i915_private *dev_priv,
		2466	enum pipe pipe,
		2467	struct hsw_pipe_wm_parameters *params)
		2468	{
		2469	uint32_t pri_val, cur_val, spr_val;
		2470	/* WM0 latency values stored in 0.1us units */
		2471	uint16_t pri_latency = dev_priv->wm.pri_latency[0];
		2472	uint16_t spr_latency = dev_priv->wm.spr_latency[0];
		2473	uint16_t cur_latency = dev_priv->wm.cur_latency[0];
		2474
		2475	pri_val = ilk_compute_pri_wm(params, pri_latency, false);
		2476	spr_val = ilk_compute_spr_wm(params, spr_latency);
		2477	cur_val = ilk_compute_cur_wm(params, cur_latency);
		2478
		2479	WARN(pri_val > 127,
		2480	"Primary WM error, mode not supported for pipe %c\n",
		2481	pipe_name(pipe));
		2482	WARN(spr_val > 127,
		2483	"Sprite WM error, mode not supported for pipe %c\n",
		2484	pipe_name(pipe));
		2485	WARN(cur_val > 63,
		2486	"Cursor WM error, mode not supported for pipe %c\n",
		2487	pipe_name(pipe));
		2488
		2489	return (pri_val << WM0_PIPE_PLANE_SHIFT) \|
		2490	(spr_val << WM0_PIPE_SPRITE_SHIFT) \|
		2491	cur_val;
		2492	}
		2493
		2494	static uint32_t
		2495	hsw_compute_linetime_wm(struct drm_device dev, struct drm_crtc crtc)
		2496	{
3031	serge	2497	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2498	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2499	struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
		2500	u32 linetime, ips_linetime;
3031	serge	2501
4104	Serge	2502	if (!intel_crtc_active(crtc))
		2503	return 0;
3031	serge	2504
		2505	/* The WM are computed with base on how long it takes to fill a single
		2506	* row at the given clock rate, multiplied by 8.
		2507	* */
4104	Serge	2508	linetime = DIV_ROUND_CLOSEST(mode->htotal * 1000 * 8, mode->clock);
		2509	ips_linetime = DIV_ROUND_CLOSEST(mode->htotal * 1000 * 8,
		2510	intel_ddi_get_cdclk_freq(dev_priv));
3031	serge	2511
4104	Serge	2512	return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) \|
		2513	PIPE_WM_LINETIME_TIME(linetime);
		2514	}
		2515
		2516	static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2517	{
		2518	struct drm_i915_private *dev_priv = dev->dev_private;
		2519
		2520	if (IS_HASWELL(dev)) {
		2521	uint64_t sskpd = I915_READ64(MCH_SSKPD);
		2522
		2523	wm[0] = (sskpd >> 56) & 0xFF;
		2524	if (wm[0] == 0)
		2525	wm[0] = sskpd & 0xF;
		2526	wm[1] = (sskpd >> 4) & 0xFF;
		2527	wm[2] = (sskpd >> 12) & 0xFF;
		2528	wm[3] = (sskpd >> 20) & 0x1FF;
		2529	wm[4] = (sskpd >> 32) & 0x1FF;
		2530	} else if (INTEL_INFO(dev)->gen >= 6) {
		2531	uint32_t sskpd = I915_READ(MCH_SSKPD);
		2532
		2533	wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		2534	wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		2535	wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		2536	wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
		2537	} else if (INTEL_INFO(dev)->gen >= 5) {
		2538	uint32_t mltr = I915_READ(MLTR_ILK);
		2539
		2540	/* ILK primary LP0 latency is 700 ns */
		2541	wm[0] = 7;
		2542	wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		2543	wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
		2544	}
		2545	}
		2546
		2547	static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2548	{
		2549	/* ILK sprite LP0 latency is 1300 ns */
		2550	if (INTEL_INFO(dev)->gen == 5)
		2551	wm[0] = 13;
		2552	}
		2553
		2554	static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2555	{
		2556	/* ILK cursor LP0 latency is 1300 ns */
		2557	if (INTEL_INFO(dev)->gen == 5)
		2558	wm[0] = 13;
		2559
		2560	/* WaDoubleCursorLP3Latency:ivb */
		2561	if (IS_IVYBRIDGE(dev))
		2562	wm[3] *= 2;
		2563	}
		2564
		2565	static void intel_print_wm_latency(struct drm_device *dev,
		2566	const char *name,
		2567	const uint16_t wm[5])
		2568	{
		2569	int level, max_level;
		2570
		2571	/* how many WM levels are we expecting */
		2572	if (IS_HASWELL(dev))
		2573	max_level = 4;
		2574	else if (INTEL_INFO(dev)->gen >= 6)
		2575	max_level = 3;
		2576	else
		2577	max_level = 2;
		2578
		2579	for (level = 0; level <= max_level; level++) {
		2580	unsigned int latency = wm[level];
		2581
		2582	if (latency == 0) {
		2583	DRM_ERROR("%s WM%d latency not provided\n",
		2584	name, level);
		2585	continue;
		2586	}
		2587
		2588	/* WM1+ latency values in 0.5us units */
		2589	if (level > 0)
		2590	latency *= 5;
		2591
		2592	DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
		2593	name, level, wm[level],
		2594	latency / 10, latency % 10);
		2595	}
		2596	}
		2597
		2598	static void intel_setup_wm_latency(struct drm_device *dev)
		2599	{
		2600	struct drm_i915_private *dev_priv = dev->dev_private;
		2601
		2602	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
		2603
		2604	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
		2605	sizeof(dev_priv->wm.pri_latency));
		2606	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
		2607	sizeof(dev_priv->wm.pri_latency));
		2608
		2609	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
		2610	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
		2611
		2612	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2613	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2614	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
		2615	}
		2616
		2617	static void hsw_compute_wm_parameters(struct drm_device *dev,
		2618	struct hsw_pipe_wm_parameters *params,
		2619	struct hsw_wm_maximums *lp_max_1_2,
		2620	struct hsw_wm_maximums *lp_max_5_6)
		2621	{
		2622	struct drm_crtc *crtc;
		2623	struct drm_plane *plane;
		2624	enum pipe pipe;
		2625	struct intel_wm_config config = {};
		2626
		2627	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
		2628	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2629	struct hsw_pipe_wm_parameters *p;
		2630
		2631	pipe = intel_crtc->pipe;
		2632	p = ¶ms[pipe];
		2633
		2634	p->active = intel_crtc_active(crtc);
		2635	if (!p->active)
		2636	continue;
		2637
		2638	config.num_pipes_active++;
		2639
		2640	p->pipe_htotal = intel_crtc->config.adjusted_mode.htotal;
		2641	p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
		2642	p->pri.bytes_per_pixel = crtc->fb->bits_per_pixel / 8;
		2643	p->cur.bytes_per_pixel = 4;
		2644	p->pri.horiz_pixels =
		2645	intel_crtc->config.requested_mode.hdisplay;
		2646	p->cur.horiz_pixels = 64;
		2647	/* TODO: for now, assume primary and cursor planes are always enabled. */
		2648	p->pri.enabled = true;
		2649	p->cur.enabled = true;
		2650	}
		2651
		2652	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		2653	struct intel_plane *intel_plane = to_intel_plane(plane);
		2654	struct hsw_pipe_wm_parameters *p;
		2655
		2656	pipe = intel_plane->pipe;
		2657	p = ¶ms[pipe];
		2658
		2659	p->spr = intel_plane->wm;
		2660
		2661	config.sprites_enabled \|= p->spr.enabled;
		2662	config.sprites_scaled \|= p->spr.scaled;
		2663	}
		2664
		2665	ilk_wm_max(dev, 1, &config, INTEL_DDB_PART_1_2, lp_max_1_2);
		2666
		2667	/* 5/6 split only in single pipe config on IVB+ */
		2668	if (INTEL_INFO(dev)->gen >= 7 && config.num_pipes_active <= 1)
		2669	ilk_wm_max(dev, 1, &config, INTEL_DDB_PART_5_6, lp_max_5_6);
		2670	else
		2671	lp_max_5_6 = lp_max_1_2;
		2672	}
		2673
		2674	static void hsw_compute_wm_results(struct drm_device *dev,
		2675	struct hsw_pipe_wm_parameters *params,
		2676	struct hsw_wm_maximums *lp_maximums,
		2677	struct hsw_wm_values *results)
		2678	{
		2679	struct drm_i915_private *dev_priv = dev->dev_private;
		2680	struct drm_crtc *crtc;
		2681	struct intel_wm_level lp_results[4] = {};
		2682	enum pipe pipe;
		2683	int level, max_level, wm_lp;
		2684
		2685	for (level = 1; level <= 4; level++)
		2686	if (!hsw_compute_lp_wm(dev_priv, level,
		2687	lp_maximums, params,
		2688	&lp_results[level - 1]))
		2689	break;
		2690	max_level = level - 1;
		2691
		2692	memset(results, 0, sizeof(*results));
		2693
		2694	/* The spec says it is preferred to disable FBC WMs instead of disabling
		2695	* a WM level. */
		2696	results->enable_fbc_wm = true;
		2697	for (level = 1; level <= max_level; level++) {
		2698	if (lp_results[level - 1].fbc_val > lp_maximums->fbc) {
		2699	results->enable_fbc_wm = false;
		2700	lp_results[level - 1].fbc_val = 0;
		2701	}
		2702	}
		2703
		2704	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2705	const struct intel_wm_level *r;
		2706
		2707	level = (max_level == 4 && wm_lp > 1) ? wm_lp + 1 : wm_lp;
		2708	if (level > max_level)
		2709	break;
		2710
		2711	r = &lp_results[level - 1];
		2712	results->wm_lp[wm_lp - 1] = HSW_WM_LP_VAL(level * 2,
		2713	r->fbc_val,
		2714	r->pri_val,
		2715	r->cur_val);
		2716	results->wm_lp_spr[wm_lp - 1] = r->spr_val;
		2717	}
		2718
		2719	for_each_pipe(pipe)
		2720	results->wm_pipe[pipe] = hsw_compute_wm_pipe(dev_priv, pipe,
		2721	¶ms[pipe]);
		2722
		2723	for_each_pipe(pipe) {
		2724	crtc = dev_priv->pipe_to_crtc_mapping[pipe];
		2725	results->wm_linetime[pipe] = hsw_compute_linetime_wm(dev, crtc);
		2726	}
		2727	}
		2728
		2729	/* Find the result with the highest level enabled. Check for enable_fbc_wm in
		2730	* case both are at the same level. Prefer r1 in case they're the same. */
		2731	static struct hsw_wm_values hsw_find_best_result(struct hsw_wm_values r1,
		2732	struct hsw_wm_values *r2)
		2733	{
		2734	int i, val_r1 = 0, val_r2 = 0;
		2735
		2736	for (i = 0; i < 3; i++) {
		2737	if (r1->wm_lp[i] & WM3_LP_EN)
		2738	val_r1 = r1->wm_lp[i] & WM1_LP_LATENCY_MASK;
		2739	if (r2->wm_lp[i] & WM3_LP_EN)
		2740	val_r2 = r2->wm_lp[i] & WM1_LP_LATENCY_MASK;
		2741	}
		2742
		2743	if (val_r1 == val_r2) {
		2744	if (r2->enable_fbc_wm && !r1->enable_fbc_wm)
		2745	return r2;
		2746	else
		2747	return r1;
		2748	} else if (val_r1 > val_r2) {
		2749	return r1;
		2750	} else {
		2751	return r2;
		2752	}
		2753	}
		2754
		2755	/*
		2756	* The spec says we shouldn't write when we don't need, because every write
		2757	* causes WMs to be re-evaluated, expending some power.
3031	serge	2758	*/
4104	Serge	2759	static void hsw_write_wm_values(struct drm_i915_private *dev_priv,
		2760	struct hsw_wm_values *results,
		2761	enum intel_ddb_partitioning partitioning)
		2762	{
		2763	struct hsw_wm_values previous;
		2764	uint32_t val;
		2765	enum intel_ddb_partitioning prev_partitioning;
		2766	bool prev_enable_fbc_wm;
3031	serge	2767
4104	Serge	2768	previous.wm_pipe[0] = I915_READ(WM0_PIPEA_ILK);
		2769	previous.wm_pipe[1] = I915_READ(WM0_PIPEB_ILK);
		2770	previous.wm_pipe[2] = I915_READ(WM0_PIPEC_IVB);
		2771	previous.wm_lp[0] = I915_READ(WM1_LP_ILK);
		2772	previous.wm_lp[1] = I915_READ(WM2_LP_ILK);
		2773	previous.wm_lp[2] = I915_READ(WM3_LP_ILK);
		2774	previous.wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
		2775	previous.wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		2776	previous.wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
		2777	previous.wm_linetime[0] = I915_READ(PIPE_WM_LINETIME(PIPE_A));
		2778	previous.wm_linetime[1] = I915_READ(PIPE_WM_LINETIME(PIPE_B));
		2779	previous.wm_linetime[2] = I915_READ(PIPE_WM_LINETIME(PIPE_C));
		2780
		2781	prev_partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
		2782	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
		2783
		2784	prev_enable_fbc_wm = !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
		2785
		2786	if (memcmp(results->wm_pipe, previous.wm_pipe,
		2787	sizeof(results->wm_pipe)) == 0 &&
		2788	memcmp(results->wm_lp, previous.wm_lp,
		2789	sizeof(results->wm_lp)) == 0 &&
		2790	memcmp(results->wm_lp_spr, previous.wm_lp_spr,
		2791	sizeof(results->wm_lp_spr)) == 0 &&
		2792	memcmp(results->wm_linetime, previous.wm_linetime,
		2793	sizeof(results->wm_linetime)) == 0 &&
		2794	partitioning == prev_partitioning &&
		2795	results->enable_fbc_wm == prev_enable_fbc_wm)
		2796	return;
		2797
		2798	if (previous.wm_lp[2] != 0)
		2799	I915_WRITE(WM3_LP_ILK, 0);
		2800	if (previous.wm_lp[1] != 0)
		2801	I915_WRITE(WM2_LP_ILK, 0);
		2802	if (previous.wm_lp[0] != 0)
		2803	I915_WRITE(WM1_LP_ILK, 0);
		2804
		2805	if (previous.wm_pipe[0] != results->wm_pipe[0])
		2806	I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
		2807	if (previous.wm_pipe[1] != results->wm_pipe[1])
		2808	I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
		2809	if (previous.wm_pipe[2] != results->wm_pipe[2])
		2810	I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
		2811
		2812	if (previous.wm_linetime[0] != results->wm_linetime[0])
		2813	I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
		2814	if (previous.wm_linetime[1] != results->wm_linetime[1])
		2815	I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
		2816	if (previous.wm_linetime[2] != results->wm_linetime[2])
		2817	I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
		2818
		2819	if (prev_partitioning != partitioning) {
		2820	val = I915_READ(WM_MISC);
		2821	if (partitioning == INTEL_DDB_PART_1_2)
		2822	val &= ~WM_MISC_DATA_PARTITION_5_6;
		2823	else
		2824	val \|= WM_MISC_DATA_PARTITION_5_6;
		2825	I915_WRITE(WM_MISC, val);
		2826	}
		2827
		2828	if (prev_enable_fbc_wm != results->enable_fbc_wm) {
		2829	val = I915_READ(DISP_ARB_CTL);
		2830	if (results->enable_fbc_wm)
		2831	val &= ~DISP_FBC_WM_DIS;
		2832	else
		2833	val \|= DISP_FBC_WM_DIS;
		2834	I915_WRITE(DISP_ARB_CTL, val);
		2835	}
		2836
		2837	if (previous.wm_lp_spr[0] != results->wm_lp_spr[0])
		2838	I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
		2839	if (previous.wm_lp_spr[1] != results->wm_lp_spr[1])
		2840	I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
		2841	if (previous.wm_lp_spr[2] != results->wm_lp_spr[2])
		2842	I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
		2843
		2844	if (results->wm_lp[0] != 0)
		2845	I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
		2846	if (results->wm_lp[1] != 0)
		2847	I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
		2848	if (results->wm_lp[2] != 0)
		2849	I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
3031	serge	2850	}
		2851
4104	Serge	2852	static void haswell_update_wm(struct drm_device *dev)
		2853	{
		2854	struct drm_i915_private *dev_priv = dev->dev_private;
		2855	struct hsw_wm_maximums lp_max_1_2, lp_max_5_6;
		2856	struct hsw_pipe_wm_parameters params[3];
		2857	struct hsw_wm_values results_1_2, results_5_6, *best_results;
		2858	enum intel_ddb_partitioning partitioning;
		2859
		2860	hsw_compute_wm_parameters(dev, params, &lp_max_1_2, &lp_max_5_6);
		2861
		2862	hsw_compute_wm_results(dev, params,
		2863	&lp_max_1_2, &results_1_2);
		2864	if (lp_max_1_2.pri != lp_max_5_6.pri) {
		2865	hsw_compute_wm_results(dev, params,
		2866	&lp_max_5_6, &results_5_6);
		2867	best_results = hsw_find_best_result(&results_1_2, &results_5_6);
		2868	} else {
		2869	best_results = &results_1_2;
		2870	}
		2871
		2872	partitioning = (best_results == &results_1_2) ?
		2873	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
		2874
		2875	hsw_write_wm_values(dev_priv, best_results, partitioning);
		2876	}
		2877
		2878	static void haswell_update_sprite_wm(struct drm_plane *plane,
		2879	struct drm_crtc *crtc,
		2880	uint32_t sprite_width, int pixel_size,
		2881	bool enabled, bool scaled)
		2882	{
		2883	struct intel_plane *intel_plane = to_intel_plane(plane);
		2884
		2885	intel_plane->wm.enabled = enabled;
		2886	intel_plane->wm.scaled = scaled;
		2887	intel_plane->wm.horiz_pixels = sprite_width;
		2888	intel_plane->wm.bytes_per_pixel = pixel_size;
		2889
		2890	haswell_update_wm(plane->dev);
		2891	}
		2892
3031	serge	2893	static bool
		2894	sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
		2895	uint32_t sprite_width, int pixel_size,
		2896	const struct intel_watermark_params *display,
		2897	int display_latency_ns, int *sprite_wm)
		2898	{
		2899	struct drm_crtc *crtc;
		2900	int clock;
		2901	int entries, tlb_miss;
		2902
		2903	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	2904	if (!intel_crtc_active(crtc)) {
3031	serge	2905	*sprite_wm = display->guard_size;
		2906	return false;
		2907	}
		2908
		2909	clock = crtc->mode.clock;
		2910
		2911	/* Use the small buffer method to calculate the sprite watermark */
		2912	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		2913	tlb_miss = display->fifo_size*display->cacheline_size -
		2914	sprite_width * 8;
		2915	if (tlb_miss > 0)
		2916	entries += tlb_miss;
		2917	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		2918	*sprite_wm = entries + display->guard_size;
		2919	if (*sprite_wm > (int)display->max_wm)
		2920	*sprite_wm = display->max_wm;
		2921
		2922	return true;
		2923	}
		2924
		2925	static bool
		2926	sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
		2927	uint32_t sprite_width, int pixel_size,
		2928	const struct intel_watermark_params *display,
		2929	int latency_ns, int *sprite_wm)
		2930	{
		2931	struct drm_crtc *crtc;
		2932	unsigned long line_time_us;
		2933	int clock;
		2934	int line_count, line_size;
		2935	int small, large;
		2936	int entries;
		2937
		2938	if (!latency_ns) {
		2939	*sprite_wm = 0;
		2940	return false;
		2941	}
		2942
		2943	crtc = intel_get_crtc_for_plane(dev, plane);
		2944	clock = crtc->mode.clock;
		2945	if (!clock) {
		2946	*sprite_wm = 0;
		2947	return false;
		2948	}
		2949
		2950	line_time_us = (sprite_width * 1000) / clock;
		2951	if (!line_time_us) {
		2952	*sprite_wm = 0;
		2953	return false;
		2954	}
		2955
		2956	line_count = (latency_ns / line_time_us + 1000) / 1000;
		2957	line_size = sprite_width * pixel_size;
		2958
		2959	/* Use the minimum of the small and large buffer method for primary */
		2960	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		2961	large = line_count * line_size;
		2962
		2963	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		2964	*sprite_wm = entries + display->guard_size;
		2965
		2966	return *sprite_wm > 0x3ff ? false : true;
		2967	}
		2968
4104	Serge	2969	static void sandybridge_update_sprite_wm(struct drm_plane *plane,
		2970	struct drm_crtc *crtc,
		2971	uint32_t sprite_width, int pixel_size,
		2972	bool enabled, bool scaled)
3031	serge	2973	{
4104	Serge	2974	struct drm_device *dev = plane->dev;
3031	serge	2975	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2976	int pipe = to_intel_plane(plane)->pipe;
		2977	int latency = dev_priv->wm.spr_latency[0] * 100; /* In unit 0.1us */
3031	serge	2978	u32 val;
		2979	int sprite_wm, reg;
		2980	int ret;
		2981
4104	Serge	2982	if (!enabled)
		2983	return;
		2984
3031	serge	2985	switch (pipe) {
		2986	case 0:
		2987	reg = WM0_PIPEA_ILK;
		2988	break;
		2989	case 1:
		2990	reg = WM0_PIPEB_ILK;
		2991	break;
		2992	case 2:
		2993	reg = WM0_PIPEC_IVB;
		2994	break;
		2995	default:
		2996	return; /* bad pipe */
		2997	}
		2998
		2999	ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
		3000	&sandybridge_display_wm_info,
		3001	latency, &sprite_wm);
		3002	if (!ret) {
4104	Serge	3003	DRM_DEBUG_KMS("failed to compute sprite wm for pipe %c\n",
		3004	pipe_name(pipe));
3031	serge	3005	return;
		3006	}
		3007
		3008	val = I915_READ(reg);
		3009	val &= ~WM0_PIPE_SPRITE_MASK;
		3010	I915_WRITE(reg, val \| (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
4104	Serge	3011	DRM_DEBUG_KMS("sprite watermarks For pipe %c - %d\n", pipe_name(pipe), sprite_wm);
3031	serge	3012
		3013
		3014	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		3015	pixel_size,
		3016	&sandybridge_display_srwm_info,
4104	Serge	3017	dev_priv->wm.spr_latency[1] * 500,
3031	serge	3018	&sprite_wm);
		3019	if (!ret) {
4104	Serge	3020	DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %c\n",
		3021	pipe_name(pipe));
3031	serge	3022	return;
		3023	}
		3024	I915_WRITE(WM1S_LP_ILK, sprite_wm);
		3025
		3026	/* Only IVB has two more LP watermarks for sprite */
		3027	if (!IS_IVYBRIDGE(dev))
		3028	return;
		3029
		3030	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		3031	pixel_size,
		3032	&sandybridge_display_srwm_info,
4104	Serge	3033	dev_priv->wm.spr_latency[2] * 500,
3031	serge	3034	&sprite_wm);
		3035	if (!ret) {
4104	Serge	3036	DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %c\n",
		3037	pipe_name(pipe));
3031	serge	3038	return;
		3039	}
		3040	I915_WRITE(WM2S_LP_IVB, sprite_wm);
		3041
		3042	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		3043	pixel_size,
		3044	&sandybridge_display_srwm_info,
4104	Serge	3045	dev_priv->wm.spr_latency[3] * 500,
3031	serge	3046	&sprite_wm);
		3047	if (!ret) {
4104	Serge	3048	DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %c\n",
		3049	pipe_name(pipe));
3031	serge	3050	return;
		3051	}
		3052	I915_WRITE(WM3S_LP_IVB, sprite_wm);
		3053	}
		3054
		3055	/**
		3056	* intel_update_watermarks - update FIFO watermark values based on current modes
		3057	*
		3058	* Calculate watermark values for the various WM regs based on current mode
		3059	* and plane configuration.
		3060	*
		3061	* There are several cases to deal with here:
		3062	* - normal (i.e. non-self-refresh)
		3063	* - self-refresh (SR) mode
		3064	* - lines are large relative to FIFO size (buffer can hold up to 2)
		3065	* - lines are small relative to FIFO size (buffer can hold more than 2
		3066	* lines), so need to account for TLB latency
		3067	*
		3068	* The normal calculation is:
		3069	* watermark = dotclock * bytes per pixel * latency
		3070	* where latency is platform & configuration dependent (we assume pessimal
		3071	* values here).
		3072	*
		3073	* The SR calculation is:
		3074	* watermark = (trunc(latency/line time)+1) * surface width *
		3075	* bytes per pixel
		3076	* where
		3077	* line time = htotal / dotclock
		3078	* surface width = hdisplay for normal plane and 64 for cursor
		3079	* and latency is assumed to be high, as above.
		3080	*
		3081	* The final value programmed to the register should always be rounded up,
		3082	* and include an extra 2 entries to account for clock crossings.
		3083	*
		3084	* We don't use the sprite, so we can ignore that. And on Crestline we have
		3085	* to set the non-SR watermarks to 8.
		3086	*/
		3087	void intel_update_watermarks(struct drm_device *dev)
		3088	{
		3089	struct drm_i915_private *dev_priv = dev->dev_private;
		3090
		3091	if (dev_priv->display.update_wm)
		3092	dev_priv->display.update_wm(dev);
		3093	}
		3094
4104	Serge	3095	void intel_update_sprite_watermarks(struct drm_plane *plane,
		3096	struct drm_crtc *crtc,
		3097	uint32_t sprite_width, int pixel_size,
		3098	bool enabled, bool scaled)
3031	serge	3099	{
4104	Serge	3100	struct drm_i915_private *dev_priv = plane->dev->dev_private;
3031	serge	3101
		3102	if (dev_priv->display.update_sprite_wm)
4104	Serge	3103	dev_priv->display.update_sprite_wm(plane, crtc, sprite_width,
		3104	pixel_size, enabled, scaled);
3031	serge	3105	}
		3106
		3107	static struct drm_i915_gem_object *
		3108	intel_alloc_context_page(struct drm_device *dev)
		3109	{
		3110	struct drm_i915_gem_object *ctx;
		3111	int ret;
		3112
		3113	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		3114
		3115	ctx = i915_gem_alloc_object(dev, 4096);
		3116	if (!ctx) {
		3117	DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
		3118	return NULL;
		3119	}
		3120
4104	Serge	3121	ret = i915_gem_obj_ggtt_pin(ctx, 4096, true, false);
3031	serge	3122	if (ret) {
		3123	DRM_ERROR("failed to pin power context: %d\n", ret);
		3124	goto err_unref;
		3125	}
		3126
		3127	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
		3128	if (ret) {
		3129	DRM_ERROR("failed to set-domain on power context: %d\n", ret);
		3130	goto err_unpin;
		3131	}
		3132
		3133	return ctx;
		3134
		3135	err_unpin:
		3136	i915_gem_object_unpin(ctx);
		3137	err_unref:
		3138	drm_gem_object_unreference(&ctx->base);
		3139	return NULL;
		3140	}
		3141
		3142	/**
		3143	* Lock protecting IPS related data structures
		3144	*/
		3145	DEFINE_SPINLOCK(mchdev_lock);
		3146
		3147	/* Global for IPS driver to get at the current i915 device. Protected by
		3148	* mchdev_lock. */
		3149	static struct drm_i915_private *i915_mch_dev;
		3150
		3151	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		3152	{
		3153	struct drm_i915_private *dev_priv = dev->dev_private;
		3154	u16 rgvswctl;
		3155
		3156	assert_spin_locked(&mchdev_lock);
		3157
		3158	rgvswctl = I915_READ16(MEMSWCTL);
		3159	if (rgvswctl & MEMCTL_CMD_STS) {
		3160	DRM_DEBUG("gpu busy, RCS change rejected\n");
		3161	return false; /* still busy with another command */
		3162	}
		3163
		3164	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		3165	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		3166	I915_WRITE16(MEMSWCTL, rgvswctl);
		3167	POSTING_READ16(MEMSWCTL);
		3168
		3169	rgvswctl \|= MEMCTL_CMD_STS;
		3170	I915_WRITE16(MEMSWCTL, rgvswctl);
		3171
		3172	return true;
		3173	}
		3174
		3175	static void ironlake_enable_drps(struct drm_device *dev)
		3176	{
		3177	struct drm_i915_private *dev_priv = dev->dev_private;
		3178	u32 rgvmodectl = I915_READ(MEMMODECTL);
		3179	u8 fmax, fmin, fstart, vstart;
		3180
		3181	spin_lock_irq(&mchdev_lock);
		3182
		3183	/* Enable temp reporting */
		3184	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		3185	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		3186
		3187	/* 100ms RC evaluation intervals */
		3188	I915_WRITE(RCUPEI, 100000);
		3189	I915_WRITE(RCDNEI, 100000);
		3190
		3191	/* Set max/min thresholds to 90ms and 80ms respectively */
		3192	I915_WRITE(RCBMAXAVG, 90000);
		3193	I915_WRITE(RCBMINAVG, 80000);
		3194
		3195	I915_WRITE(MEMIHYST, 1);
		3196
		3197	/* Set up min, max, and cur for interrupt handling */
		3198	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		3199	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		3200	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		3201	MEMMODE_FSTART_SHIFT;
		3202
		3203	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
		3204	PXVFREQ_PX_SHIFT;
		3205
		3206	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		3207	dev_priv->ips.fstart = fstart;
		3208
		3209	dev_priv->ips.max_delay = fstart;
		3210	dev_priv->ips.min_delay = fmin;
		3211	dev_priv->ips.cur_delay = fstart;
		3212
		3213	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		3214	fmax, fmin, fstart);
		3215
		3216	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		3217
		3218	/*
		3219	* Interrupts will be enabled in ironlake_irq_postinstall
		3220	*/
		3221
		3222	I915_WRITE(VIDSTART, vstart);
		3223	POSTING_READ(VIDSTART);
		3224
		3225	rgvmodectl \|= MEMMODE_SWMODE_EN;
		3226	I915_WRITE(MEMMODECTL, rgvmodectl);
		3227
		3228	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		3229	DRM_ERROR("stuck trying to change perf mode\n");
		3230	mdelay(1);
		3231
		3232	ironlake_set_drps(dev, fstart);
		3233
		3234	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
		3235	I915_READ(0x112e0);
		3236	dev_priv->ips.last_time1 = jiffies_to_msecs(GetTimerTicks());
		3237	dev_priv->ips.last_count2 = I915_READ(0x112f4);
3482	Serge	3238	getrawmonotonic(&dev_priv->ips.last_time2);
3031	serge	3239
		3240	spin_unlock_irq(&mchdev_lock);
		3241	}
		3242
		3243	static void ironlake_disable_drps(struct drm_device *dev)
		3244	{
		3245	struct drm_i915_private *dev_priv = dev->dev_private;
		3246	u16 rgvswctl;
		3247
		3248	spin_lock_irq(&mchdev_lock);
		3249
		3250	rgvswctl = I915_READ16(MEMSWCTL);
		3251
		3252	/* Ack interrupts, disable EFC interrupt */
		3253	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		3254	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		3255	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		3256	I915_WRITE(DEIIR, DE_PCU_EVENT);
		3257	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		3258
		3259	/* Go back to the starting frequency */
		3260	ironlake_set_drps(dev, dev_priv->ips.fstart);
		3261	mdelay(1);
		3262	rgvswctl \|= MEMCTL_CMD_STS;
		3263	I915_WRITE(MEMSWCTL, rgvswctl);
		3264	mdelay(1);
		3265
		3266	spin_unlock_irq(&mchdev_lock);
		3267	}
		3268
		3269	/* There's a funny hw issue where the hw returns all 0 when reading from
		3270	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		3271	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		3272	* all limits and the gpu stuck at whatever frequency it is at atm).
		3273	*/
		3274	static u32 gen6_rps_limits(struct drm_i915_private dev_priv, u8 val)
		3275	{
		3276	u32 limits;
		3277
		3278	limits = 0;
		3279
		3280	if (*val >= dev_priv->rps.max_delay)
		3281	*val = dev_priv->rps.max_delay;
		3282	limits \|= dev_priv->rps.max_delay << 24;
		3283
		3284	/* Only set the down limit when we've reached the lowest level to avoid
		3285	* getting more interrupts, otherwise leave this clear. This prevents a
		3286	* race in the hw when coming out of rc6: There's a tiny window where
		3287	* the hw runs at the minimal clock before selecting the desired
		3288	* frequency, if the down threshold expires in that window we will not
		3289	* receive a down interrupt. */
		3290	if (*val <= dev_priv->rps.min_delay) {
		3291	*val = dev_priv->rps.min_delay;
		3292	limits \|= dev_priv->rps.min_delay << 16;
		3293	}
		3294
		3295	return limits;
		3296	}
		3297
		3298	void gen6_set_rps(struct drm_device *dev, u8 val)
		3299	{
		3300	struct drm_i915_private *dev_priv = dev->dev_private;
		3301	u32 limits = gen6_rps_limits(dev_priv, &val);
		3302
3243	Serge	3303	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3304	WARN_ON(val > dev_priv->rps.max_delay);
		3305	WARN_ON(val < dev_priv->rps.min_delay);
		3306
		3307	if (val == dev_priv->rps.cur_delay)
		3308	return;
		3309
3746	Serge	3310	if (IS_HASWELL(dev))
		3311	I915_WRITE(GEN6_RPNSWREQ,
		3312	HSW_FREQUENCY(val));
		3313	else
3031	serge	3314	I915_WRITE(GEN6_RPNSWREQ,
		3315	GEN6_FREQUENCY(val) \|
		3316	GEN6_OFFSET(0) \|
		3317	GEN6_AGGRESSIVE_TURBO);
		3318
		3319	/* Make sure we continue to get interrupts
		3320	* until we hit the minimum or maximum frequencies.
		3321	*/
		3322	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
		3323
		3324	POSTING_READ(GEN6_RPNSWREQ);
		3325
		3326	dev_priv->rps.cur_delay = val;
		3327
		3328	trace_intel_gpu_freq_change(val * 50);
		3329	}
		3330
4104	Serge	3331	/*
		3332	* Wait until the previous freq change has completed,
		3333	* or the timeout elapsed, and then update our notion
		3334	* of the current GPU frequency.
		3335	*/
		3336	static void vlv_update_rps_cur_delay(struct drm_i915_private *dev_priv)
3031	serge	3337	{
4104	Serge	3338	u32 pval;
		3339
		3340	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		3341
		3342	if (wait_for(((pval = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS)) & GENFREQSTATUS) == 0, 10))
		3343	DRM_DEBUG_DRIVER("timed out waiting for Punit\n");
		3344
		3345	pval >>= 8;
		3346
		3347	if (pval != dev_priv->rps.cur_delay)
		3348	DRM_DEBUG_DRIVER("Punit overrode GPU freq: %d MHz (%u) requested, but got %d Mhz (%u)\n",
		3349	vlv_gpu_freq(dev_priv->mem_freq, dev_priv->rps.cur_delay),
		3350	dev_priv->rps.cur_delay,
		3351	vlv_gpu_freq(dev_priv->mem_freq, pval), pval);
		3352
		3353	dev_priv->rps.cur_delay = pval;
		3354	}
		3355
		3356	void valleyview_set_rps(struct drm_device *dev, u8 val)
		3357	{
3031	serge	3358	struct drm_i915_private *dev_priv = dev->dev_private;
		3359
4104	Serge	3360	gen6_rps_limits(dev_priv, &val);
		3361
		3362	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		3363	WARN_ON(val > dev_priv->rps.max_delay);
		3364	WARN_ON(val < dev_priv->rps.min_delay);
		3365
		3366	vlv_update_rps_cur_delay(dev_priv);
		3367
		3368	DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
		3369	vlv_gpu_freq(dev_priv->mem_freq,
		3370	dev_priv->rps.cur_delay),
		3371	dev_priv->rps.cur_delay,
		3372	vlv_gpu_freq(dev_priv->mem_freq, val), val);
		3373
		3374	if (val == dev_priv->rps.cur_delay)
		3375	return;
		3376
		3377	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
		3378
		3379	dev_priv->rps.cur_delay = val;
		3380
		3381	trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv->mem_freq, val));
		3382	}
		3383
		3384	static void gen6_disable_rps_interrupts(struct drm_device *dev)
		3385	{
		3386	struct drm_i915_private *dev_priv = dev->dev_private;
		3387
3031	serge	3388	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
4104	Serge	3389	I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & ~GEN6_PM_RPS_EVENTS);
3031	serge	3390	/* Complete PM interrupt masking here doesn't race with the rps work
		3391	* item again unmasking PM interrupts because that is using a different
		3392	* register (PMIMR) to mask PM interrupts. The only risk is in leaving
		3393	* stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
		3394
4104	Serge	3395	spin_lock_irq(&dev_priv->irq_lock);
3031	serge	3396	dev_priv->rps.pm_iir = 0;
4104	Serge	3397	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	3398
4104	Serge	3399	I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
3031	serge	3400	}
		3401
4104	Serge	3402	static void gen6_disable_rps(struct drm_device *dev)
		3403	{
		3404	struct drm_i915_private *dev_priv = dev->dev_private;
		3405
		3406	I915_WRITE(GEN6_RC_CONTROL, 0);
		3407	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		3408
		3409	gen6_disable_rps_interrupts(dev);
		3410	}
		3411
		3412	static void valleyview_disable_rps(struct drm_device *dev)
		3413	{
		3414	struct drm_i915_private *dev_priv = dev->dev_private;
		3415
		3416	I915_WRITE(GEN6_RC_CONTROL, 0);
		3417
		3418	gen6_disable_rps_interrupts(dev);
		3419
		3420	if (dev_priv->vlv_pctx) {
		3421	drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
		3422	dev_priv->vlv_pctx = NULL;
		3423	}
		3424	}
		3425
3031	serge	3426	int intel_enable_rc6(const struct drm_device *dev)
		3427	{
4104	Serge	3428	/* No RC6 before Ironlake */
		3429	if (INTEL_INFO(dev)->gen < 5)
		3430	return 0;
		3431
3031	serge	3432	/* Respect the kernel parameter if it is set */
		3433	if (i915_enable_rc6 >= 0)
		3434	return i915_enable_rc6;
		3435
3120	serge	3436	/* Disable RC6 on Ironlake */
		3437	if (INTEL_INFO(dev)->gen == 5)
		3438	return 0;
3031	serge	3439
		3440	if (IS_HASWELL(dev)) {
		3441	DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
		3442	return INTEL_RC6_ENABLE;
		3443	}
		3444
		3445	/* snb/ivb have more than one rc6 state. */
		3446	if (INTEL_INFO(dev)->gen == 6) {
		3447	DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
		3448	return INTEL_RC6_ENABLE;
		3449	}
		3450
		3451	DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
		3452	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
		3453	}
		3454
4104	Serge	3455	static void gen6_enable_rps_interrupts(struct drm_device *dev)
		3456	{
		3457	struct drm_i915_private *dev_priv = dev->dev_private;
		3458	u32 enabled_intrs;
		3459
		3460	spin_lock_irq(&dev_priv->irq_lock);
		3461	WARN_ON(dev_priv->rps.pm_iir);
		3462	snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
		3463	I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
		3464	spin_unlock_irq(&dev_priv->irq_lock);
		3465
		3466	/* only unmask PM interrupts we need. Mask all others. */
		3467	enabled_intrs = GEN6_PM_RPS_EVENTS;
		3468
		3469	/* IVB and SNB hard hangs on looping batchbuffer
		3470	* if GEN6_PM_UP_EI_EXPIRED is masked.
		3471	*/
		3472	if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
		3473	enabled_intrs \|= GEN6_PM_RP_UP_EI_EXPIRED;
		3474
		3475	I915_WRITE(GEN6_PMINTRMSK, ~enabled_intrs);
		3476	}
		3477
3031	serge	3478	static void gen6_enable_rps(struct drm_device *dev)
		3479	{
		3480	struct drm_i915_private *dev_priv = dev->dev_private;
		3481	struct intel_ring_buffer *ring;
		3482	u32 rp_state_cap;
		3483	u32 gt_perf_status;
3243	Serge	3484	u32 rc6vids, pcu_mbox, rc6_mask = 0;
3031	serge	3485	u32 gtfifodbg;
		3486	int rc6_mode;
3243	Serge	3487	int i, ret;
3031	serge	3488
3243	Serge	3489	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3490
		3491	/* Here begins a magic sequence of register writes to enable
		3492	* auto-downclocking.
		3493	*
		3494	* Perhaps there might be some value in exposing these to
		3495	* userspace...
		3496	*/
		3497	I915_WRITE(GEN6_RC_STATE, 0);
		3498
		3499	/* Clear the DBG now so we don't confuse earlier errors */
		3500	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		3501	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		3502	I915_WRITE(GTFIFODBG, gtfifodbg);
		3503	}
		3504
		3505	gen6_gt_force_wake_get(dev_priv);
		3506
		3507	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		3508	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
		3509
3746	Serge	3510	/* In units of 50MHz */
		3511	dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff;
3031	serge	3512	dev_priv->rps.min_delay = (rp_state_cap & 0xff0000) >> 16;
		3513	dev_priv->rps.cur_delay = 0;
		3514
		3515	/* disable the counters and set deterministic thresholds */
		3516	I915_WRITE(GEN6_RC_CONTROL, 0);
		3517
		3518	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		3519	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		3520	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		3521	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		3522	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		3523
		3524	for_each_ring(ring, dev_priv, i)
		3525	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		3526
		3527	I915_WRITE(GEN6_RC_SLEEP, 0);
		3528	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4104	Serge	3529	if (INTEL_INFO(dev)->gen <= 6 \|\| IS_IVYBRIDGE(dev))
		3530	I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
		3531	else
3031	serge	3532	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	3533	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	3534	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		3535
		3536	/* Check if we are enabling RC6 */
		3537	rc6_mode = intel_enable_rc6(dev_priv->dev);
		3538	if (rc6_mode & INTEL_RC6_ENABLE)
		3539	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		3540
		3541	/* We don't use those on Haswell */
		3542	if (!IS_HASWELL(dev)) {
		3543	if (rc6_mode & INTEL_RC6p_ENABLE)
		3544	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		3545
		3546	if (rc6_mode & INTEL_RC6pp_ENABLE)
		3547	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		3548	}
		3549
		3550	DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
		3551	(rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		3552	(rc6_mask & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		3553	(rc6_mask & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
		3554
		3555	I915_WRITE(GEN6_RC_CONTROL,
		3556	rc6_mask \|
		3557	GEN6_RC_CTL_EI_MODE(1) \|
		3558	GEN6_RC_CTL_HW_ENABLE);
		3559
3746	Serge	3560	if (IS_HASWELL(dev)) {
		3561	I915_WRITE(GEN6_RPNSWREQ,
		3562	HSW_FREQUENCY(10));
		3563	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		3564	HSW_FREQUENCY(12));
		3565	} else {
3031	serge	3566	I915_WRITE(GEN6_RPNSWREQ,
		3567	GEN6_FREQUENCY(10) \|
		3568	GEN6_OFFSET(0) \|
		3569	GEN6_AGGRESSIVE_TURBO);
		3570	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		3571	GEN6_FREQUENCY(12));
3746	Serge	3572	}
3031	serge	3573
		3574	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
		3575	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
		3576	dev_priv->rps.max_delay << 24 \|
		3577	dev_priv->rps.min_delay << 16);
		3578
		3579	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		3580	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		3581	I915_WRITE(GEN6_RP_UP_EI, 66000);
		3582	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		3583
		3584	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		3585	I915_WRITE(GEN6_RP_CONTROL,
		3586	GEN6_RP_MEDIA_TURBO \|
		3587	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3588	GEN6_RP_MEDIA_IS_GFX \|
		3589	GEN6_RP_ENABLE \|
		3590	GEN6_RP_UP_BUSY_AVG \|
		3591	(IS_HASWELL(dev) ? GEN7_RP_DOWN_IDLE_AVG : GEN6_RP_DOWN_IDLE_CONT));
		3592
3243	Serge	3593	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
		3594	if (!ret) {
		3595	pcu_mbox = 0;
		3596	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
3746	Serge	3597	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		3598	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
		3599	(dev_priv->rps.max_delay & 0xff) * 50,
		3600	(pcu_mbox & 0xff) * 50);
		3601	dev_priv->rps.hw_max = pcu_mbox & 0xff;
3031	serge	3602	}
3243	Serge	3603	} else {
		3604	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		3605	}
3031	serge	3606
		3607	gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
		3608
4104	Serge	3609	gen6_enable_rps_interrupts(dev);
3031	serge	3610
3243	Serge	3611	rc6vids = 0;
		3612	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		3613	if (IS_GEN6(dev) && ret) {
		3614	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		3615	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		3616	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		3617	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		3618	rc6vids &= 0xffff00;
		3619	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		3620	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		3621	if (ret)
		3622	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		3623	}
		3624
3031	serge	3625	gen6_gt_force_wake_put(dev_priv);
		3626	}
		3627
4104	Serge	3628	void gen6_update_ring_freq(struct drm_device *dev)
3031	serge	3629	{
		3630	struct drm_i915_private *dev_priv = dev->dev_private;
		3631	int min_freq = 15;
3746	Serge	3632	unsigned int gpu_freq;
		3633	unsigned int max_ia_freq, min_ring_freq;
3031	serge	3634	int scaling_factor = 180;
		3635
3243	Serge	3636	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3637
		3638	max_ia_freq = cpufreq_quick_get_max(0);
		3639	/*
		3640	* Default to measured freq if none found, PCU will ensure we don't go
		3641	* over
		3642	*/
		3643	if (!max_ia_freq)
		3644	max_ia_freq = tsc_khz;
		3645
		3646	/* Convert from kHz to MHz */
		3647	max_ia_freq /= 1000;
		3648
3746	Serge	3649	min_ring_freq = I915_READ(MCHBAR_MIRROR_BASE_SNB + DCLK);
		3650	/* convert DDR frequency from units of 133.3MHz to bandwidth */
		3651	min_ring_freq = (2 * 4 * min_ring_freq + 2) / 3;
		3652
3031	serge	3653	/*
		3654	* For each potential GPU frequency, load a ring frequency we'd like
		3655	* to use for memory access. We do this by specifying the IA frequency
		3656	* the PCU should use as a reference to determine the ring frequency.
		3657	*/
		3658	for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
		3659	gpu_freq--) {
		3660	int diff = dev_priv->rps.max_delay - gpu_freq;
3746	Serge	3661	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	3662
3746	Serge	3663	if (IS_HASWELL(dev)) {
		3664	ring_freq = (gpu_freq * 5 + 3) / 4;
		3665	ring_freq = max(min_ring_freq, ring_freq);
		3666	/* leave ia_freq as the default, chosen by cpufreq */
		3667	} else {
		3668	/* On older processors, there is no separate ring
		3669	* clock domain, so in order to boost the bandwidth
		3670	* of the ring, we need to upclock the CPU (ia_freq).
		3671	*
		3672	* For GPU frequencies less than 750MHz,
		3673	* just use the lowest ring freq.
3031	serge	3674	*/
		3675	if (gpu_freq < min_freq)
		3676	ia_freq = 800;
		3677	else
		3678	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		3679	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	3680	}
3031	serge	3681
3243	Serge	3682	sandybridge_pcode_write(dev_priv,
		3683	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	3684	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		3685	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		3686	gpu_freq);
3031	serge	3687	}
		3688	}
		3689
4104	Serge	3690	int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
		3691	{
		3692	u32 val, rp0;
		3693
		3694	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		3695
		3696	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
		3697	/* Clamp to max */
		3698	rp0 = min_t(u32, rp0, 0xea);
		3699
		3700	return rp0;
		3701	}
		3702
		3703	static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		3704	{
		3705	u32 val, rpe;
		3706
		3707	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
		3708	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
		3709	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
		3710	rpe \|= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
		3711
		3712	return rpe;
		3713	}
		3714
		3715	int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
		3716	{
		3717	return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
		3718	}
		3719
		3720	static void vlv_rps_timer_work(struct work_struct *work)
		3721	{
		3722	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
		3723	rps.vlv_work.work);
		3724
		3725	/*
		3726	* Timer fired, we must be idle. Drop to min voltage state.
		3727	* Note: we use RPe here since it should match the
		3728	* Vmin we were shooting for. That should give us better
		3729	* perf when we come back out of RC6 than if we used the
		3730	* min freq available.
		3731	*/
		3732	mutex_lock(&dev_priv->rps.hw_lock);
		3733	if (dev_priv->rps.cur_delay > dev_priv->rps.rpe_delay)
		3734	valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay);
		3735	mutex_unlock(&dev_priv->rps.hw_lock);
		3736	}
		3737
		3738	static void valleyview_setup_pctx(struct drm_device *dev)
		3739	{
		3740	struct drm_i915_private *dev_priv = dev->dev_private;
		3741	struct drm_i915_gem_object *pctx;
		3742	unsigned long pctx_paddr;
		3743	u32 pcbr;
		3744	int pctx_size = 24*1024;
		3745
		3746	pcbr = I915_READ(VLV_PCBR);
		3747	if (pcbr) {
		3748	/* BIOS set it up already, grab the pre-alloc'd space */
		3749	int pcbr_offset;
		3750
		3751	pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
		3752	pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
		3753	pcbr_offset,
		3754	I915_GTT_OFFSET_NONE,
		3755	pctx_size);
		3756	goto out;
		3757	}
		3758
		3759	/*
		3760	* From the Gunit register HAS:
		3761	* The Gfx driver is expected to program this register and ensure
		3762	* proper allocation within Gfx stolen memory. For example, this
		3763	* register should be programmed such than the PCBR range does not
		3764	* overlap with other ranges, such as the frame buffer, protected
		3765	* memory, or any other relevant ranges.
		3766	*/
		3767	pctx = i915_gem_object_create_stolen(dev, pctx_size);
		3768	if (!pctx) {
		3769	DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
		3770	return;
		3771	}
		3772
		3773	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
		3774	I915_WRITE(VLV_PCBR, pctx_paddr);
		3775
		3776	out:
		3777	dev_priv->vlv_pctx = pctx;
		3778	}
		3779
		3780	static void valleyview_enable_rps(struct drm_device *dev)
		3781	{
		3782	struct drm_i915_private *dev_priv = dev->dev_private;
		3783	struct intel_ring_buffer *ring;
		3784	u32 gtfifodbg, val;
		3785	int i;
		3786
		3787	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		3788
		3789	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		3790	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		3791	I915_WRITE(GTFIFODBG, gtfifodbg);
		3792	}
		3793
		3794	valleyview_setup_pctx(dev);
		3795
		3796	gen6_gt_force_wake_get(dev_priv);
		3797
		3798	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		3799	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		3800	I915_WRITE(GEN6_RP_UP_EI, 66000);
		3801	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		3802
		3803	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		3804
		3805	I915_WRITE(GEN6_RP_CONTROL,
		3806	GEN6_RP_MEDIA_TURBO \|
		3807	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3808	GEN6_RP_MEDIA_IS_GFX \|
		3809	GEN6_RP_ENABLE \|
		3810	GEN6_RP_UP_BUSY_AVG \|
		3811	GEN6_RP_DOWN_IDLE_CONT);
		3812
		3813	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
		3814	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		3815	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		3816
		3817	for_each_ring(ring, dev_priv, i)
		3818	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		3819
		3820	I915_WRITE(GEN6_RC6_THRESHOLD, 0xc350);
		3821
		3822	/* allows RC6 residency counter to work */
		3823	I915_WRITE(0x138104, _MASKED_BIT_ENABLE(0x3));
		3824	I915_WRITE(GEN6_RC_CONTROL,
		3825	GEN7_RC_CTL_TO_MODE);
		3826
		3827	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		3828	switch ((val >> 6) & 3) {
		3829	case 0:
		3830	case 1:
		3831	dev_priv->mem_freq = 800;
		3832	break;
		3833	case 2:
		3834	dev_priv->mem_freq = 1066;
		3835	break;
		3836	case 3:
		3837	dev_priv->mem_freq = 1333;
		3838	break;
		3839	}
		3840	DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
		3841
		3842	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
		3843	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		3844
		3845	dev_priv->rps.cur_delay = (val >> 8) & 0xff;
		3846	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
		3847	vlv_gpu_freq(dev_priv->mem_freq,
		3848	dev_priv->rps.cur_delay),
		3849	dev_priv->rps.cur_delay);
		3850
		3851	dev_priv->rps.max_delay = valleyview_rps_max_freq(dev_priv);
		3852	dev_priv->rps.hw_max = dev_priv->rps.max_delay;
		3853	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
		3854	vlv_gpu_freq(dev_priv->mem_freq,
		3855	dev_priv->rps.max_delay),
		3856	dev_priv->rps.max_delay);
		3857
		3858	dev_priv->rps.rpe_delay = valleyview_rps_rpe_freq(dev_priv);
		3859	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
		3860	vlv_gpu_freq(dev_priv->mem_freq,
		3861	dev_priv->rps.rpe_delay),
		3862	dev_priv->rps.rpe_delay);
		3863
		3864	dev_priv->rps.min_delay = valleyview_rps_min_freq(dev_priv);
		3865	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
		3866	vlv_gpu_freq(dev_priv->mem_freq,
		3867	dev_priv->rps.min_delay),
		3868	dev_priv->rps.min_delay);
		3869
		3870	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
		3871	vlv_gpu_freq(dev_priv->mem_freq,
		3872	dev_priv->rps.rpe_delay),
		3873	dev_priv->rps.rpe_delay);
		3874
		3875	valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay);
		3876
		3877	gen6_enable_rps_interrupts(dev);
		3878
		3879	gen6_gt_force_wake_put(dev_priv);
		3880	}
		3881
3031	serge	3882	void ironlake_teardown_rc6(struct drm_device *dev)
		3883	{
		3884	struct drm_i915_private *dev_priv = dev->dev_private;
		3885
3243	Serge	3886	if (dev_priv->ips.renderctx) {
		3887	i915_gem_object_unpin(dev_priv->ips.renderctx);
		3888	drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
		3889	dev_priv->ips.renderctx = NULL;
3031	serge	3890	}
		3891
3243	Serge	3892	if (dev_priv->ips.pwrctx) {
		3893	i915_gem_object_unpin(dev_priv->ips.pwrctx);
		3894	drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
		3895	dev_priv->ips.pwrctx = NULL;
3031	serge	3896	}
		3897	}
		3898
		3899	static void ironlake_disable_rc6(struct drm_device *dev)
		3900	{
		3901	struct drm_i915_private *dev_priv = dev->dev_private;
		3902
		3903	if (I915_READ(PWRCTXA)) {
		3904	/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
		3905	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) \| RCX_SW_EXIT);
		3906	wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
		3907	50);
		3908
		3909	I915_WRITE(PWRCTXA, 0);
		3910	POSTING_READ(PWRCTXA);
		3911
		3912	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		3913	POSTING_READ(RSTDBYCTL);
		3914	}
		3915	}
		3916
		3917	static int ironlake_setup_rc6(struct drm_device *dev)
		3918	{
		3919	struct drm_i915_private *dev_priv = dev->dev_private;
		3920
3243	Serge	3921	if (dev_priv->ips.renderctx == NULL)
		3922	dev_priv->ips.renderctx = intel_alloc_context_page(dev);
		3923	if (!dev_priv->ips.renderctx)
3031	serge	3924	return -ENOMEM;
		3925
3243	Serge	3926	if (dev_priv->ips.pwrctx == NULL)
		3927	dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
		3928	if (!dev_priv->ips.pwrctx) {
3031	serge	3929	ironlake_teardown_rc6(dev);
		3930	return -ENOMEM;
		3931	}
		3932
		3933	return 0;
		3934	}
		3935
		3936	static void ironlake_enable_rc6(struct drm_device *dev)
		3937	{
		3938	struct drm_i915_private *dev_priv = dev->dev_private;
		3939	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
3243	Serge	3940	bool was_interruptible;
3031	serge	3941	int ret;
		3942
		3943	/* rc6 disabled by default due to repeated reports of hanging during
		3944	* boot and resume.
		3945	*/
		3946	if (!intel_enable_rc6(dev))
		3947	return;
		3948
		3949	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		3950
		3951	ret = ironlake_setup_rc6(dev);
		3952	if (ret)
		3953	return;
		3954
3243	Serge	3955	was_interruptible = dev_priv->mm.interruptible;
		3956	dev_priv->mm.interruptible = false;
		3957
3031	serge	3958	/*
		3959	* GPU can automatically power down the render unit if given a page
		3960	* to save state.
		3961	*/
		3962	ret = intel_ring_begin(ring, 6);
		3963	if (ret) {
		3964	ironlake_teardown_rc6(dev);
3243	Serge	3965	dev_priv->mm.interruptible = was_interruptible;
3031	serge	3966	return;
		3967	}
		3968
		3969	intel_ring_emit(ring, MI_SUSPEND_FLUSH \| MI_SUSPEND_FLUSH_EN);
		3970	intel_ring_emit(ring, MI_SET_CONTEXT);
4104	Serge	3971	intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) \|
3031	serge	3972	MI_MM_SPACE_GTT \|
		3973	MI_SAVE_EXT_STATE_EN \|
		3974	MI_RESTORE_EXT_STATE_EN \|
		3975	MI_RESTORE_INHIBIT);
		3976	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
		3977	intel_ring_emit(ring, MI_NOOP);
		3978	intel_ring_emit(ring, MI_FLUSH);
		3979	intel_ring_advance(ring);
		3980
		3981	/*
		3982	* Wait for the command parser to advance past MI_SET_CONTEXT. The HW
		3983	* does an implicit flush, combined with MI_FLUSH above, it should be
		3984	* safe to assume that renderctx is valid
		3985	*/
3243	Serge	3986	ret = intel_ring_idle(ring);
		3987	dev_priv->mm.interruptible = was_interruptible;
3031	serge	3988	if (ret) {
3746	Serge	3989	DRM_ERROR("failed to enable ironlake power savings\n");
3031	serge	3990	ironlake_teardown_rc6(dev);
		3991	return;
		3992	}
		3993
4104	Serge	3994	I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) \| PWRCTX_EN);
3031	serge	3995	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		3996	}
		3997
		3998	static unsigned long intel_pxfreq(u32 vidfreq)
		3999	{
		4000	unsigned long freq;
		4001	int div = (vidfreq & 0x3f0000) >> 16;
		4002	int post = (vidfreq & 0x3000) >> 12;
		4003	int pre = (vidfreq & 0x7);
		4004
		4005	if (!pre)
		4006	return 0;
		4007
		4008	freq = ((div * 133333) / ((1<
		4009
		4010	return freq;
		4011	}
		4012
		4013	static const struct cparams {
		4014	u16 i;
		4015	u16 t;
		4016	u16 m;
		4017	u16 c;
		4018	} cparams[] = {
		4019	{ 1, 1333, 301, 28664 },
		4020	{ 1, 1066, 294, 24460 },
		4021	{ 1, 800, 294, 25192 },
		4022	{ 0, 1333, 276, 27605 },
		4023	{ 0, 1066, 276, 27605 },
		4024	{ 0, 800, 231, 23784 },
		4025	};
		4026
		4027	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		4028	{
		4029	u64 total_count, diff, ret;
		4030	u32 count1, count2, count3, m = 0, c = 0;
		4031	unsigned long now = jiffies_to_msecs(GetTimerTicks()), diff1;
		4032	int i;
		4033
		4034	assert_spin_locked(&mchdev_lock);
		4035
		4036	diff1 = now - dev_priv->ips.last_time1;
		4037
		4038	/* Prevent division-by-zero if we are asking too fast.
		4039	* Also, we don't get interesting results if we are polling
		4040	* faster than once in 10ms, so just return the saved value
		4041	* in such cases.
		4042	*/
		4043	if (diff1 <= 10)
		4044	return dev_priv->ips.chipset_power;
		4045
		4046	count1 = I915_READ(DMIEC);
		4047	count2 = I915_READ(DDREC);
		4048	count3 = I915_READ(CSIEC);
		4049
		4050	total_count = count1 + count2 + count3;
		4051
		4052	/* FIXME: handle per-counter overflow */
		4053	if (total_count < dev_priv->ips.last_count1) {
		4054	diff = ~0UL - dev_priv->ips.last_count1;
		4055	diff += total_count;
		4056	} else {
		4057	diff = total_count - dev_priv->ips.last_count1;
		4058	}
		4059
		4060	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		4061	if (cparams[i].i == dev_priv->ips.c_m &&
		4062	cparams[i].t == dev_priv->ips.r_t) {
		4063	m = cparams[i].m;
		4064	c = cparams[i].c;
		4065	break;
		4066	}
		4067	}
		4068
		4069	diff = div_u64(diff, diff1);
		4070	ret = ((m * diff) + c);
		4071	ret = div_u64(ret, 10);
		4072
		4073	dev_priv->ips.last_count1 = total_count;
		4074	dev_priv->ips.last_time1 = now;
		4075
		4076	dev_priv->ips.chipset_power = ret;
		4077
		4078	return ret;
		4079	}
		4080
		4081	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		4082	{
		4083	unsigned long val;
		4084
		4085	if (dev_priv->info->gen != 5)
		4086	return 0;
		4087
		4088	spin_lock_irq(&mchdev_lock);
		4089
		4090	val = __i915_chipset_val(dev_priv);
		4091
		4092	spin_unlock_irq(&mchdev_lock);
		4093
		4094	return val;
		4095	}
		4096
		4097	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		4098	{
		4099	unsigned long m, x, b;
		4100	u32 tsfs;
		4101
		4102	tsfs = I915_READ(TSFS);
		4103
		4104	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		4105	x = I915_READ8(TR1);
		4106
		4107	b = tsfs & TSFS_INTR_MASK;
		4108
		4109	return ((m * x) / 127) - b;
		4110	}
		4111
		4112	static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		4113	{
		4114	static const struct v_table {
		4115	u16 vd; /* in .1 mil */
		4116	u16 vm; /* in .1 mil */
		4117	} v_table[] = {
		4118	{ 0, 0, },
		4119	{ 375, 0, },
		4120	{ 500, 0, },
		4121	{ 625, 0, },
		4122	{ 750, 0, },
		4123	{ 875, 0, },
		4124	{ 1000, 0, },
		4125	{ 1125, 0, },
		4126	{ 4125, 3000, },
		4127	{ 4125, 3000, },
		4128	{ 4125, 3000, },
		4129	{ 4125, 3000, },
		4130	{ 4125, 3000, },
		4131	{ 4125, 3000, },
		4132	{ 4125, 3000, },
		4133	{ 4125, 3000, },
		4134	{ 4125, 3000, },
		4135	{ 4125, 3000, },
		4136	{ 4125, 3000, },
		4137	{ 4125, 3000, },
		4138	{ 4125, 3000, },
		4139	{ 4125, 3000, },
		4140	{ 4125, 3000, },
		4141	{ 4125, 3000, },
		4142	{ 4125, 3000, },
		4143	{ 4125, 3000, },
		4144	{ 4125, 3000, },
		4145	{ 4125, 3000, },
		4146	{ 4125, 3000, },
		4147	{ 4125, 3000, },
		4148	{ 4125, 3000, },
		4149	{ 4125, 3000, },
		4150	{ 4250, 3125, },
		4151	{ 4375, 3250, },
		4152	{ 4500, 3375, },
		4153	{ 4625, 3500, },
		4154	{ 4750, 3625, },
		4155	{ 4875, 3750, },
		4156	{ 5000, 3875, },
		4157	{ 5125, 4000, },
		4158	{ 5250, 4125, },
		4159	{ 5375, 4250, },
		4160	{ 5500, 4375, },
		4161	{ 5625, 4500, },
		4162	{ 5750, 4625, },
		4163	{ 5875, 4750, },
		4164	{ 6000, 4875, },
		4165	{ 6125, 5000, },
		4166	{ 6250, 5125, },
		4167	{ 6375, 5250, },
		4168	{ 6500, 5375, },
		4169	{ 6625, 5500, },
		4170	{ 6750, 5625, },
		4171	{ 6875, 5750, },
		4172	{ 7000, 5875, },
		4173	{ 7125, 6000, },
		4174	{ 7250, 6125, },
		4175	{ 7375, 6250, },
		4176	{ 7500, 6375, },
		4177	{ 7625, 6500, },
		4178	{ 7750, 6625, },
		4179	{ 7875, 6750, },
		4180	{ 8000, 6875, },
		4181	{ 8125, 7000, },
		4182	{ 8250, 7125, },
		4183	{ 8375, 7250, },
		4184	{ 8500, 7375, },
		4185	{ 8625, 7500, },
		4186	{ 8750, 7625, },
		4187	{ 8875, 7750, },
		4188	{ 9000, 7875, },
		4189	{ 9125, 8000, },
		4190	{ 9250, 8125, },
		4191	{ 9375, 8250, },
		4192	{ 9500, 8375, },
		4193	{ 9625, 8500, },
		4194	{ 9750, 8625, },
		4195	{ 9875, 8750, },
		4196	{ 10000, 8875, },
		4197	{ 10125, 9000, },
		4198	{ 10250, 9125, },
		4199	{ 10375, 9250, },
		4200	{ 10500, 9375, },
		4201	{ 10625, 9500, },
		4202	{ 10750, 9625, },
		4203	{ 10875, 9750, },
		4204	{ 11000, 9875, },
		4205	{ 11125, 10000, },
		4206	{ 11250, 10125, },
		4207	{ 11375, 10250, },
		4208	{ 11500, 10375, },
		4209	{ 11625, 10500, },
		4210	{ 11750, 10625, },
		4211	{ 11875, 10750, },
		4212	{ 12000, 10875, },
		4213	{ 12125, 11000, },
		4214	{ 12250, 11125, },
		4215	{ 12375, 11250, },
		4216	{ 12500, 11375, },
		4217	{ 12625, 11500, },
		4218	{ 12750, 11625, },
		4219	{ 12875, 11750, },
		4220	{ 13000, 11875, },
		4221	{ 13125, 12000, },
		4222	{ 13250, 12125, },
		4223	{ 13375, 12250, },
		4224	{ 13500, 12375, },
		4225	{ 13625, 12500, },
		4226	{ 13750, 12625, },
		4227	{ 13875, 12750, },
		4228	{ 14000, 12875, },
		4229	{ 14125, 13000, },
		4230	{ 14250, 13125, },
		4231	{ 14375, 13250, },
		4232	{ 14500, 13375, },
		4233	{ 14625, 13500, },
		4234	{ 14750, 13625, },
		4235	{ 14875, 13750, },
		4236	{ 15000, 13875, },
		4237	{ 15125, 14000, },
		4238	{ 15250, 14125, },
		4239	{ 15375, 14250, },
		4240	{ 15500, 14375, },
		4241	{ 15625, 14500, },
		4242	{ 15750, 14625, },
		4243	{ 15875, 14750, },
		4244	{ 16000, 14875, },
		4245	{ 16125, 15000, },
		4246	};
		4247	if (dev_priv->info->is_mobile)
		4248	return v_table[pxvid].vm;
		4249	else
		4250	return v_table[pxvid].vd;
		4251	}
		4252
		4253	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		4254	{
		4255	struct timespec now, diff1;
		4256	u64 diff;
		4257	unsigned long diffms;
		4258	u32 count;
		4259
		4260	assert_spin_locked(&mchdev_lock);
		4261
		4262	getrawmonotonic(&now);
		4263	diff1 = timespec_sub(now, dev_priv->ips.last_time2);
		4264
		4265	/* Don't divide by 0 */
		4266	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
		4267	if (!diffms)
		4268	return;
		4269
		4270	count = I915_READ(GFXEC);
		4271
		4272	if (count < dev_priv->ips.last_count2) {
		4273	diff = ~0UL - dev_priv->ips.last_count2;
		4274	diff += count;
		4275	} else {
		4276	diff = count - dev_priv->ips.last_count2;
		4277	}
		4278
		4279	dev_priv->ips.last_count2 = count;
		4280	dev_priv->ips.last_time2 = now;
		4281
		4282	/* More magic constants... */
		4283	diff = diff * 1181;
		4284	diff = div_u64(diff, diffms * 10);
		4285	dev_priv->ips.gfx_power = diff;
		4286	}
		4287
		4288	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		4289	{
		4290	if (dev_priv->info->gen != 5)
		4291	return;
		4292
		4293	spin_lock_irq(&mchdev_lock);
		4294
		4295	__i915_update_gfx_val(dev_priv);
		4296
		4297	spin_unlock_irq(&mchdev_lock);
		4298	}
		4299
		4300	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		4301	{
		4302	unsigned long t, corr, state1, corr2, state2;
		4303	u32 pxvid, ext_v;
		4304
		4305	assert_spin_locked(&mchdev_lock);
		4306
		4307	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
		4308	pxvid = (pxvid >> 24) & 0x7f;
		4309	ext_v = pvid_to_extvid(dev_priv, pxvid);
		4310
		4311	state1 = ext_v;
		4312
		4313	t = i915_mch_val(dev_priv);
		4314
		4315	/* Revel in the empirically derived constants */
		4316
		4317	/* Correction factor in 1/100000 units */
		4318	if (t > 80)
		4319	corr = ((t * 2349) + 135940);
		4320	else if (t >= 50)
		4321	corr = ((t * 964) + 29317);
		4322	else /* < 50 */
		4323	corr = ((t * 301) + 1004);
		4324
		4325	corr = corr * ((150142 * state1) / 10000 - 78642);
		4326	corr /= 100000;
		4327	corr2 = (corr * dev_priv->ips.corr);
		4328
		4329	state2 = (corr2 * state1) / 10000;
		4330	state2 /= 100; /* convert to mW */
		4331
		4332	__i915_update_gfx_val(dev_priv);
		4333
		4334	return dev_priv->ips.gfx_power + state2;
		4335	}
		4336
		4337	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		4338	{
		4339	unsigned long val;
		4340
		4341	if (dev_priv->info->gen != 5)
		4342	return 0;
		4343
		4344	spin_lock_irq(&mchdev_lock);
		4345
		4346	val = __i915_gfx_val(dev_priv);
		4347
		4348	spin_unlock_irq(&mchdev_lock);
		4349
		4350	return val;
		4351	}
		4352
		4353	/**
		4354	* i915_read_mch_val - return value for IPS use
		4355	*
		4356	* Calculate and return a value for the IPS driver to use when deciding whether
		4357	* we have thermal and power headroom to increase CPU or GPU power budget.
		4358	*/
		4359	unsigned long i915_read_mch_val(void)
		4360	{
		4361	struct drm_i915_private *dev_priv;
		4362	unsigned long chipset_val, graphics_val, ret = 0;
		4363
		4364	spin_lock_irq(&mchdev_lock);
		4365	if (!i915_mch_dev)
		4366	goto out_unlock;
		4367	dev_priv = i915_mch_dev;
		4368
		4369	chipset_val = __i915_chipset_val(dev_priv);
		4370	graphics_val = __i915_gfx_val(dev_priv);
		4371
		4372	ret = chipset_val + graphics_val;
		4373
		4374	out_unlock:
		4375	spin_unlock_irq(&mchdev_lock);
		4376
		4377	return ret;
		4378	}
		4379	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		4380
		4381	/**
		4382	* i915_gpu_raise - raise GPU frequency limit
		4383	*
		4384	* Raise the limit; IPS indicates we have thermal headroom.
		4385	*/
		4386	bool i915_gpu_raise(void)
		4387	{
		4388	struct drm_i915_private *dev_priv;
		4389	bool ret = true;
		4390
		4391	spin_lock_irq(&mchdev_lock);
		4392	if (!i915_mch_dev) {
		4393	ret = false;
		4394	goto out_unlock;
		4395	}
		4396	dev_priv = i915_mch_dev;
		4397
		4398	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		4399	dev_priv->ips.max_delay--;
		4400
		4401	out_unlock:
		4402	spin_unlock_irq(&mchdev_lock);
		4403
		4404	return ret;
		4405	}
		4406	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		4407
		4408	/**
		4409	* i915_gpu_lower - lower GPU frequency limit
		4410	*
		4411	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		4412	* frequency maximum.
		4413	*/
		4414	bool i915_gpu_lower(void)
		4415	{
		4416	struct drm_i915_private *dev_priv;
		4417	bool ret = true;
		4418
		4419	spin_lock_irq(&mchdev_lock);
		4420	if (!i915_mch_dev) {
		4421	ret = false;
		4422	goto out_unlock;
		4423	}
		4424	dev_priv = i915_mch_dev;
		4425
		4426	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		4427	dev_priv->ips.max_delay++;
		4428
		4429	out_unlock:
		4430	spin_unlock_irq(&mchdev_lock);
		4431
		4432	return ret;
		4433	}
		4434	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		4435
		4436	/**
		4437	* i915_gpu_busy - indicate GPU business to IPS
		4438	*
		4439	* Tell the IPS driver whether or not the GPU is busy.
		4440	*/
		4441	bool i915_gpu_busy(void)
		4442	{
		4443	struct drm_i915_private *dev_priv;
		4444	struct intel_ring_buffer *ring;
		4445	bool ret = false;
		4446	int i;
		4447
		4448	spin_lock_irq(&mchdev_lock);
		4449	if (!i915_mch_dev)
		4450	goto out_unlock;
		4451	dev_priv = i915_mch_dev;
		4452
		4453	for_each_ring(ring, dev_priv, i)
		4454	ret \|= !list_empty(&ring->request_list);
		4455
		4456	out_unlock:
		4457	spin_unlock_irq(&mchdev_lock);
		4458
		4459	return ret;
		4460	}
		4461	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		4462
		4463	/**
		4464	* i915_gpu_turbo_disable - disable graphics turbo
		4465	*
		4466	* Disable graphics turbo by resetting the max frequency and setting the
		4467	* current frequency to the default.
		4468	*/
		4469	bool i915_gpu_turbo_disable(void)
		4470	{
		4471	struct drm_i915_private *dev_priv;
		4472	bool ret = true;
		4473
		4474	spin_lock_irq(&mchdev_lock);
		4475	if (!i915_mch_dev) {
		4476	ret = false;
		4477	goto out_unlock;
		4478	}
		4479	dev_priv = i915_mch_dev;
		4480
		4481	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		4482
		4483	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		4484	ret = false;
		4485
		4486	out_unlock:
		4487	spin_unlock_irq(&mchdev_lock);
		4488
		4489	return ret;
		4490	}
		4491	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		4492
		4493	/**
		4494	* Tells the intel_ips driver that the i915 driver is now loaded, if
		4495	* IPS got loaded first.
		4496	*
		4497	* This awkward dance is so that neither module has to depend on the
		4498	* other in order for IPS to do the appropriate communication of
		4499	* GPU turbo limits to i915.
		4500	*/
		4501	static void
		4502	ips_ping_for_i915_load(void)
		4503	{
		4504	void (*link)(void);
		4505
		4506	// link = symbol_get(ips_link_to_i915_driver);
		4507	// if (link) {
		4508	// link();
		4509	// symbol_put(ips_link_to_i915_driver);
		4510	// }
		4511	}
		4512
		4513	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		4514	{
		4515	/* We only register the i915 ips part with intel-ips once everything is
		4516	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		4517	spin_lock_irq(&mchdev_lock);
		4518	i915_mch_dev = dev_priv;
		4519	spin_unlock_irq(&mchdev_lock);
		4520
		4521	ips_ping_for_i915_load();
		4522	}
		4523
		4524	void intel_gpu_ips_teardown(void)
		4525	{
		4526	spin_lock_irq(&mchdev_lock);
		4527	i915_mch_dev = NULL;
		4528	spin_unlock_irq(&mchdev_lock);
		4529	}
		4530	static void intel_init_emon(struct drm_device *dev)
		4531	{
		4532	struct drm_i915_private *dev_priv = dev->dev_private;
		4533	u32 lcfuse;
		4534	u8 pxw[16];
		4535	int i;
		4536
		4537	/* Disable to program */
		4538	I915_WRITE(ECR, 0);
		4539	POSTING_READ(ECR);
		4540
		4541	/* Program energy weights for various events */
		4542	I915_WRITE(SDEW, 0x15040d00);
		4543	I915_WRITE(CSIEW0, 0x007f0000);
		4544	I915_WRITE(CSIEW1, 0x1e220004);
		4545	I915_WRITE(CSIEW2, 0x04000004);
		4546
		4547	for (i = 0; i < 5; i++)
		4548	I915_WRITE(PEW + (i * 4), 0);
		4549	for (i = 0; i < 3; i++)
		4550	I915_WRITE(DEW + (i * 4), 0);
		4551
		4552	/* Program P-state weights to account for frequency power adjustment */
		4553	for (i = 0; i < 16; i++) {
		4554	u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
		4555	unsigned long freq = intel_pxfreq(pxvidfreq);
		4556	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		4557	PXVFREQ_PX_SHIFT;
		4558	unsigned long val;
		4559
		4560	val = vid * vid;
		4561	val *= (freq / 1000);
		4562	val *= 255;
		4563	val /= (127127900);
		4564	if (val > 0xff)
		4565	DRM_ERROR("bad pxval: %ld\n", val);
		4566	pxw[i] = val;
		4567	}
		4568	/* Render standby states get 0 weight */
		4569	pxw[14] = 0;
		4570	pxw[15] = 0;
		4571
		4572	for (i = 0; i < 4; i++) {
		4573	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		4574	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
		4575	I915_WRITE(PXW + (i * 4), val);
		4576	}
		4577
		4578	/* Adjust magic regs to magic values (more experimental results) */
		4579	I915_WRITE(OGW0, 0);
		4580	I915_WRITE(OGW1, 0);
		4581	I915_WRITE(EG0, 0x00007f00);
		4582	I915_WRITE(EG1, 0x0000000e);
		4583	I915_WRITE(EG2, 0x000e0000);
		4584	I915_WRITE(EG3, 0x68000300);
		4585	I915_WRITE(EG4, 0x42000000);
		4586	I915_WRITE(EG5, 0x00140031);
		4587	I915_WRITE(EG6, 0);
		4588	I915_WRITE(EG7, 0);
		4589
		4590	for (i = 0; i < 8; i++)
		4591	I915_WRITE(PXWL + (i * 4), 0);
		4592
		4593	/* Enable PMON + select events */
		4594	I915_WRITE(ECR, 0x80000019);
		4595
		4596	lcfuse = I915_READ(LCFUSE02);
		4597
		4598	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		4599	}
		4600
		4601	void intel_disable_gt_powersave(struct drm_device *dev)
		4602	{
3243	Serge	4603	struct drm_i915_private *dev_priv = dev->dev_private;
		4604
4104	Serge	4605	/* Interrupts should be disabled already to avoid re-arming. */
		4606	WARN_ON(dev->irq_enabled);
		4607
3031	serge	4608	if (IS_IRONLAKE_M(dev)) {
		4609	ironlake_disable_drps(dev);
		4610	ironlake_disable_rc6(dev);
4293	Serge	4611	} else if (INTEL_INFO(dev)->gen >= 6) {
		4612	cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
		4613	cancel_work_sync(&dev_priv->rps.work);
		4614	if (IS_VALLEYVIEW(dev))
		4615	cancel_delayed_work_sync(&dev_priv->rps.vlv_work);
3482	Serge	4616	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	4617	if (IS_VALLEYVIEW(dev))
		4618	valleyview_disable_rps(dev);
		4619	else
3031	serge	4620	gen6_disable_rps(dev);
3480	Serge	4621	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	4622	}
		4623	}
		4624
3482	Serge	4625	static void intel_gen6_powersave_work(struct work_struct *work)
		4626	{
		4627	struct drm_i915_private *dev_priv =
		4628	container_of(work, struct drm_i915_private,
		4629	rps.delayed_resume_work.work);
		4630	struct drm_device *dev = dev_priv->dev;
		4631
		4632	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	4633
		4634	if (IS_VALLEYVIEW(dev)) {
		4635	valleyview_enable_rps(dev);
		4636	} else {
3482	Serge	4637	gen6_enable_rps(dev);
		4638	gen6_update_ring_freq(dev);
4104	Serge	4639	}
3482	Serge	4640	mutex_unlock(&dev_priv->rps.hw_lock);
		4641	}
		4642
3031	serge	4643	void intel_enable_gt_powersave(struct drm_device *dev)
		4644	{
3243	Serge	4645	struct drm_i915_private *dev_priv = dev->dev_private;
		4646
3031	serge	4647	if (IS_IRONLAKE_M(dev)) {
		4648	ironlake_enable_drps(dev);
		4649	ironlake_enable_rc6(dev);
		4650	intel_init_emon(dev);
4104	Serge	4651	} else if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {
3243	Serge	4652	/*
		4653	* PCU communication is slow and this doesn't need to be
		4654	* done at any specific time, so do this out of our fast path
		4655	* to make resume and init faster.
		4656	*/
3482	Serge	4657	schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		4658	round_jiffies_up_relative(HZ));
3031	serge	4659	}
		4660	}
		4661
3243	Serge	4662	static void ibx_init_clock_gating(struct drm_device *dev)
		4663	{
		4664	struct drm_i915_private *dev_priv = dev->dev_private;
		4665
		4666	/*
		4667	* On Ibex Peak and Cougar Point, we need to disable clock
		4668	* gating for the panel power sequencer or it will fail to
		4669	* start up when no ports are active.
		4670	*/
		4671	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		4672	}
		4673
4104	Serge	4674	static void g4x_disable_trickle_feed(struct drm_device *dev)
		4675	{
		4676	struct drm_i915_private *dev_priv = dev->dev_private;
		4677	int pipe;
		4678
		4679	for_each_pipe(pipe) {
		4680	I915_WRITE(DSPCNTR(pipe),
		4681	I915_READ(DSPCNTR(pipe)) \|
		4682	DISPPLANE_TRICKLE_FEED_DISABLE);
		4683	intel_flush_display_plane(dev_priv, pipe);
		4684	}
		4685	}
		4686
3031	serge	4687	static void ironlake_init_clock_gating(struct drm_device *dev)
		4688	{
		4689	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	4690	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	4691
4104	Serge	4692	/*
		4693	* Required for FBC
		4694	* WaFbcDisableDpfcClockGating:ilk
		4695	*/
3243	Serge	4696	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		4697	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		4698	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	4699
		4700	I915_WRITE(PCH_3DCGDIS0,
		4701	MARIUNIT_CLOCK_GATE_DISABLE \|
		4702	SVSMUNIT_CLOCK_GATE_DISABLE);
		4703	I915_WRITE(PCH_3DCGDIS1,
		4704	VFMUNIT_CLOCK_GATE_DISABLE);
		4705
		4706	/*
		4707	* According to the spec the following bits should be set in
		4708	* order to enable memory self-refresh
		4709	* The bit 22/21 of 0x42004
		4710	* The bit 5 of 0x42020
		4711	* The bit 15 of 0x45000
		4712	*/
		4713	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4714	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4715	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	4716	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	4717	I915_WRITE(DISP_ARB_CTL,
		4718	(I915_READ(DISP_ARB_CTL) \|
		4719	DISP_FBC_WM_DIS));
		4720	I915_WRITE(WM3_LP_ILK, 0);
		4721	I915_WRITE(WM2_LP_ILK, 0);
		4722	I915_WRITE(WM1_LP_ILK, 0);
		4723
		4724	/*
		4725	* Based on the document from hardware guys the following bits
		4726	* should be set unconditionally in order to enable FBC.
		4727	* The bit 22 of 0x42000
		4728	* The bit 22 of 0x42004
		4729	* The bit 7,8,9 of 0x42020.
		4730	*/
		4731	if (IS_IRONLAKE_M(dev)) {
4104	Serge	4732	/* WaFbcAsynchFlipDisableFbcQueue:ilk */
3031	serge	4733	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		4734	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		4735	ILK_FBCQ_DIS);
		4736	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4737	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4738	ILK_DPARB_GATE);
		4739	}
		4740
3243	Serge	4741	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		4742
3031	serge	4743	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4744	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4745	ILK_ELPIN_409_SELECT);
		4746	I915_WRITE(_3D_CHICKEN2,
		4747	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		4748	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	4749
4104	Serge	4750	/* WaDisableRenderCachePipelinedFlush:ilk */
3243	Serge	4751	I915_WRITE(CACHE_MODE_0,
		4752	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		4753
4104	Serge	4754	g4x_disable_trickle_feed(dev);
		4755
3243	Serge	4756	ibx_init_clock_gating(dev);
3031	serge	4757	}
		4758
3243	Serge	4759	static void cpt_init_clock_gating(struct drm_device *dev)
		4760	{
		4761	struct drm_i915_private *dev_priv = dev->dev_private;
		4762	int pipe;
3746	Serge	4763	uint32_t val;
3243	Serge	4764
		4765	/*
		4766	* On Ibex Peak and Cougar Point, we need to disable clock
		4767	* gating for the panel power sequencer or it will fail to
		4768	* start up when no ports are active.
		4769	*/
4280	Serge	4770	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE \|
		4771	PCH_DPLUNIT_CLOCK_GATE_DISABLE \|
		4772	PCH_CPUNIT_CLOCK_GATE_DISABLE);
3243	Serge	4773	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		4774	DPLS_EDP_PPS_FIX_DIS);
		4775	/* The below fixes the weird display corruption, a few pixels shifted
		4776	* downward, on (only) LVDS of some HP laptops with IVY.
		4777	*/
3746	Serge	4778	for_each_pipe(pipe) {
		4779	val = I915_READ(TRANS_CHICKEN2(pipe));
		4780	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		4781	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4104	Serge	4782	if (dev_priv->vbt.fdi_rx_polarity_inverted)
3746	Serge	4783	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		4784	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		4785	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		4786	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		4787	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		4788	}
3243	Serge	4789	/* WADP0ClockGatingDisable */
		4790	for_each_pipe(pipe) {
		4791	I915_WRITE(TRANS_CHICKEN1(pipe),
		4792	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		4793	}
		4794	}
		4795
3480	Serge	4796	static void gen6_check_mch_setup(struct drm_device *dev)
		4797	{
		4798	struct drm_i915_private *dev_priv = dev->dev_private;
		4799	uint32_t tmp;
		4800
		4801	tmp = I915_READ(MCH_SSKPD);
		4802	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) {
		4803	DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp);
		4804	DRM_INFO("This can cause pipe underruns and display issues.\n");
		4805	DRM_INFO("Please upgrade your BIOS to fix this.\n");
		4806	}
		4807	}
		4808
3031	serge	4809	static void gen6_init_clock_gating(struct drm_device *dev)
		4810	{
		4811	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	4812	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	4813
3243	Serge	4814	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	4815
		4816	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4817	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4818	ILK_ELPIN_409_SELECT);
		4819
4104	Serge	4820	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3243	Serge	4821	I915_WRITE(_3D_CHICKEN,
		4822	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		4823
4104	Serge	4824	/* WaSetupGtModeTdRowDispatch:snb */
3243	Serge	4825	if (IS_SNB_GT1(dev))
		4826	I915_WRITE(GEN6_GT_MODE,
		4827	_MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
		4828
3031	serge	4829	I915_WRITE(WM3_LP_ILK, 0);
		4830	I915_WRITE(WM2_LP_ILK, 0);
		4831	I915_WRITE(WM1_LP_ILK, 0);
		4832
		4833	I915_WRITE(CACHE_MODE_0,
		4834	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		4835
		4836	I915_WRITE(GEN6_UCGCTL1,
		4837	I915_READ(GEN6_UCGCTL1) \|
		4838	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		4839	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		4840
		4841	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		4842	* gating disable must be set. Failure to set it results in
		4843	* flickering pixels due to Z write ordering failures after
		4844	* some amount of runtime in the Mesa "fire" demo, and Unigine
		4845	* Sanctuary and Tropics, and apparently anything else with
		4846	* alpha test or pixel discard.
		4847	*
		4848	* According to the spec, bit 11 (RCCUNIT) must also be set,
		4849	* but we didn't debug actual testcases to find it out.
		4850	*
4104	Serge	4851	* Also apply WaDisableVDSUnitClockGating:snb and
		4852	* WaDisableRCPBUnitClockGating:snb.
3031	serge	4853	*/
		4854	I915_WRITE(GEN6_UCGCTL2,
		4855	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		4856	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		4857	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		4858
		4859	/* Bspec says we need to always set all mask bits. */
		4860	I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) \|
		4861	_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
		4862
		4863	/*
		4864	* According to the spec the following bits should be
		4865	* set in order to enable memory self-refresh and fbc:
		4866	* The bit21 and bit22 of 0x42000
		4867	* The bit21 and bit22 of 0x42004
		4868	* The bit5 and bit7 of 0x42020
		4869	* The bit14 of 0x70180
		4870	* The bit14 of 0x71180
4104	Serge	4871	*
		4872	* WaFbcAsynchFlipDisableFbcQueue:snb
3031	serge	4873	*/
		4874	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		4875	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		4876	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		4877	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4878	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4879	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	4880	I915_WRITE(ILK_DSPCLK_GATE_D,
		4881	I915_READ(ILK_DSPCLK_GATE_D) \|
		4882	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		4883	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	4884
4104	Serge	4885	g4x_disable_trickle_feed(dev);
3031	serge	4886
		4887	/* The default value should be 0x200 according to docs, but the two
		4888	* platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
		4889	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
		4890	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
3243	Serge	4891
		4892	cpt_init_clock_gating(dev);
3480	Serge	4893
		4894	gen6_check_mch_setup(dev);
3031	serge	4895	}
		4896
		4897	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		4898	{
		4899	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		4900
		4901	reg &= ~GEN7_FF_SCHED_MASK;
		4902	reg \|= GEN7_FF_TS_SCHED_HW;
		4903	reg \|= GEN7_FF_VS_SCHED_HW;
		4904	reg \|= GEN7_FF_DS_SCHED_HW;
		4905
3480	Serge	4906	if (IS_HASWELL(dev_priv->dev))
		4907	reg &= ~GEN7_FF_VS_REF_CNT_FFME;
		4908
3031	serge	4909	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		4910	}
		4911
3243	Serge	4912	static void lpt_init_clock_gating(struct drm_device *dev)
		4913	{
		4914	struct drm_i915_private *dev_priv = dev->dev_private;
		4915
		4916	/*
		4917	* TODO: this bit should only be enabled when really needed, then
		4918	* disabled when not needed anymore in order to save power.
		4919	*/
		4920	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
		4921	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		4922	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		4923	PCH_LP_PARTITION_LEVEL_DISABLE);
4104	Serge	4924
		4925	/* WADPOClockGatingDisable:hsw */
		4926	I915_WRITE(_TRANSA_CHICKEN1,
		4927	I915_READ(_TRANSA_CHICKEN1) \|
		4928	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3243	Serge	4929	}
		4930
4104	Serge	4931	static void lpt_suspend_hw(struct drm_device *dev)
		4932	{
		4933	struct drm_i915_private *dev_priv = dev->dev_private;
		4934
		4935	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
		4936	uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
		4937
		4938	val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		4939	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
		4940	}
		4941	}
		4942
3031	serge	4943	static void haswell_init_clock_gating(struct drm_device *dev)
		4944	{
		4945	struct drm_i915_private *dev_priv = dev->dev_private;
		4946
		4947	I915_WRITE(WM3_LP_ILK, 0);
		4948	I915_WRITE(WM2_LP_ILK, 0);
		4949	I915_WRITE(WM1_LP_ILK, 0);
		4950
		4951	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	4952	* This implements the WaDisableRCZUnitClockGating:hsw workaround.
3031	serge	4953	*/
		4954	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
		4955
4104	Serge	4956	/* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */
3031	serge	4957	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		4958	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		4959
4104	Serge	4960	/* WaApplyL3ControlAndL3ChickenMode:hsw */
3031	serge	4961	I915_WRITE(GEN7_L3CNTLREG1,
		4962	GEN7_WA_FOR_GEN7_L3_CONTROL);
		4963	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		4964	GEN7_WA_L3_CHICKEN_MODE);
		4965
4104	Serge	4966	/* L3 caching of data atomics doesn't work -- disable it. */
		4967	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
		4968	I915_WRITE(HSW_ROW_CHICKEN3,
		4969	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
		4970
		4971	/* This is required by WaCatErrorRejectionIssue:hsw */
3031	serge	4972	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		4973	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		4974	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		4975
4104	Serge	4976	/* WaVSRefCountFullforceMissDisable:hsw */
3031	serge	4977	gen7_setup_fixed_func_scheduler(dev_priv);
		4978
4104	Serge	4979	/* WaDisable4x2SubspanOptimization:hsw */
3031	serge	4980	I915_WRITE(CACHE_MODE_1,
		4981	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		4982
4104	Serge	4983	/* WaSwitchSolVfFArbitrationPriority:hsw */
3746	Serge	4984	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		4985
4104	Serge	4986	/* WaRsPkgCStateDisplayPMReq:hsw */
		4987	I915_WRITE(CHICKEN_PAR1_1,
		4988	I915_READ(CHICKEN_PAR1_1) \| FORCE_ARB_IDLE_PLANES);
3031	serge	4989
3243	Serge	4990	lpt_init_clock_gating(dev);
3031	serge	4991	}
		4992
		4993	static void ivybridge_init_clock_gating(struct drm_device *dev)
		4994	{
		4995	struct drm_i915_private *dev_priv = dev->dev_private;
		4996	uint32_t snpcr;
		4997
		4998	I915_WRITE(WM3_LP_ILK, 0);
		4999	I915_WRITE(WM2_LP_ILK, 0);
		5000	I915_WRITE(WM1_LP_ILK, 0);
		5001
3243	Serge	5002	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	5003
4104	Serge	5004	/* WaDisableEarlyCull:ivb */
3243	Serge	5005	I915_WRITE(_3D_CHICKEN3,
		5006	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		5007
4104	Serge	5008	/* WaDisableBackToBackFlipFix:ivb */
3031	serge	5009	I915_WRITE(IVB_CHICKEN3,
		5010	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		5011	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		5012
4104	Serge	5013	/* WaDisablePSDDualDispatchEnable:ivb */
3243	Serge	5014	if (IS_IVB_GT1(dev))
		5015	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		5016	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		5017	else
		5018	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
		5019	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		5020
4104	Serge	5021	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
3031	serge	5022	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		5023	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		5024
4104	Serge	5025	/* WaApplyL3ControlAndL3ChickenMode:ivb */
3031	serge	5026	I915_WRITE(GEN7_L3CNTLREG1,
		5027	GEN7_WA_FOR_GEN7_L3_CONTROL);
		5028	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		5029	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	5030	if (IS_IVB_GT1(dev))
		5031	I915_WRITE(GEN7_ROW_CHICKEN2,
		5032	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		5033	else
		5034	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		5035	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3031	serge	5036
3243	Serge	5037
4104	Serge	5038	/* WaForceL3Serialization:ivb */
3243	Serge	5039	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		5040	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		5041
3031	serge	5042	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		5043	* gating disable must be set. Failure to set it results in
		5044	* flickering pixels due to Z write ordering failures after
		5045	* some amount of runtime in the Mesa "fire" demo, and Unigine
		5046	* Sanctuary and Tropics, and apparently anything else with
		5047	* alpha test or pixel discard.
		5048	*
		5049	* According to the spec, bit 11 (RCCUNIT) must also be set,
		5050	* but we didn't debug actual testcases to find it out.
		5051	*
		5052	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	5053	* This implements the WaDisableRCZUnitClockGating:ivb workaround.
3031	serge	5054	*/
		5055	I915_WRITE(GEN6_UCGCTL2,
		5056	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		5057	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		5058
4104	Serge	5059	/* This is required by WaCatErrorRejectionIssue:ivb */
3031	serge	5060	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		5061	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		5062	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		5063
4104	Serge	5064	g4x_disable_trickle_feed(dev);
3031	serge	5065
4104	Serge	5066	/* WaVSRefCountFullforceMissDisable:ivb */
3031	serge	5067	gen7_setup_fixed_func_scheduler(dev_priv);
		5068
4104	Serge	5069	/* WaDisable4x2SubspanOptimization:ivb */
3031	serge	5070	I915_WRITE(CACHE_MODE_1,
		5071	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		5072
		5073	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		5074	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		5075	snpcr \|= GEN6_MBC_SNPCR_MED;
		5076	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	5077
3746	Serge	5078	if (!HAS_PCH_NOP(dev))
3243	Serge	5079	cpt_init_clock_gating(dev);
3480	Serge	5080
		5081	gen6_check_mch_setup(dev);
3031	serge	5082	}
		5083
		5084	static void valleyview_init_clock_gating(struct drm_device *dev)
		5085	{
		5086	struct drm_i915_private *dev_priv = dev->dev_private;
		5087
4104	Serge	5088	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	5089
4104	Serge	5090	/* WaDisableEarlyCull:vlv */
3243	Serge	5091	I915_WRITE(_3D_CHICKEN3,
		5092	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		5093
4104	Serge	5094	/* WaDisableBackToBackFlipFix:vlv */
3031	serge	5095	I915_WRITE(IVB_CHICKEN3,
		5096	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		5097	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		5098
4104	Serge	5099	/* WaDisablePSDDualDispatchEnable:vlv */
3243	Serge	5100	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	5101	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		5102	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	5103
4104	Serge	5104	/* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */
3031	serge	5105	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		5106	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		5107
4104	Serge	5108	/* WaApplyL3ControlAndL3ChickenMode:vlv */
3243	Serge	5109	I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) \| GEN7_L3AGDIS);
3031	serge	5110	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
		5111
4104	Serge	5112	/* WaForceL3Serialization:vlv */
3243	Serge	5113	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		5114	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		5115
4104	Serge	5116	/* WaDisableDopClockGating:vlv */
3243	Serge	5117	I915_WRITE(GEN7_ROW_CHICKEN2,
		5118	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		5119
4104	Serge	5120	/* This is required by WaCatErrorRejectionIssue:vlv */
3031	serge	5121	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		5122	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		5123	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		5124
		5125	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		5126	* gating disable must be set. Failure to set it results in
		5127	* flickering pixels due to Z write ordering failures after
		5128	* some amount of runtime in the Mesa "fire" demo, and Unigine
		5129	* Sanctuary and Tropics, and apparently anything else with
		5130	* alpha test or pixel discard.
		5131	*
		5132	* According to the spec, bit 11 (RCCUNIT) must also be set,
		5133	* but we didn't debug actual testcases to find it out.
		5134	*
		5135	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	5136	* This implements the WaDisableRCZUnitClockGating:vlv workaround.
3031	serge	5137	*
4104	Serge	5138	* Also apply WaDisableVDSUnitClockGating:vlv and
		5139	* WaDisableRCPBUnitClockGating:vlv.
3031	serge	5140	*/
		5141	I915_WRITE(GEN6_UCGCTL2,
		5142	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		5143	GEN7_TDLUNIT_CLOCK_GATE_DISABLE \|
		5144	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		5145	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		5146	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		5147
		5148	I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
		5149
4104	Serge	5150	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
3031	serge	5151
		5152	I915_WRITE(CACHE_MODE_1,
		5153	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		5154
		5155	/*
4104	Serge	5156	* WaDisableVLVClockGating_VBIIssue:vlv
3243	Serge	5157	* Disable clock gating on th GCFG unit to prevent a delay
		5158	* in the reporting of vblank events.
		5159	*/
3746	Serge	5160	I915_WRITE(VLV_GUNIT_CLOCK_GATE, 0xffffffff);
		5161
		5162	/* Conservative clock gating settings for now */
		5163	I915_WRITE(0x9400, 0xffffffff);
		5164	I915_WRITE(0x9404, 0xffffffff);
		5165	I915_WRITE(0x9408, 0xffffffff);
		5166	I915_WRITE(0x940c, 0xffffffff);
		5167	I915_WRITE(0x9410, 0xffffffff);
		5168	I915_WRITE(0x9414, 0xffffffff);
		5169	I915_WRITE(0x9418, 0xffffffff);
3031	serge	5170	}
		5171
		5172	static void g4x_init_clock_gating(struct drm_device *dev)
		5173	{
		5174	struct drm_i915_private *dev_priv = dev->dev_private;
		5175	uint32_t dspclk_gate;
		5176
		5177	I915_WRITE(RENCLK_GATE_D1, 0);
		5178	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		5179	GS_UNIT_CLOCK_GATE_DISABLE \|
		5180	CL_UNIT_CLOCK_GATE_DISABLE);
		5181	I915_WRITE(RAMCLK_GATE_D, 0);
		5182	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		5183	OVRUNIT_CLOCK_GATE_DISABLE \|
		5184	OVCUNIT_CLOCK_GATE_DISABLE;
		5185	if (IS_GM45(dev))
		5186	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		5187	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	5188
		5189	/* WaDisableRenderCachePipelinedFlush */
		5190	I915_WRITE(CACHE_MODE_0,
		5191	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4104	Serge	5192
		5193	g4x_disable_trickle_feed(dev);
3031	serge	5194	}
		5195
		5196	static void crestline_init_clock_gating(struct drm_device *dev)
		5197	{
		5198	struct drm_i915_private *dev_priv = dev->dev_private;
		5199
		5200	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		5201	I915_WRITE(RENCLK_GATE_D2, 0);
		5202	I915_WRITE(DSPCLK_GATE_D, 0);
		5203	I915_WRITE(RAMCLK_GATE_D, 0);
		5204	I915_WRITE16(DEUC, 0);
4104	Serge	5205	I915_WRITE(MI_ARB_STATE,
		5206	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	5207	}
		5208
		5209	static void broadwater_init_clock_gating(struct drm_device *dev)
		5210	{
		5211	struct drm_i915_private *dev_priv = dev->dev_private;
		5212
		5213	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		5214	I965_RCC_CLOCK_GATE_DISABLE \|
		5215	I965_RCPB_CLOCK_GATE_DISABLE \|
		5216	I965_ISC_CLOCK_GATE_DISABLE \|
		5217	I965_FBC_CLOCK_GATE_DISABLE);
		5218	I915_WRITE(RENCLK_GATE_D2, 0);
4104	Serge	5219	I915_WRITE(MI_ARB_STATE,
		5220	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	5221	}
		5222
		5223	static void gen3_init_clock_gating(struct drm_device *dev)
		5224	{
		5225	struct drm_i915_private *dev_priv = dev->dev_private;
		5226	u32 dstate = I915_READ(D_STATE);
		5227
		5228	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		5229	DSTATE_DOT_CLOCK_GATING;
		5230	I915_WRITE(D_STATE, dstate);
		5231
		5232	if (IS_PINEVIEW(dev))
		5233	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		5234
		5235	/* IIR "flip pending" means done if this bit is set */
		5236	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
		5237	}
		5238
		5239	static void i85x_init_clock_gating(struct drm_device *dev)
		5240	{
		5241	struct drm_i915_private *dev_priv = dev->dev_private;
		5242
		5243	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
		5244	}
		5245
		5246	static void i830_init_clock_gating(struct drm_device *dev)
		5247	{
		5248	struct drm_i915_private *dev_priv = dev->dev_private;
		5249
		5250	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
		5251	}
		5252
		5253	void intel_init_clock_gating(struct drm_device *dev)
		5254	{
		5255	struct drm_i915_private *dev_priv = dev->dev_private;
		5256
		5257	dev_priv->display.init_clock_gating(dev);
		5258	}
		5259
4104	Serge	5260	void intel_suspend_hw(struct drm_device *dev)
		5261	{
		5262	if (HAS_PCH_LPT(dev))
		5263	lpt_suspend_hw(dev);
		5264	}
		5265
3746	Serge	5266	/**
		5267	* We should only use the power well if we explicitly asked the hardware to
		5268	* enable it, so check if it's enabled and also check if we've requested it to
		5269	* be enabled.
		5270	*/
4104	Serge	5271	bool intel_display_power_enabled(struct drm_device *dev,
		5272	enum intel_display_power_domain domain)
3746	Serge	5273	{
		5274	struct drm_i915_private *dev_priv = dev->dev_private;
		5275
4104	Serge	5276	if (!HAS_POWER_WELL(dev))
		5277	return true;
		5278
		5279	switch (domain) {
		5280	case POWER_DOMAIN_PIPE_A:
		5281	case POWER_DOMAIN_TRANSCODER_EDP:
		5282	return true;
		5283	case POWER_DOMAIN_PIPE_B:
		5284	case POWER_DOMAIN_PIPE_C:
		5285	case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
		5286	case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
		5287	case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
		5288	case POWER_DOMAIN_TRANSCODER_A:
		5289	case POWER_DOMAIN_TRANSCODER_B:
		5290	case POWER_DOMAIN_TRANSCODER_C:
3746	Serge	5291	return I915_READ(HSW_PWR_WELL_DRIVER) ==
4104	Serge	5292	(HSW_PWR_WELL_ENABLE_REQUEST \| HSW_PWR_WELL_STATE_ENABLED);
		5293	default:
		5294	BUG();
		5295	}
3746	Serge	5296	}
		5297
4104	Serge	5298	static void __intel_set_power_well(struct drm_device *dev, bool enable)
3031	serge	5299	{
		5300	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	5301	bool is_enabled, enable_requested;
		5302	uint32_t tmp;
3031	serge	5303
3480	Serge	5304	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
4104	Serge	5305	is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
		5306	enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
3031	serge	5307
3480	Serge	5308	if (enable) {
		5309	if (!enable_requested)
4104	Serge	5310	I915_WRITE(HSW_PWR_WELL_DRIVER,
		5311	HSW_PWR_WELL_ENABLE_REQUEST);
3031	serge	5312
3480	Serge	5313	if (!is_enabled) {
		5314	DRM_DEBUG_KMS("Enabling power well\n");
		5315	if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
4104	Serge	5316	HSW_PWR_WELL_STATE_ENABLED), 20))
3480	Serge	5317	DRM_ERROR("Timeout enabling power well\n");
3031	serge	5318	}
3480	Serge	5319	} else {
		5320	if (enable_requested) {
4104	Serge	5321	unsigned long irqflags;
		5322	enum pipe p;
		5323
3480	Serge	5324	I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
4104	Serge	5325	POSTING_READ(HSW_PWR_WELL_DRIVER);
3480	Serge	5326	DRM_DEBUG_KMS("Requesting to disable the power well\n");
4104	Serge	5327
		5328	/*
		5329	* After this, the registers on the pipes that are part
		5330	* of the power well will become zero, so we have to
		5331	* adjust our counters according to that.
		5332	*
		5333	* FIXME: Should we do this in general in
		5334	* drm_vblank_post_modeset?
		5335	*/
		5336	spin_lock_irqsave(&dev->vbl_lock, irqflags);
		5337	for_each_pipe(p)
		5338	if (p != PIPE_A)
		5339	dev->last_vblank[p] = 0;
		5340	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
		5341	}
		5342	}
		5343	}
		5344
		5345	static struct i915_power_well *hsw_pwr;
		5346
		5347	/* Display audio driver power well request */
		5348	void i915_request_power_well(void)
		5349	{
		5350	if (WARN_ON(!hsw_pwr))
		5351	return;
		5352
		5353	spin_lock_irq(&hsw_pwr->lock);
		5354	if (!hsw_pwr->count++ &&
		5355	!hsw_pwr->i915_request)
		5356	__intel_set_power_well(hsw_pwr->device, true);
		5357	spin_unlock_irq(&hsw_pwr->lock);
		5358	}
		5359	EXPORT_SYMBOL_GPL(i915_request_power_well);
		5360
		5361	/* Display audio driver power well release */
		5362	void i915_release_power_well(void)
		5363	{
		5364	if (WARN_ON(!hsw_pwr))
		5365	return;
		5366
		5367	spin_lock_irq(&hsw_pwr->lock);
		5368	WARN_ON(!hsw_pwr->count);
		5369	if (!--hsw_pwr->count &&
		5370	!hsw_pwr->i915_request)
		5371	__intel_set_power_well(hsw_pwr->device, false);
		5372	spin_unlock_irq(&hsw_pwr->lock);
		5373	}
		5374	EXPORT_SYMBOL_GPL(i915_release_power_well);
		5375
		5376	int i915_init_power_well(struct drm_device *dev)
		5377	{
		5378	struct drm_i915_private *dev_priv = dev->dev_private;
		5379
		5380	hsw_pwr = &dev_priv->power_well;
		5381
		5382	hsw_pwr->device = dev;
		5383	spin_lock_init(&hsw_pwr->lock);
		5384	hsw_pwr->count = 0;
		5385
		5386	return 0;
		5387	}
		5388
		5389	void i915_remove_power_well(struct drm_device *dev)
		5390	{
		5391	hsw_pwr = NULL;
		5392	}
		5393
		5394	void intel_set_power_well(struct drm_device *dev, bool enable)
		5395	{
		5396	struct drm_i915_private *dev_priv = dev->dev_private;
		5397	struct i915_power_well *power_well = &dev_priv->power_well;
		5398
		5399	if (!HAS_POWER_WELL(dev))
		5400	return;
		5401
		5402	if (!i915_disable_power_well && !enable)
		5403	return;
		5404
		5405	spin_lock_irq(&power_well->lock);
		5406	power_well->i915_request = enable;
		5407
		5408	/* only reject "disable" power well request */
		5409	if (power_well->count && !enable) {
		5410	spin_unlock_irq(&power_well->lock);
		5411	return;
3031	serge	5412	}
4104	Serge	5413
		5414	__intel_set_power_well(dev, enable);
		5415	spin_unlock_irq(&power_well->lock);
3480	Serge	5416	}
3031	serge	5417
3480	Serge	5418	/*
		5419	* Starting with Haswell, we have a "Power Down Well" that can be turned off
		5420	* when not needed anymore. We have 4 registers that can request the power well
		5421	* to be enabled, and it will only be disabled if none of the registers is
		5422	* requesting it to be enabled.
		5423	*/
		5424	void intel_init_power_well(struct drm_device *dev)
		5425	{
		5426	struct drm_i915_private *dev_priv = dev->dev_private;
		5427
3746	Serge	5428	if (!HAS_POWER_WELL(dev))
3480	Serge	5429	return;
		5430
		5431	/* For now, we need the power well to be always enabled. */
		5432	intel_set_power_well(dev, true);
		5433
		5434	/* We're taking over the BIOS, so clear any requests made by it since
		5435	* the driver is in charge now. */
4104	Serge	5436	if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
3480	Serge	5437	I915_WRITE(HSW_PWR_WELL_BIOS, 0);
3031	serge	5438	}
		5439
4104	Serge	5440	/* Disables PC8 so we can use the GMBUS and DP AUX interrupts. */
		5441	void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
		5442	{
		5443	hsw_disable_package_c8(dev_priv);
		5444	}
		5445
		5446	void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
		5447	{
		5448	hsw_enable_package_c8(dev_priv);
		5449	}
		5450
3031	serge	5451	/* Set up chip specific power management-related functions */
		5452	void intel_init_pm(struct drm_device *dev)
		5453	{
		5454	struct drm_i915_private *dev_priv = dev->dev_private;
		5455
		5456	if (I915_HAS_FBC(dev)) {
		5457	if (HAS_PCH_SPLIT(dev)) {
		5458	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
4104	Serge	5459	if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev))
		5460	dev_priv->display.enable_fbc =
		5461	gen7_enable_fbc;
		5462	else
		5463	dev_priv->display.enable_fbc =
		5464	ironlake_enable_fbc;
3031	serge	5465	dev_priv->display.disable_fbc = ironlake_disable_fbc;
		5466	} else if (IS_GM45(dev)) {
		5467	dev_priv->display.fbc_enabled = g4x_fbc_enabled;
		5468	dev_priv->display.enable_fbc = g4x_enable_fbc;
		5469	dev_priv->display.disable_fbc = g4x_disable_fbc;
		5470	} else if (IS_CRESTLINE(dev)) {
		5471	dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
		5472	dev_priv->display.enable_fbc = i8xx_enable_fbc;
		5473	dev_priv->display.disable_fbc = i8xx_disable_fbc;
		5474	}
		5475	/* 855GM needs testing */
		5476	}
		5477
		5478	/* For cxsr */
		5479	if (IS_PINEVIEW(dev))
		5480	i915_pineview_get_mem_freq(dev);
		5481	else if (IS_GEN5(dev))
		5482	i915_ironlake_get_mem_freq(dev);
		5483
		5484	/* For FIFO watermark updates */
		5485	if (HAS_PCH_SPLIT(dev)) {
4104	Serge	5486	intel_setup_wm_latency(dev);
		5487
3031	serge	5488	if (IS_GEN5(dev)) {
4104	Serge	5489	if (dev_priv->wm.pri_latency[1] &&
		5490	dev_priv->wm.spr_latency[1] &&
		5491	dev_priv->wm.cur_latency[1])
3031	serge	5492	dev_priv->display.update_wm = ironlake_update_wm;
		5493	else {
		5494	DRM_DEBUG_KMS("Failed to get proper latency. "
		5495	"Disable CxSR\n");
		5496	dev_priv->display.update_wm = NULL;
		5497	}
		5498	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		5499	} else if (IS_GEN6(dev)) {
4104	Serge	5500	if (dev_priv->wm.pri_latency[0] &&
		5501	dev_priv->wm.spr_latency[0] &&
		5502	dev_priv->wm.cur_latency[0]) {
3031	serge	5503	dev_priv->display.update_wm = sandybridge_update_wm;
		5504	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		5505	} else {
		5506	DRM_DEBUG_KMS("Failed to read display plane latency. "
		5507	"Disable CxSR\n");
		5508	dev_priv->display.update_wm = NULL;
		5509	}
		5510	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
		5511	} else if (IS_IVYBRIDGE(dev)) {
4104	Serge	5512	if (dev_priv->wm.pri_latency[0] &&
		5513	dev_priv->wm.spr_latency[0] &&
		5514	dev_priv->wm.cur_latency[0]) {
3243	Serge	5515	dev_priv->display.update_wm = ivybridge_update_wm;
3031	serge	5516	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		5517	} else {
		5518	DRM_DEBUG_KMS("Failed to read display plane latency. "
		5519	"Disable CxSR\n");
		5520	dev_priv->display.update_wm = NULL;
		5521	}
		5522	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
		5523	} else if (IS_HASWELL(dev)) {
4104	Serge	5524	if (dev_priv->wm.pri_latency[0] &&
		5525	dev_priv->wm.spr_latency[0] &&
		5526	dev_priv->wm.cur_latency[0]) {
		5527	dev_priv->display.update_wm = haswell_update_wm;
		5528	dev_priv->display.update_sprite_wm =
		5529	haswell_update_sprite_wm;
3031	serge	5530	} else {
		5531	DRM_DEBUG_KMS("Failed to read display plane latency. "
		5532	"Disable CxSR\n");
		5533	dev_priv->display.update_wm = NULL;
		5534	}
		5535	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
		5536	} else
		5537	dev_priv->display.update_wm = NULL;
		5538	} else if (IS_VALLEYVIEW(dev)) {
		5539	dev_priv->display.update_wm = valleyview_update_wm;
		5540	dev_priv->display.init_clock_gating =
		5541	valleyview_init_clock_gating;
		5542	} else if (IS_PINEVIEW(dev)) {
		5543	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		5544	dev_priv->is_ddr3,
		5545	dev_priv->fsb_freq,
		5546	dev_priv->mem_freq)) {
		5547	DRM_INFO("failed to find known CxSR latency "
		5548	"(found ddr%s fsb freq %d, mem freq %d), "
		5549	"disabling CxSR\n",
		5550	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		5551	dev_priv->fsb_freq, dev_priv->mem_freq);
		5552	/* Disable CxSR and never update its watermark again */
		5553	pineview_disable_cxsr(dev);
		5554	dev_priv->display.update_wm = NULL;
		5555	} else
		5556	dev_priv->display.update_wm = pineview_update_wm;
		5557	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		5558	} else if (IS_G4X(dev)) {
		5559	dev_priv->display.update_wm = g4x_update_wm;
		5560	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		5561	} else if (IS_GEN4(dev)) {
		5562	dev_priv->display.update_wm = i965_update_wm;
		5563	if (IS_CRESTLINE(dev))
		5564	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		5565	else if (IS_BROADWATER(dev))
		5566	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		5567	} else if (IS_GEN3(dev)) {
		5568	dev_priv->display.update_wm = i9xx_update_wm;
		5569	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		5570	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		5571	} else if (IS_I865G(dev)) {
		5572	dev_priv->display.update_wm = i830_update_wm;
		5573	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		5574	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		5575	} else if (IS_I85X(dev)) {
		5576	dev_priv->display.update_wm = i9xx_update_wm;
		5577	dev_priv->display.get_fifo_size = i85x_get_fifo_size;
		5578	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		5579	} else {
		5580	dev_priv->display.update_wm = i830_update_wm;
		5581	dev_priv->display.init_clock_gating = i830_init_clock_gating;
		5582	if (IS_845G(dev))
		5583	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		5584	else
		5585	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		5586	}
		5587	}
		5588
3243	Serge	5589	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u8 mbox, u32 val)
		5590	{
		5591	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	5592
3243	Serge	5593	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		5594	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		5595	return -EAGAIN;
		5596	}
3031	serge	5597
3243	Serge	5598	I915_WRITE(GEN6_PCODE_DATA, *val);
		5599	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		5600
		5601	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		5602	500)) {
		5603	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		5604	return -ETIMEDOUT;
3031	serge	5605	}
3243	Serge	5606
		5607	*val = I915_READ(GEN6_PCODE_DATA);
		5608	I915_WRITE(GEN6_PCODE_DATA, 0);
		5609
		5610	return 0;
		5611	}
		5612
		5613	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
		5614	{
		5615	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5616
		5617	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		5618	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		5619	return -EAGAIN;
3031	serge	5620	}
3243	Serge	5621
		5622	I915_WRITE(GEN6_PCODE_DATA, val);
		5623	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		5624
		5625	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		5626	500)) {
		5627	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		5628	return -ETIMEDOUT;
3031	serge	5629	}
3243	Serge	5630
		5631	I915_WRITE(GEN6_PCODE_DATA, 0);
		5632
		5633	return 0;
3031	serge	5634	}
3746	Serge	5635
4104	Serge	5636	int vlv_gpu_freq(int ddr_freq, int val)
3746	Serge	5637	{
4104	Serge	5638	int mult, base;
3746	Serge	5639
4104	Serge	5640	switch (ddr_freq) {
		5641	case 800:
		5642	mult = 20;
		5643	base = 120;
		5644	break;
		5645	case 1066:
		5646	mult = 22;
		5647	base = 133;
		5648	break;
		5649	case 1333:
		5650	mult = 21;
		5651	base = 125;
		5652	break;
		5653	default:
		5654	return -1;
		5655	}
3746	Serge	5656
4104	Serge	5657	return ((val - 0xbd) * mult) + base;
		5658	}
3746	Serge	5659
4104	Serge	5660	int vlv_freq_opcode(int ddr_freq, int val)
		5661	{
		5662	int mult, base;
3746	Serge	5663
4104	Serge	5664	switch (ddr_freq) {
		5665	case 800:
		5666	mult = 20;
		5667	base = 120;
		5668	break;
		5669	case 1066:
		5670	mult = 22;
		5671	base = 133;
		5672	break;
		5673	case 1333:
		5674	mult = 21;
		5675	base = 125;
		5676	break;
		5677	default:
		5678	return -1;
3746	Serge	5679	}
		5680
4104	Serge	5681	val /= mult;
		5682	val -= base / mult;
		5683	val += 0xbd;
3746	Serge	5684
4104	Serge	5685	if (val > 0xea)
		5686	val = 0xea;
3746	Serge	5687
4104	Serge	5688	return val;
3746	Serge	5689	}
		5690
4104	Serge	5691	void intel_pm_init(struct drm_device *dev)
3746	Serge	5692	{
4104	Serge	5693	struct drm_i915_private *dev_priv = dev->dev_private;
		5694
		5695	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		5696	intel_gen6_powersave_work);
		5697
		5698	INIT_DELAYED_WORK(&dev_priv->rps.vlv_work, vlv_rps_timer_work);
3746	Serge	5699	}
		5700

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 4539