WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	#include
		32	//#include "../../../platform/x86/intel_ips.h"
		33	#include
		34
4560	Serge	35	#include
		36
3031	serge	37	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		38
		39	#define assert_spin_locked(x)
		40
		41	void getrawmonotonic(struct timespec *ts);
		42
4560	Serge	43	static inline void outb(u8 v, u16 port)
		44	{
		45	asm volatile("outb %0,%1" : : "a" (v), "dN" (port));
		46	}
		47	static inline u8 inb(u16 port)
		48	{
		49	u8 v;
		50	asm volatile("inb %1,%0" : "=a" (v) : "dN" (port));
		51	return v;
		52	}
3031	serge	53
		54
4560	Serge	55	/**
		56	* RC6 is a special power stage which allows the GPU to enter an very
		57	* low-voltage mode when idle, using down to 0V while at this stage. This
		58	* stage is entered automatically when the GPU is idle when RC6 support is
		59	* enabled, and as soon as new workload arises GPU wakes up automatically as well.
		60	*
		61	* There are different RC6 modes available in Intel GPU, which differentiate
		62	* among each other with the latency required to enter and leave RC6 and
		63	* voltage consumed by the GPU in different states.
		64	*
		65	* The combination of the following flags define which states GPU is allowed
		66	* to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
		67	* RC6pp is deepest RC6. Their support by hardware varies according to the
		68	* GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
		69	* which brings the most power savings; deeper states save more power, but
		70	* require higher latency to switch to and wake up.
		71	*/
		72	#define INTEL_RC6_ENABLE (1<<0)
		73	#define INTEL_RC6p_ENABLE (1<<1)
		74	#define INTEL_RC6pp_ENABLE (1<<2)
		75
3031	serge	76	/* FBC, or Frame Buffer Compression, is a technique employed to compress the
		77	* framebuffer contents in-memory, aiming at reducing the required bandwidth
		78	* during in-memory transfers and, therefore, reduce the power packet.
		79	*
		80	* The benefits of FBC are mostly visible with solid backgrounds and
		81	* variation-less patterns.
		82	*
		83	* FBC-related functionality can be enabled by the means of the
		84	* i915.i915_enable_fbc parameter
		85	*/
		86
		87	static void i8xx_disable_fbc(struct drm_device *dev)
		88	{
		89	struct drm_i915_private *dev_priv = dev->dev_private;
		90	u32 fbc_ctl;
		91
		92	/* Disable compression */
		93	fbc_ctl = I915_READ(FBC_CONTROL);
		94	if ((fbc_ctl & FBC_CTL_EN) == 0)
		95	return;
		96
		97	fbc_ctl &= ~FBC_CTL_EN;
		98	I915_WRITE(FBC_CONTROL, fbc_ctl);
		99
		100	/* Wait for compressing bit to clear */
		101	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
		102	DRM_DEBUG_KMS("FBC idle timed out\n");
		103	return;
		104	}
		105
		106	DRM_DEBUG_KMS("disabled FBC\n");
		107	}
		108
4560	Serge	109	static void i8xx_enable_fbc(struct drm_crtc *crtc)
3031	serge	110	{
		111	struct drm_device *dev = crtc->dev;
		112	struct drm_i915_private *dev_priv = dev->dev_private;
		113	struct drm_framebuffer *fb = crtc->fb;
		114	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		115	struct drm_i915_gem_object *obj = intel_fb->obj;
		116	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		117	int cfb_pitch;
		118	int plane, i;
4560	Serge	119	u32 fbc_ctl;
3031	serge	120
4104	Serge	121	cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE;
3031	serge	122	if (fb->pitches[0] < cfb_pitch)
		123	cfb_pitch = fb->pitches[0];
		124
4560	Serge	125	/* FBC_CTL wants 32B or 64B units */
		126	if (IS_GEN2(dev))
		127	cfb_pitch = (cfb_pitch / 32) - 1;
		128	else
3031	serge	129	cfb_pitch = (cfb_pitch / 64) - 1;
		130	plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
		131
		132	/* Clear old tags */
		133	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
		134	I915_WRITE(FBC_TAG + (i * 4), 0);
		135
4560	Serge	136	if (IS_GEN4(dev)) {
		137	u32 fbc_ctl2;
		138
3031	serge	139	/* Set it up... */
		140	fbc_ctl2 = FBC_CTL_FENCE_DBL \| FBC_CTL_IDLE_IMM \| FBC_CTL_CPU_FENCE;
		141	fbc_ctl2 \|= plane;
		142	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
		143	I915_WRITE(FBC_FENCE_OFF, crtc->y);
4560	Serge	144	}
3031	serge	145
		146	/* enable it... */
4560	Serge	147	fbc_ctl = I915_READ(FBC_CONTROL);
		148	fbc_ctl &= 0x3fff << FBC_CTL_INTERVAL_SHIFT;
		149	fbc_ctl \|= FBC_CTL_EN \| FBC_CTL_PERIODIC;
3031	serge	150	if (IS_I945GM(dev))
		151	fbc_ctl \|= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
		152	fbc_ctl \|= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
		153	fbc_ctl \|= obj->fence_reg;
		154	I915_WRITE(FBC_CONTROL, fbc_ctl);
		155
4104	Serge	156	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ",
		157	cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
3031	serge	158	}
		159
		160	static bool i8xx_fbc_enabled(struct drm_device *dev)
		161	{
		162	struct drm_i915_private *dev_priv = dev->dev_private;
		163
		164	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
		165	}
		166
4560	Serge	167	static void g4x_enable_fbc(struct drm_crtc *crtc)
3031	serge	168	{
		169	struct drm_device *dev = crtc->dev;
		170	struct drm_i915_private *dev_priv = dev->dev_private;
		171	struct drm_framebuffer *fb = crtc->fb;
		172	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		173	struct drm_i915_gem_object *obj = intel_fb->obj;
		174	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		175	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		176	u32 dpfc_ctl;
		177
		178	dpfc_ctl = plane \| DPFC_SR_EN \| DPFC_CTL_LIMIT_1X;
		179	dpfc_ctl \|= DPFC_CTL_FENCE_EN \| obj->fence_reg;
		180	I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
		181
		182	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
		183
		184	/* enable it... */
		185	I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) \| DPFC_CTL_EN);
		186
4104	Serge	187	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
3031	serge	188	}
		189
		190	static void g4x_disable_fbc(struct drm_device *dev)
		191	{
		192	struct drm_i915_private *dev_priv = dev->dev_private;
		193	u32 dpfc_ctl;
		194
		195	/* Disable compression */
		196	dpfc_ctl = I915_READ(DPFC_CONTROL);
		197	if (dpfc_ctl & DPFC_CTL_EN) {
		198	dpfc_ctl &= ~DPFC_CTL_EN;
		199	I915_WRITE(DPFC_CONTROL, dpfc_ctl);
		200
		201	DRM_DEBUG_KMS("disabled FBC\n");
		202	}
		203	}
		204
		205	static bool g4x_fbc_enabled(struct drm_device *dev)
		206	{
		207	struct drm_i915_private *dev_priv = dev->dev_private;
		208
		209	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
		210	}
		211
		212	static void sandybridge_blit_fbc_update(struct drm_device *dev)
		213	{
		214	struct drm_i915_private *dev_priv = dev->dev_private;
		215	u32 blt_ecoskpd;
		216
		217	/* Make sure blitter notifies FBC of writes */
4560	Serge	218
		219	/* Blitter is part of Media powerwell on VLV. No impact of
		220	* his param in other platforms for now */
		221	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_MEDIA);
		222
3031	serge	223	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
		224	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY <<
		225	GEN6_BLITTER_LOCK_SHIFT;
		226	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		227	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY;
		228	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		229	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
		230	GEN6_BLITTER_LOCK_SHIFT);
		231	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		232	POSTING_READ(GEN6_BLITTER_ECOSKPD);
4560	Serge	233
		234	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_MEDIA);
3031	serge	235	}
		236
4560	Serge	237	static void ironlake_enable_fbc(struct drm_crtc *crtc)
3031	serge	238	{
		239	struct drm_device *dev = crtc->dev;
		240	struct drm_i915_private *dev_priv = dev->dev_private;
		241	struct drm_framebuffer *fb = crtc->fb;
		242	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		243	struct drm_i915_gem_object *obj = intel_fb->obj;
		244	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		245	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		246	u32 dpfc_ctl;
		247
		248	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		249	dpfc_ctl &= DPFC_RESERVED;
		250	dpfc_ctl \|= (plane \| DPFC_CTL_LIMIT_1X);
		251	/* Set persistent mode for front-buffer rendering, ala X. */
		252	dpfc_ctl \|= DPFC_CTL_PERSISTENT_MODE;
4560	Serge	253	dpfc_ctl \|= DPFC_CTL_FENCE_EN;
		254	if (IS_GEN5(dev))
		255	dpfc_ctl \|= obj->fence_reg;
3031	serge	256	I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
		257
		258	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
4104	Serge	259	I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) \| ILK_FBC_RT_VALID);
3031	serge	260	/* enable it... */
		261	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		262
		263	if (IS_GEN6(dev)) {
		264	I915_WRITE(SNB_DPFC_CTL_SA,
		265	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		266	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		267	sandybridge_blit_fbc_update(dev);
		268	}
		269
4104	Serge	270	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
3031	serge	271	}
		272
		273	static void ironlake_disable_fbc(struct drm_device *dev)
		274	{
		275	struct drm_i915_private *dev_priv = dev->dev_private;
		276	u32 dpfc_ctl;
		277
		278	/* Disable compression */
		279	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		280	if (dpfc_ctl & DPFC_CTL_EN) {
		281	dpfc_ctl &= ~DPFC_CTL_EN;
		282	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
		283
		284	DRM_DEBUG_KMS("disabled FBC\n");
		285	}
		286	}
		287
		288	static bool ironlake_fbc_enabled(struct drm_device *dev)
		289	{
		290	struct drm_i915_private *dev_priv = dev->dev_private;
		291
		292	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
		293	}
		294
4560	Serge	295	static void gen7_enable_fbc(struct drm_crtc *crtc)
4104	Serge	296	{
		297	struct drm_device *dev = crtc->dev;
		298	struct drm_i915_private *dev_priv = dev->dev_private;
		299	struct drm_framebuffer *fb = crtc->fb;
		300	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		301	struct drm_i915_gem_object *obj = intel_fb->obj;
		302	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		303
		304	I915_WRITE(IVB_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj));
		305
		306	I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN \| DPFC_CTL_LIMIT_1X \|
		307	IVB_DPFC_CTL_FENCE_EN \|
		308	intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT);
		309
		310	if (IS_IVYBRIDGE(dev)) {
		311	/* WaFbcAsynchFlipDisableFbcQueue:ivb */
		312	I915_WRITE(ILK_DISPLAY_CHICKEN1, ILK_FBCQ_DIS);
		313	} else {
		314	/* WaFbcAsynchFlipDisableFbcQueue:hsw */
		315	I915_WRITE(HSW_PIPE_SLICE_CHICKEN_1(intel_crtc->pipe),
		316	HSW_BYPASS_FBC_QUEUE);
		317	}
		318
		319	I915_WRITE(SNB_DPFC_CTL_SA,
		320	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		321	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		322
		323	sandybridge_blit_fbc_update(dev);
		324
4560	Serge	325	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
4104	Serge	326	}
		327
3031	serge	328	bool intel_fbc_enabled(struct drm_device *dev)
		329	{
		330	struct drm_i915_private *dev_priv = dev->dev_private;
		331
		332	if (!dev_priv->display.fbc_enabled)
		333	return false;
		334
		335	return dev_priv->display.fbc_enabled(dev);
		336	}
		337
		338	static void intel_fbc_work_fn(struct work_struct *__work)
		339	{
		340	struct intel_fbc_work *work =
		341	container_of(to_delayed_work(__work),
		342	struct intel_fbc_work, work);
		343	struct drm_device *dev = work->crtc->dev;
		344	struct drm_i915_private *dev_priv = dev->dev_private;
		345
		346	mutex_lock(&dev->struct_mutex);
4104	Serge	347	if (work == dev_priv->fbc.fbc_work) {
3031	serge	348	/* Double check that we haven't switched fb without cancelling
		349	* the prior work.
		350	*/
		351	if (work->crtc->fb == work->fb) {
4560	Serge	352	dev_priv->display.enable_fbc(work->crtc);
3031	serge	353
4104	Serge	354	dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane;
		355	dev_priv->fbc.fb_id = work->crtc->fb->base.id;
		356	dev_priv->fbc.y = work->crtc->y;
3031	serge	357	}
		358
4104	Serge	359	dev_priv->fbc.fbc_work = NULL;
3031	serge	360	}
		361	mutex_unlock(&dev->struct_mutex);
		362
		363	kfree(work);
		364	}
		365
		366	static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
		367	{
4104	Serge	368	if (dev_priv->fbc.fbc_work == NULL)
3031	serge	369	return;
		370
		371	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
		372
		373	/* Synchronisation is provided by struct_mutex and checking of
4104	Serge	374	* dev_priv->fbc.fbc_work, so we can perform the cancellation
3031	serge	375	* entirely asynchronously.
		376	*/
4293	Serge	377	if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work))
3031	serge	378	/* tasklet was killed before being run, clean up */
4293	Serge	379	kfree(dev_priv->fbc.fbc_work);
3031	serge	380
		381	/* Mark the work as no longer wanted so that if it does
		382	* wake-up (because the work was already running and waiting
		383	* for our mutex), it will discover that is no longer
		384	* necessary to run.
		385	*/
4104	Serge	386	dev_priv->fbc.fbc_work = NULL;
3031	serge	387	}
		388
4560	Serge	389	static void intel_enable_fbc(struct drm_crtc *crtc)
3031	serge	390	{
		391	struct intel_fbc_work *work;
		392	struct drm_device *dev = crtc->dev;
		393	struct drm_i915_private *dev_priv = dev->dev_private;
		394
3482	Serge	395	if (!dev_priv->display.enable_fbc)
3031	serge	396	return;
3482	Serge	397
3031	serge	398	intel_cancel_fbc_work(dev_priv);
		399
4560	Serge	400	work = kzalloc(sizeof(*work), GFP_KERNEL);
3031	serge	401	if (work == NULL) {
4104	Serge	402	DRM_ERROR("Failed to allocate FBC work structure\n");
4560	Serge	403	dev_priv->display.enable_fbc(crtc);
3031	serge	404	return;
		405	}
		406
		407	work->crtc = crtc;
		408	work->fb = crtc->fb;
		409	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
		410
4104	Serge	411	dev_priv->fbc.fbc_work = work;
3031	serge	412
		413	/* Delay the actual enabling to let pageflipping cease and the
		414	* display to settle before starting the compression. Note that
		415	* this delay also serves a second purpose: it allows for a
		416	* vblank to pass after disabling the FBC before we attempt
		417	* to modify the control registers.
		418	*
		419	* A more complicated solution would involve tracking vblanks
		420	* following the termination of the page-flipping sequence
		421	* and indeed performing the enable as a co-routine and not
		422	* waiting synchronously upon the vblank.
4104	Serge	423	*
		424	* WaFbcWaitForVBlankBeforeEnable:ilk,snb
3031	serge	425	*/
		426	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
		427	}
		428
		429	void intel_disable_fbc(struct drm_device *dev)
		430	{
		431	struct drm_i915_private *dev_priv = dev->dev_private;
		432
3482	Serge	433	intel_cancel_fbc_work(dev_priv);
3031	serge	434
3482	Serge	435	if (!dev_priv->display.disable_fbc)
		436	return;
3031	serge	437
3482	Serge	438	dev_priv->display.disable_fbc(dev);
4104	Serge	439	dev_priv->fbc.plane = -1;
3031	serge	440	}
		441
4104	Serge	442	static bool set_no_fbc_reason(struct drm_i915_private *dev_priv,
		443	enum no_fbc_reason reason)
		444	{
		445	if (dev_priv->fbc.no_fbc_reason == reason)
		446	return false;
		447
		448	dev_priv->fbc.no_fbc_reason = reason;
		449	return true;
		450	}
		451
3031	serge	452	/**
		453	* intel_update_fbc - enable/disable FBC as needed
		454	* @dev: the drm_device
		455	*
		456	* Set up the framebuffer compression hardware at mode set time. We
		457	* enable it if possible:
		458	* - plane A only (on pre-965)
		459	* - no pixel mulitply/line duplication
		460	* - no alpha buffer discard
		461	* - no dual wide
4104	Serge	462	* - framebuffer <= max_hdisplay in width, max_vdisplay in height
3031	serge	463	*
		464	* We can't assume that any compression will take place (worst case),
		465	* so the compressed buffer has to be the same size as the uncompressed
		466	* one. It also must reside (along with the line length buffer) in
		467	* stolen memory.
		468	*
		469	* We need to enable/disable FBC on a global basis.
		470	*/
		471	void intel_update_fbc(struct drm_device *dev)
		472	{
		473	struct drm_i915_private *dev_priv = dev->dev_private;
		474	struct drm_crtc crtc = NULL, tmp_crtc;
		475	struct intel_crtc *intel_crtc;
		476	struct drm_framebuffer *fb;
		477	struct intel_framebuffer *intel_fb;
		478	struct drm_i915_gem_object *obj;
4560	Serge	479	const struct drm_display_mode *adjusted_mode;
		480	unsigned int max_width, max_height;
3031	serge	481
4560	Serge	482	if (!HAS_FBC(dev)) {
4104	Serge	483	set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED);
3031	serge	484	return;
4104	Serge	485	}
3031	serge	486
4104	Serge	487	if (!i915_powersave) {
		488	if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
		489	DRM_DEBUG_KMS("fbc disabled per module param\n");
3031	serge	490	return;
4104	Serge	491	}
3031	serge	492
		493	/*
		494	* If FBC is already on, we just have to verify that we can
		495	* keep it that way...
		496	* Need to disable if:
		497	* - more than one pipe is active
		498	* - changing FBC params (stride, fence, mode)
		499	* - new fb is too large to fit in compressed buffer
		500	* - going to an unsupported config (interlace, pixel multiply, etc.)
		501	*/
		502	list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	503	if (intel_crtc_active(tmp_crtc) &&
4560	Serge	504	to_intel_crtc(tmp_crtc)->primary_enabled) {
3031	serge	505	if (crtc) {
4104	Serge	506	if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES))
3031	serge	507	DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
		508	goto out_disable;
		509	}
		510	crtc = tmp_crtc;
		511	}
		512	}
		513
		514	if (!crtc \|\| crtc->fb == NULL) {
4104	Serge	515	if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT))
3031	serge	516	DRM_DEBUG_KMS("no output, disabling\n");
		517	goto out_disable;
		518	}
		519
		520	intel_crtc = to_intel_crtc(crtc);
		521	fb = crtc->fb;
		522	intel_fb = to_intel_framebuffer(fb);
		523	obj = intel_fb->obj;
4560	Serge	524	adjusted_mode = &intel_crtc->config.adjusted_mode;
3031	serge	525
4104	Serge	526	if (i915_enable_fbc < 0 &&
		527	INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) {
		528	if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT))
		529	DRM_DEBUG_KMS("disabled per chip default\n");
		530	goto out_disable;
3031	serge	531	}
4104	Serge	532	if (!i915_enable_fbc) {
		533	if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
3031	serge	534	DRM_DEBUG_KMS("fbc disabled per module param\n");
		535	goto out_disable;
		536	}
4560	Serge	537	if ((adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) \|\|
		538	(adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)) {
4104	Serge	539	if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
3031	serge	540	DRM_DEBUG_KMS("mode incompatible with compression, "
		541	"disabling\n");
		542	goto out_disable;
		543	}
4104	Serge	544
		545	if (IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
4560	Serge	546	max_width = 4096;
		547	max_height = 2048;
4104	Serge	548	} else {
4560	Serge	549	max_width = 2048;
		550	max_height = 1536;
4104	Serge	551	}
4560	Serge	552	if (intel_crtc->config.pipe_src_w > max_width \|\|
		553	intel_crtc->config.pipe_src_h > max_height) {
4104	Serge	554	if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE))
3031	serge	555	DRM_DEBUG_KMS("mode too large for compression, disabling\n");
		556	goto out_disable;
		557	}
4560	Serge	558	if ((INTEL_INFO(dev)->gen < 4 \|\| IS_HASWELL(dev)) &&
		559	intel_crtc->plane != PLANE_A) {
4104	Serge	560	if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE))
4560	Serge	561	DRM_DEBUG_KMS("plane not A, disabling compression\n");
3031	serge	562	goto out_disable;
		563	}
		564
		565	/* The use of a CPU fence is mandatory in order to detect writes
		566	* by the CPU to the scanout and trigger updates to the FBC.
		567	*/
		568	if (obj->tiling_mode != I915_TILING_X \|\|
		569	obj->fence_reg == I915_FENCE_REG_NONE) {
4104	Serge	570	if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED))
3031	serge	571	DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
		572	goto out_disable;
		573	}
		574
		575	/* If the kernel debugger is active, always disable compression */
		576	if (in_dbg_master())
		577	goto out_disable;
		578
3480	Serge	579	if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
4104	Serge	580	if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL))
3480	Serge	581	DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
		582	goto out_disable;
		583	}
		584
3031	serge	585	/* If the scanout has not changed, don't modify the FBC settings.
		586	* Note that we make the fundamental assumption that the fb->obj
		587	* cannot be unpinned (and have its GTT offset and fence revoked)
		588	* without first being decoupled from the scanout and FBC disabled.
		589	*/
4104	Serge	590	if (dev_priv->fbc.plane == intel_crtc->plane &&
		591	dev_priv->fbc.fb_id == fb->base.id &&
		592	dev_priv->fbc.y == crtc->y)
3031	serge	593	return;
		594
		595	if (intel_fbc_enabled(dev)) {
		596	/* We update FBC along two paths, after changing fb/crtc
		597	* configuration (modeswitching) and after page-flipping
		598	* finishes. For the latter, we know that not only did
		599	* we disable the FBC at the start of the page-flip
		600	* sequence, but also more than one vblank has passed.
		601	*
		602	* For the former case of modeswitching, it is possible
		603	* to switch between two FBC valid configurations
		604	* instantaneously so we do need to disable the FBC
		605	* before we can modify its control registers. We also
		606	* have to wait for the next vblank for that to take
		607	* effect. However, since we delay enabling FBC we can
		608	* assume that a vblank has passed since disabling and
		609	* that we can safely alter the registers in the deferred
		610	* callback.
		611	*
		612	* In the scenario that we go from a valid to invalid
		613	* and then back to valid FBC configuration we have
		614	* no strict enforcement that a vblank occurred since
		615	* disabling the FBC. However, along all current pipe
		616	* disabling paths we do need to wait for a vblank at
		617	* some point. And we wait before enabling FBC anyway.
		618	*/
		619	DRM_DEBUG_KMS("disabling active FBC for update\n");
		620	intel_disable_fbc(dev);
		621	}
		622
4560	Serge	623	intel_enable_fbc(crtc);
4104	Serge	624	dev_priv->fbc.no_fbc_reason = FBC_OK;
3031	serge	625	return;
		626
		627	out_disable:
		628	/* Multiple disables should be harmless */
		629	if (intel_fbc_enabled(dev)) {
		630	DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
		631	intel_disable_fbc(dev);
		632	}
3480	Serge	633	i915_gem_stolen_cleanup_compression(dev);
3031	serge	634	}
		635
		636	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		637	{
		638	drm_i915_private_t *dev_priv = dev->dev_private;
		639	u32 tmp;
		640
		641	tmp = I915_READ(CLKCFG);
		642
		643	switch (tmp & CLKCFG_FSB_MASK) {
		644	case CLKCFG_FSB_533:
		645	dev_priv->fsb_freq = 533; /* 1334 /
		646	break;
		647	case CLKCFG_FSB_800:
		648	dev_priv->fsb_freq = 800; /* 2004 /
		649	break;
		650	case CLKCFG_FSB_667:
		651	dev_priv->fsb_freq = 667; /* 1674 /
		652	break;
		653	case CLKCFG_FSB_400:
		654	dev_priv->fsb_freq = 400; /* 1004 /
		655	break;
		656	}
		657
		658	switch (tmp & CLKCFG_MEM_MASK) {
		659	case CLKCFG_MEM_533:
		660	dev_priv->mem_freq = 533;
		661	break;
		662	case CLKCFG_MEM_667:
		663	dev_priv->mem_freq = 667;
		664	break;
		665	case CLKCFG_MEM_800:
		666	dev_priv->mem_freq = 800;
		667	break;
		668	}
		669
		670	/* detect pineview DDR3 setting */
		671	tmp = I915_READ(CSHRDDR3CTL);
		672	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		673	}
		674
		675	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		676	{
		677	drm_i915_private_t *dev_priv = dev->dev_private;
		678	u16 ddrpll, csipll;
		679
		680	ddrpll = I915_READ16(DDRMPLL1);
		681	csipll = I915_READ16(CSIPLL0);
		682
		683	switch (ddrpll & 0xff) {
		684	case 0xc:
		685	dev_priv->mem_freq = 800;
		686	break;
		687	case 0x10:
		688	dev_priv->mem_freq = 1066;
		689	break;
		690	case 0x14:
		691	dev_priv->mem_freq = 1333;
		692	break;
		693	case 0x18:
		694	dev_priv->mem_freq = 1600;
		695	break;
		696	default:
		697	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		698	ddrpll & 0xff);
		699	dev_priv->mem_freq = 0;
		700	break;
		701	}
		702
		703	dev_priv->ips.r_t = dev_priv->mem_freq;
		704
		705	switch (csipll & 0x3ff) {
		706	case 0x00c:
		707	dev_priv->fsb_freq = 3200;
		708	break;
		709	case 0x00e:
		710	dev_priv->fsb_freq = 3733;
		711	break;
		712	case 0x010:
		713	dev_priv->fsb_freq = 4266;
		714	break;
		715	case 0x012:
		716	dev_priv->fsb_freq = 4800;
		717	break;
		718	case 0x014:
		719	dev_priv->fsb_freq = 5333;
		720	break;
		721	case 0x016:
		722	dev_priv->fsb_freq = 5866;
		723	break;
		724	case 0x018:
		725	dev_priv->fsb_freq = 6400;
		726	break;
		727	default:
		728	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		729	csipll & 0x3ff);
		730	dev_priv->fsb_freq = 0;
		731	break;
		732	}
		733
		734	if (dev_priv->fsb_freq == 3200) {
		735	dev_priv->ips.c_m = 0;
		736	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		737	dev_priv->ips.c_m = 1;
		738	} else {
		739	dev_priv->ips.c_m = 2;
		740	}
		741	}
		742
		743	static const struct cxsr_latency cxsr_latency_table[] = {
		744	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		745	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		746	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		747	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		748	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		749
		750	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		751	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		752	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		753	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		754	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		755
		756	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		757	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		758	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		759	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		760	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		761
		762	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		763	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		764	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		765	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		766	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		767
		768	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		769	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		770	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		771	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		772	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		773
		774	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		775	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		776	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		777	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		778	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		779	};
		780
		781	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		782	int is_ddr3,
		783	int fsb,
		784	int mem)
		785	{
		786	const struct cxsr_latency *latency;
		787	int i;
		788
		789	if (fsb == 0 \|\| mem == 0)
		790	return NULL;
		791
		792	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		793	latency = &cxsr_latency_table[i];
		794	if (is_desktop == latency->is_desktop &&
		795	is_ddr3 == latency->is_ddr3 &&
		796	fsb == latency->fsb_freq && mem == latency->mem_freq)
		797	return latency;
		798	}
		799
		800	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		801
		802	return NULL;
		803	}
		804
		805	static void pineview_disable_cxsr(struct drm_device *dev)
		806	{
		807	struct drm_i915_private *dev_priv = dev->dev_private;
		808
		809	/* deactivate cxsr */
		810	I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
		811	}
		812
		813	/*
		814	* Latency for FIFO fetches is dependent on several factors:
		815	* - memory configuration (speed, channels)
		816	* - chipset
		817	* - current MCH state
		818	* It can be fairly high in some situations, so here we assume a fairly
		819	* pessimal value. It's a tradeoff between extra memory fetches (if we
		820	* set this value too high, the FIFO will fetch frequently to stay full)
		821	* and power consumption (set it too low to save power and we might see
		822	* FIFO underruns and display "flicker").
		823	*
		824	* A value of 5us seems to be a good balance; safe for very low end
		825	* platforms but not overly aggressive on lower latency configs.
		826	*/
		827	static const int latency_ns = 5000;
		828
		829	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		830	{
		831	struct drm_i915_private *dev_priv = dev->dev_private;
		832	uint32_t dsparb = I915_READ(DSPARB);
		833	int size;
		834
		835	size = dsparb & 0x7f;
		836	if (plane)
		837	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		838
		839	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		840	plane ? "B" : "A", size);
		841
		842	return size;
		843	}
		844
4560	Serge	845	static int i830_get_fifo_size(struct drm_device *dev, int plane)
3031	serge	846	{
		847	struct drm_i915_private *dev_priv = dev->dev_private;
		848	uint32_t dsparb = I915_READ(DSPARB);
		849	int size;
		850
		851	size = dsparb & 0x1ff;
		852	if (plane)
		853	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		854	size >>= 1; /* Convert to cachelines */
		855
		856	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		857	plane ? "B" : "A", size);
		858
		859	return size;
		860	}
		861
		862	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		863	{
		864	struct drm_i915_private *dev_priv = dev->dev_private;
		865	uint32_t dsparb = I915_READ(DSPARB);
		866	int size;
		867
		868	size = dsparb & 0x7f;
		869	size >>= 2; /* Convert to cachelines */
		870
		871	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		872	plane ? "B" : "A",
		873	size);
		874
		875	return size;
		876	}
		877
		878	/* Pineview has different values for various configs */
		879	static const struct intel_watermark_params pineview_display_wm = {
		880	PINEVIEW_DISPLAY_FIFO,
		881	PINEVIEW_MAX_WM,
		882	PINEVIEW_DFT_WM,
		883	PINEVIEW_GUARD_WM,
		884	PINEVIEW_FIFO_LINE_SIZE
		885	};
		886	static const struct intel_watermark_params pineview_display_hplloff_wm = {
		887	PINEVIEW_DISPLAY_FIFO,
		888	PINEVIEW_MAX_WM,
		889	PINEVIEW_DFT_HPLLOFF_WM,
		890	PINEVIEW_GUARD_WM,
		891	PINEVIEW_FIFO_LINE_SIZE
		892	};
		893	static const struct intel_watermark_params pineview_cursor_wm = {
		894	PINEVIEW_CURSOR_FIFO,
		895	PINEVIEW_CURSOR_MAX_WM,
		896	PINEVIEW_CURSOR_DFT_WM,
		897	PINEVIEW_CURSOR_GUARD_WM,
		898	PINEVIEW_FIFO_LINE_SIZE,
		899	};
		900	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
		901	PINEVIEW_CURSOR_FIFO,
		902	PINEVIEW_CURSOR_MAX_WM,
		903	PINEVIEW_CURSOR_DFT_WM,
		904	PINEVIEW_CURSOR_GUARD_WM,
		905	PINEVIEW_FIFO_LINE_SIZE
		906	};
		907	static const struct intel_watermark_params g4x_wm_info = {
		908	G4X_FIFO_SIZE,
		909	G4X_MAX_WM,
		910	G4X_MAX_WM,
		911	2,
		912	G4X_FIFO_LINE_SIZE,
		913	};
		914	static const struct intel_watermark_params g4x_cursor_wm_info = {
		915	I965_CURSOR_FIFO,
		916	I965_CURSOR_MAX_WM,
		917	I965_CURSOR_DFT_WM,
		918	2,
		919	G4X_FIFO_LINE_SIZE,
		920	};
		921	static const struct intel_watermark_params valleyview_wm_info = {
		922	VALLEYVIEW_FIFO_SIZE,
		923	VALLEYVIEW_MAX_WM,
		924	VALLEYVIEW_MAX_WM,
		925	2,
		926	G4X_FIFO_LINE_SIZE,
		927	};
		928	static const struct intel_watermark_params valleyview_cursor_wm_info = {
		929	I965_CURSOR_FIFO,
		930	VALLEYVIEW_CURSOR_MAX_WM,
		931	I965_CURSOR_DFT_WM,
		932	2,
		933	G4X_FIFO_LINE_SIZE,
		934	};
		935	static const struct intel_watermark_params i965_cursor_wm_info = {
		936	I965_CURSOR_FIFO,
		937	I965_CURSOR_MAX_WM,
		938	I965_CURSOR_DFT_WM,
		939	2,
		940	I915_FIFO_LINE_SIZE,
		941	};
		942	static const struct intel_watermark_params i945_wm_info = {
		943	I945_FIFO_SIZE,
		944	I915_MAX_WM,
		945	1,
		946	2,
		947	I915_FIFO_LINE_SIZE
		948	};
		949	static const struct intel_watermark_params i915_wm_info = {
		950	I915_FIFO_SIZE,
		951	I915_MAX_WM,
		952	1,
		953	2,
		954	I915_FIFO_LINE_SIZE
		955	};
4560	Serge	956	static const struct intel_watermark_params i830_wm_info = {
3031	serge	957	I855GM_FIFO_SIZE,
		958	I915_MAX_WM,
		959	1,
		960	2,
		961	I830_FIFO_LINE_SIZE
		962	};
4560	Serge	963	static const struct intel_watermark_params i845_wm_info = {
3031	serge	964	I830_FIFO_SIZE,
		965	I915_MAX_WM,
		966	1,
		967	2,
		968	I830_FIFO_LINE_SIZE
		969	};
		970
		971	/**
		972	* intel_calculate_wm - calculate watermark level
		973	* @clock_in_khz: pixel clock
		974	* @wm: chip FIFO params
		975	* @pixel_size: display pixel size
		976	* @latency_ns: memory latency for the platform
		977	*
		978	* Calculate the watermark level (the level at which the display plane will
		979	* start fetching from memory again). Each chip has a different display
		980	* FIFO size and allocation, so the caller needs to figure that out and pass
		981	* in the correct intel_watermark_params structure.
		982	*
		983	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		984	* on the pixel size. When it reaches the watermark level, it'll start
		985	* fetching FIFO line sized based chunks from memory until the FIFO fills
		986	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		987	* will occur, and a display engine hang could result.
		988	*/
		989	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		990	const struct intel_watermark_params *wm,
		991	int fifo_size,
		992	int pixel_size,
		993	unsigned long latency_ns)
		994	{
		995	long entries_required, wm_size;
		996
		997	/*
		998	* Note: we need to make sure we don't overflow for various clock &
		999	* latency values.
		1000	* clocks go from a few thousand to several hundred thousand.
		1001	* latency is usually a few thousand
		1002	*/
		1003	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		1004	1000;
		1005	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		1006
		1007	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		1008
		1009	wm_size = fifo_size - (entries_required + wm->guard_size);
		1010
		1011	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		1012
		1013	/* Don't promote wm_size to unsigned... */
		1014	if (wm_size > (long)wm->max_wm)
		1015	wm_size = wm->max_wm;
		1016	if (wm_size <= 0)
		1017	wm_size = wm->default_wm;
		1018	return wm_size;
		1019	}
		1020
		1021	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		1022	{
		1023	struct drm_crtc crtc, enabled = NULL;
		1024
		1025	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	1026	if (intel_crtc_active(crtc)) {
3031	serge	1027	if (enabled)
		1028	return NULL;
		1029	enabled = crtc;
		1030	}
		1031	}
		1032
		1033	return enabled;
		1034	}
		1035
4560	Serge	1036	static void pineview_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1037	{
4560	Serge	1038	struct drm_device *dev = unused_crtc->dev;
3031	serge	1039	struct drm_i915_private *dev_priv = dev->dev_private;
		1040	struct drm_crtc *crtc;
		1041	const struct cxsr_latency *latency;
		1042	u32 reg;
		1043	unsigned long wm;
		1044
		1045	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		1046	dev_priv->fsb_freq, dev_priv->mem_freq);
		1047	if (!latency) {
		1048	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		1049	pineview_disable_cxsr(dev);
		1050	return;
		1051	}
		1052
		1053	crtc = single_enabled_crtc(dev);
		1054	if (crtc) {
4560	Serge	1055	const struct drm_display_mode *adjusted_mode;
3031	serge	1056	int pixel_size = crtc->fb->bits_per_pixel / 8;
4560	Serge	1057	int clock;
3031	serge	1058
4560	Serge	1059	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1060	clock = adjusted_mode->crtc_clock;
		1061
3031	serge	1062	/* Display SR */
		1063	wm = intel_calculate_wm(clock, &pineview_display_wm,
		1064	pineview_display_wm.fifo_size,
		1065	pixel_size, latency->display_sr);
		1066	reg = I915_READ(DSPFW1);
		1067	reg &= ~DSPFW_SR_MASK;
		1068	reg \|= wm << DSPFW_SR_SHIFT;
		1069	I915_WRITE(DSPFW1, reg);
		1070	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		1071
		1072	/* cursor SR */
		1073	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		1074	pineview_display_wm.fifo_size,
		1075	pixel_size, latency->cursor_sr);
		1076	reg = I915_READ(DSPFW3);
		1077	reg &= ~DSPFW_CURSOR_SR_MASK;
		1078	reg \|= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
		1079	I915_WRITE(DSPFW3, reg);
		1080
		1081	/* Display HPLL off SR */
		1082	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		1083	pineview_display_hplloff_wm.fifo_size,
		1084	pixel_size, latency->display_hpll_disable);
		1085	reg = I915_READ(DSPFW3);
		1086	reg &= ~DSPFW_HPLL_SR_MASK;
		1087	reg \|= wm & DSPFW_HPLL_SR_MASK;
		1088	I915_WRITE(DSPFW3, reg);
		1089
		1090	/* cursor HPLL off SR */
		1091	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		1092	pineview_display_hplloff_wm.fifo_size,
		1093	pixel_size, latency->cursor_hpll_disable);
		1094	reg = I915_READ(DSPFW3);
		1095	reg &= ~DSPFW_HPLL_CURSOR_MASK;
		1096	reg \|= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
		1097	I915_WRITE(DSPFW3, reg);
		1098	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		1099
		1100	/* activate cxsr */
		1101	I915_WRITE(DSPFW3,
		1102	I915_READ(DSPFW3) \| PINEVIEW_SELF_REFRESH_EN);
		1103	DRM_DEBUG_KMS("Self-refresh is enabled\n");
		1104	} else {
		1105	pineview_disable_cxsr(dev);
		1106	DRM_DEBUG_KMS("Self-refresh is disabled\n");
		1107	}
		1108	}
		1109
		1110	static bool g4x_compute_wm0(struct drm_device *dev,
		1111	int plane,
		1112	const struct intel_watermark_params *display,
		1113	int display_latency_ns,
		1114	const struct intel_watermark_params *cursor,
		1115	int cursor_latency_ns,
		1116	int *plane_wm,
		1117	int *cursor_wm)
		1118	{
		1119	struct drm_crtc *crtc;
4560	Serge	1120	const struct drm_display_mode *adjusted_mode;
3031	serge	1121	int htotal, hdisplay, clock, pixel_size;
		1122	int line_time_us, line_count;
		1123	int entries, tlb_miss;
		1124
		1125	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1126	if (!intel_crtc_active(crtc)) {
3031	serge	1127	*cursor_wm = cursor->guard_size;
		1128	*plane_wm = display->guard_size;
		1129	return false;
		1130	}
		1131
4560	Serge	1132	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1133	clock = adjusted_mode->crtc_clock;
		1134	htotal = adjusted_mode->crtc_htotal;
		1135	hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
3031	serge	1136	pixel_size = crtc->fb->bits_per_pixel / 8;
		1137
		1138	/* Use the small buffer method to calculate plane watermark */
		1139	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1140	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		1141	if (tlb_miss > 0)
		1142	entries += tlb_miss;
		1143	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1144	*plane_wm = entries + display->guard_size;
		1145	if (*plane_wm > (int)display->max_wm)
		1146	*plane_wm = display->max_wm;
		1147
		1148	/* Use the large buffer method to calculate cursor watermark */
		1149	line_time_us = ((htotal * 1000) / clock);
		1150	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
		1151	entries = line_count * 64 * pixel_size;
		1152	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		1153	if (tlb_miss > 0)
		1154	entries += tlb_miss;
		1155	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1156	*cursor_wm = entries + cursor->guard_size;
		1157	if (*cursor_wm > (int)cursor->max_wm)
		1158	*cursor_wm = (int)cursor->max_wm;
		1159
		1160	return true;
		1161	}
		1162
		1163	/*
		1164	* Check the wm result.
		1165	*
		1166	* If any calculated watermark values is larger than the maximum value that
		1167	* can be programmed into the associated watermark register, that watermark
		1168	* must be disabled.
		1169	*/
		1170	static bool g4x_check_srwm(struct drm_device *dev,
		1171	int display_wm, int cursor_wm,
		1172	const struct intel_watermark_params *display,
		1173	const struct intel_watermark_params *cursor)
		1174	{
		1175	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		1176	display_wm, cursor_wm);
		1177
		1178	if (display_wm > display->max_wm) {
		1179	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		1180	display_wm, display->max_wm);
		1181	return false;
		1182	}
		1183
		1184	if (cursor_wm > cursor->max_wm) {
		1185	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		1186	cursor_wm, cursor->max_wm);
		1187	return false;
		1188	}
		1189
		1190	if (!(display_wm \|\| cursor_wm)) {
		1191	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		1192	return false;
		1193	}
		1194
		1195	return true;
		1196	}
		1197
		1198	static bool g4x_compute_srwm(struct drm_device *dev,
		1199	int plane,
		1200	int latency_ns,
		1201	const struct intel_watermark_params *display,
		1202	const struct intel_watermark_params *cursor,
		1203	int display_wm, int cursor_wm)
		1204	{
		1205	struct drm_crtc *crtc;
4560	Serge	1206	const struct drm_display_mode *adjusted_mode;
3031	serge	1207	int hdisplay, htotal, pixel_size, clock;
		1208	unsigned long line_time_us;
		1209	int line_count, line_size;
		1210	int small, large;
		1211	int entries;
		1212
		1213	if (!latency_ns) {
		1214	display_wm = cursor_wm = 0;
		1215	return false;
		1216	}
		1217
		1218	crtc = intel_get_crtc_for_plane(dev, plane);
4560	Serge	1219	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1220	clock = adjusted_mode->crtc_clock;
		1221	htotal = adjusted_mode->crtc_htotal;
		1222	hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
3031	serge	1223	pixel_size = crtc->fb->bits_per_pixel / 8;
		1224
		1225	line_time_us = (htotal * 1000) / clock;
		1226	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1227	line_size = hdisplay * pixel_size;
		1228
		1229	/* Use the minimum of the small and large buffer method for primary */
		1230	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1231	large = line_count * line_size;
		1232
		1233	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1234	*display_wm = entries + display->guard_size;
		1235
		1236	/* calculate the self-refresh watermark for display cursor */
		1237	entries = line_count * pixel_size * 64;
		1238	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1239	*cursor_wm = entries + cursor->guard_size;
		1240
		1241	return g4x_check_srwm(dev,
		1242	display_wm, cursor_wm,
		1243	display, cursor);
		1244	}
		1245
		1246	static bool vlv_compute_drain_latency(struct drm_device *dev,
		1247	int plane,
		1248	int *plane_prec_mult,
		1249	int *plane_dl,
		1250	int *cursor_prec_mult,
		1251	int *cursor_dl)
		1252	{
		1253	struct drm_crtc *crtc;
		1254	int clock, pixel_size;
		1255	int entries;
		1256
		1257	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1258	if (!intel_crtc_active(crtc))
3031	serge	1259	return false;
		1260
4560	Serge	1261	clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
3031	serge	1262	pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */
		1263
		1264	entries = (clock / 1000) * pixel_size;
		1265	*plane_prec_mult = (entries > 256) ?
		1266	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1267	plane_dl = (64 (plane_prec_mult) 4) / ((clock / 1000) *
		1268	pixel_size);
		1269
		1270	entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
		1271	*cursor_prec_mult = (entries > 256) ?
		1272	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1273	cursor_dl = (64 (cursor_prec_mult) 4) / ((clock / 1000) * 4);
		1274
		1275	return true;
		1276	}
		1277
		1278	/*
		1279	* Update drain latency registers of memory arbiter
		1280	*
		1281	* Valleyview SoC has a new memory arbiter and needs drain latency registers
		1282	* to be programmed. Each plane has a drain latency multiplier and a drain
		1283	* latency value.
		1284	*/
		1285
		1286	static void vlv_update_drain_latency(struct drm_device *dev)
		1287	{
		1288	struct drm_i915_private *dev_priv = dev->dev_private;
		1289	int planea_prec, planea_dl, planeb_prec, planeb_dl;
		1290	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
		1291	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
		1292	either 16 or 32 */
		1293
		1294	/* For plane A, Cursor A */
		1295	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
		1296	&cursor_prec_mult, &cursora_dl)) {
		1297	cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1298	DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
		1299	planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1300	DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
		1301
		1302	I915_WRITE(VLV_DDL1, cursora_prec \|
		1303	(cursora_dl << DDL_CURSORA_SHIFT) \|
		1304	planea_prec \| planea_dl);
		1305	}
		1306
		1307	/* For plane B, Cursor B */
		1308	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
		1309	&cursor_prec_mult, &cursorb_dl)) {
		1310	cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1311	DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
		1312	planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1313	DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
		1314
		1315	I915_WRITE(VLV_DDL2, cursorb_prec \|
		1316	(cursorb_dl << DDL_CURSORB_SHIFT) \|
		1317	planeb_prec \| planeb_dl);
		1318	}
		1319	}
		1320
		1321	#define single_plane_enabled(mask) is_power_of_2(mask)
		1322
4560	Serge	1323	static void valleyview_update_wm(struct drm_crtc *crtc)
3031	serge	1324	{
4560	Serge	1325	struct drm_device *dev = crtc->dev;
3031	serge	1326	static const int sr_latency_ns = 12000;
		1327	struct drm_i915_private *dev_priv = dev->dev_private;
		1328	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1329	int plane_sr, cursor_sr;
3243	Serge	1330	int ignore_plane_sr, ignore_cursor_sr;
3031	serge	1331	unsigned int enabled = 0;
		1332
		1333	vlv_update_drain_latency(dev);
		1334
3746	Serge	1335	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1336	&valleyview_wm_info, latency_ns,
		1337	&valleyview_cursor_wm_info, latency_ns,
		1338	&planea_wm, &cursora_wm))
3746	Serge	1339	enabled \|= 1 << PIPE_A;
3031	serge	1340
3746	Serge	1341	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1342	&valleyview_wm_info, latency_ns,
		1343	&valleyview_cursor_wm_info, latency_ns,
		1344	&planeb_wm, &cursorb_wm))
3746	Serge	1345	enabled \|= 1 << PIPE_B;
3031	serge	1346
		1347	if (single_plane_enabled(enabled) &&
		1348	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1349	sr_latency_ns,
		1350	&valleyview_wm_info,
		1351	&valleyview_cursor_wm_info,
3243	Serge	1352	&plane_sr, &ignore_cursor_sr) &&
		1353	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1354	2*sr_latency_ns,
		1355	&valleyview_wm_info,
		1356	&valleyview_cursor_wm_info,
		1357	&ignore_plane_sr, &cursor_sr)) {
3031	serge	1358	I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
3243	Serge	1359	} else {
3031	serge	1360	I915_WRITE(FW_BLC_SELF_VLV,
		1361	I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
3243	Serge	1362	plane_sr = cursor_sr = 0;
		1363	}
3031	serge	1364
		1365	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1366	planea_wm, cursora_wm,
		1367	planeb_wm, cursorb_wm,
		1368	plane_sr, cursor_sr);
		1369
		1370	I915_WRITE(DSPFW1,
		1371	(plane_sr << DSPFW_SR_SHIFT) \|
		1372	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1373	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1374	planea_wm);
		1375	I915_WRITE(DSPFW2,
3243	Serge	1376	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1377	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1378	I915_WRITE(DSPFW3,
3243	Serge	1379	(I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) \|
		1380	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
3031	serge	1381	}
		1382
4560	Serge	1383	static void g4x_update_wm(struct drm_crtc *crtc)
3031	serge	1384	{
4560	Serge	1385	struct drm_device *dev = crtc->dev;
3031	serge	1386	static const int sr_latency_ns = 12000;
		1387	struct drm_i915_private *dev_priv = dev->dev_private;
		1388	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1389	int plane_sr, cursor_sr;
		1390	unsigned int enabled = 0;
		1391
3746	Serge	1392	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1393	&g4x_wm_info, latency_ns,
		1394	&g4x_cursor_wm_info, latency_ns,
		1395	&planea_wm, &cursora_wm))
3746	Serge	1396	enabled \|= 1 << PIPE_A;
3031	serge	1397
3746	Serge	1398	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1399	&g4x_wm_info, latency_ns,
		1400	&g4x_cursor_wm_info, latency_ns,
		1401	&planeb_wm, &cursorb_wm))
3746	Serge	1402	enabled \|= 1 << PIPE_B;
3031	serge	1403
		1404	if (single_plane_enabled(enabled) &&
		1405	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1406	sr_latency_ns,
		1407	&g4x_wm_info,
		1408	&g4x_cursor_wm_info,
3243	Serge	1409	&plane_sr, &cursor_sr)) {
3031	serge	1410	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
3243	Serge	1411	} else {
3031	serge	1412	I915_WRITE(FW_BLC_SELF,
		1413	I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
3243	Serge	1414	plane_sr = cursor_sr = 0;
		1415	}
3031	serge	1416
		1417	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1418	planea_wm, cursora_wm,
		1419	planeb_wm, cursorb_wm,
		1420	plane_sr, cursor_sr);
		1421
		1422	I915_WRITE(DSPFW1,
		1423	(plane_sr << DSPFW_SR_SHIFT) \|
		1424	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1425	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1426	planea_wm);
		1427	I915_WRITE(DSPFW2,
3243	Serge	1428	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1429	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1430	/* HPLL off in SR has some issues on G4x... disable it */
		1431	I915_WRITE(DSPFW3,
3243	Serge	1432	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
3031	serge	1433	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1434	}
		1435
4560	Serge	1436	static void i965_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1437	{
4560	Serge	1438	struct drm_device *dev = unused_crtc->dev;
3031	serge	1439	struct drm_i915_private *dev_priv = dev->dev_private;
		1440	struct drm_crtc *crtc;
		1441	int srwm = 1;
		1442	int cursor_sr = 16;
		1443
		1444	/* Calc sr entries for one plane configs */
		1445	crtc = single_enabled_crtc(dev);
		1446	if (crtc) {
		1447	/* self-refresh has much higher latency */
		1448	static const int sr_latency_ns = 12000;
4560	Serge	1449	const struct drm_display_mode *adjusted_mode =
		1450	&to_intel_crtc(crtc)->config.adjusted_mode;
		1451	int clock = adjusted_mode->crtc_clock;
		1452	int htotal = adjusted_mode->crtc_htotal;
		1453	int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
3031	serge	1454	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1455	unsigned long line_time_us;
		1456	int entries;
		1457
		1458	line_time_us = ((htotal * 1000) / clock);
		1459
		1460	/* Use ns/us then divide to preserve precision */
		1461	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1462	pixel_size * hdisplay;
		1463	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1464	srwm = I965_FIFO_SIZE - entries;
		1465	if (srwm < 0)
		1466	srwm = 1;
		1467	srwm &= 0x1ff;
		1468	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1469	entries, srwm);
		1470
		1471	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1472	pixel_size * 64;
		1473	entries = DIV_ROUND_UP(entries,
		1474	i965_cursor_wm_info.cacheline_size);
		1475	cursor_sr = i965_cursor_wm_info.fifo_size -
		1476	(entries + i965_cursor_wm_info.guard_size);
		1477
		1478	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1479	cursor_sr = i965_cursor_wm_info.max_wm;
		1480
		1481	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1482	"cursor %d\n", srwm, cursor_sr);
		1483
		1484	if (IS_CRESTLINE(dev))
		1485	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
		1486	} else {
		1487	/* Turn off self refresh if both pipes are enabled */
		1488	if (IS_CRESTLINE(dev))
		1489	I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
		1490	& ~FW_BLC_SELF_EN);
		1491	}
		1492
		1493	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1494	srwm);
		1495
		1496	/* 965 has limitations... */
		1497	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) \|
		1498	(8 << 16) \| (8 << 8) \| (8 << 0));
		1499	I915_WRITE(DSPFW2, (8 << 8) \| (8 << 0));
		1500	/* update cursor SR watermark */
		1501	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1502	}
		1503
4560	Serge	1504	static void i9xx_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1505	{
4560	Serge	1506	struct drm_device *dev = unused_crtc->dev;
3031	serge	1507	struct drm_i915_private *dev_priv = dev->dev_private;
		1508	const struct intel_watermark_params *wm_info;
		1509	uint32_t fwater_lo;
		1510	uint32_t fwater_hi;
		1511	int cwm, srwm = 1;
		1512	int fifo_size;
		1513	int planea_wm, planeb_wm;
		1514	struct drm_crtc crtc, enabled = NULL;
		1515
		1516	if (IS_I945GM(dev))
		1517	wm_info = &i945_wm_info;
		1518	else if (!IS_GEN2(dev))
		1519	wm_info = &i915_wm_info;
		1520	else
4560	Serge	1521	wm_info = &i830_wm_info;
3031	serge	1522
		1523	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1524	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1525	if (intel_crtc_active(crtc)) {
4560	Serge	1526	const struct drm_display_mode *adjusted_mode;
3243	Serge	1527	int cpp = crtc->fb->bits_per_pixel / 8;
		1528	if (IS_GEN2(dev))
		1529	cpp = 4;
		1530
4560	Serge	1531	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1532	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1533	wm_info, fifo_size, cpp,
3031	serge	1534	latency_ns);
		1535	enabled = crtc;
		1536	} else
		1537	planea_wm = fifo_size - wm_info->guard_size;
		1538
		1539	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1540	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1541	if (intel_crtc_active(crtc)) {
4560	Serge	1542	const struct drm_display_mode *adjusted_mode;
3243	Serge	1543	int cpp = crtc->fb->bits_per_pixel / 8;
		1544	if (IS_GEN2(dev))
		1545	cpp = 4;
		1546
4560	Serge	1547	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1548	planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1549	wm_info, fifo_size, cpp,
3031	serge	1550	latency_ns);
		1551	if (enabled == NULL)
		1552	enabled = crtc;
		1553	else
		1554	enabled = NULL;
		1555	} else
		1556	planeb_wm = fifo_size - wm_info->guard_size;
		1557
		1558	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1559
		1560	/*
		1561	* Overlay gets an aggressive default since video jitter is bad.
		1562	*/
		1563	cwm = 2;
		1564
		1565	/* Play safe and disable self-refresh before adjusting watermarks. */
		1566	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1567	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK \| 0);
		1568	else if (IS_I915GM(dev))
4560	Serge	1569	I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_SELF_EN));
3031	serge	1570
		1571	/* Calc sr entries for one plane configs */
		1572	if (HAS_FW_BLC(dev) && enabled) {
		1573	/* self-refresh has much higher latency */
		1574	static const int sr_latency_ns = 6000;
4560	Serge	1575	const struct drm_display_mode *adjusted_mode =
		1576	&to_intel_crtc(enabled)->config.adjusted_mode;
		1577	int clock = adjusted_mode->crtc_clock;
		1578	int htotal = adjusted_mode->crtc_htotal;
		1579	int hdisplay = to_intel_crtc(enabled)->config.pipe_src_w;
3031	serge	1580	int pixel_size = enabled->fb->bits_per_pixel / 8;
		1581	unsigned long line_time_us;
		1582	int entries;
		1583
		1584	line_time_us = (htotal * 1000) / clock;
		1585
		1586	/* Use ns/us then divide to preserve precision */
		1587	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1588	pixel_size * hdisplay;
		1589	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1590	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1591	srwm = wm_info->fifo_size - entries;
		1592	if (srwm < 0)
		1593	srwm = 1;
		1594
		1595	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1596	I915_WRITE(FW_BLC_SELF,
		1597	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1598	else if (IS_I915GM(dev))
		1599	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1600	}
		1601
		1602	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1603	planea_wm, planeb_wm, cwm, srwm);
		1604
		1605	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1606	fwater_hi = (cwm & 0x1f);
		1607
		1608	/* Set request length to 8 cachelines per fetch */
		1609	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1610	fwater_hi = fwater_hi \| (1 << 8);
		1611
		1612	I915_WRITE(FW_BLC, fwater_lo);
		1613	I915_WRITE(FW_BLC2, fwater_hi);
		1614
		1615	if (HAS_FW_BLC(dev)) {
		1616	if (enabled) {
		1617	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1618	I915_WRITE(FW_BLC_SELF,
		1619	FW_BLC_SELF_EN_MASK \| FW_BLC_SELF_EN);
		1620	else if (IS_I915GM(dev))
4560	Serge	1621	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_SELF_EN));
3031	serge	1622	DRM_DEBUG_KMS("memory self refresh enabled\n");
		1623	} else
		1624	DRM_DEBUG_KMS("memory self refresh disabled\n");
		1625	}
		1626	}
		1627
4560	Serge	1628	static void i845_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1629	{
4560	Serge	1630	struct drm_device *dev = unused_crtc->dev;
3031	serge	1631	struct drm_i915_private *dev_priv = dev->dev_private;
		1632	struct drm_crtc *crtc;
4560	Serge	1633	const struct drm_display_mode *adjusted_mode;
3031	serge	1634	uint32_t fwater_lo;
		1635	int planea_wm;
		1636
		1637	crtc = single_enabled_crtc(dev);
		1638	if (crtc == NULL)
		1639	return;
		1640
4560	Serge	1641	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1642	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
		1643	&i845_wm_info,
3031	serge	1644	dev_priv->display.get_fifo_size(dev, 0),
3243	Serge	1645	4, latency_ns);
3031	serge	1646	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1647	fwater_lo \|= (3<<8) \| planea_wm;
		1648
		1649	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1650
		1651	I915_WRITE(FW_BLC, fwater_lo);
		1652	}
		1653
4104	Serge	1654	static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
		1655	struct drm_crtc *crtc)
3031	serge	1656	{
4104	Serge	1657	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1658	uint32_t pixel_rate;
		1659
4560	Serge	1660	pixel_rate = intel_crtc->config.adjusted_mode.crtc_clock;
4104	Serge	1661
		1662	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
		1663	* adjust the pixel_rate here. */
		1664
		1665	if (intel_crtc->config.pch_pfit.enabled) {
		1666	uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
		1667	uint32_t pfit_size = intel_crtc->config.pch_pfit.size;
		1668
4560	Serge	1669	pipe_w = intel_crtc->config.pipe_src_w;
		1670	pipe_h = intel_crtc->config.pipe_src_h;
4104	Serge	1671	pfit_w = (pfit_size >> 16) & 0xFFFF;
		1672	pfit_h = pfit_size & 0xFFFF;
		1673	if (pipe_w < pfit_w)
		1674	pipe_w = pfit_w;
		1675	if (pipe_h < pfit_h)
		1676	pipe_h = pfit_h;
		1677
		1678	pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
		1679	pfit_w * pfit_h);
		1680	}
		1681
		1682	return pixel_rate;
		1683	}
		1684
		1685	/* latency must be in 0.1us units. */
		1686	static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		1687	uint32_t latency)
		1688	{
		1689	uint64_t ret;
		1690
		1691	if (WARN(latency == 0, "Latency value missing\n"))
		1692	return UINT_MAX;
		1693
		1694	ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
		1695	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
		1696
		1697	return ret;
		1698	}
		1699
		1700	/* latency must be in 0.1us units. */
		1701	static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		1702	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
		1703	uint32_t latency)
		1704	{
		1705	uint32_t ret;
		1706
		1707	if (WARN(latency == 0, "Latency value missing\n"))
		1708	return UINT_MAX;
		1709
		1710	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		1711	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		1712	ret = DIV_ROUND_UP(ret, 64) + 2;
		1713	return ret;
		1714	}
		1715
		1716	static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
		1717	uint8_t bytes_per_pixel)
		1718	{
		1719	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
		1720	}
		1721
4560	Serge	1722	struct ilk_pipe_wm_parameters {
4104	Serge	1723	bool active;
		1724	uint32_t pipe_htotal;
		1725	uint32_t pixel_rate;
		1726	struct intel_plane_wm_parameters pri;
		1727	struct intel_plane_wm_parameters spr;
		1728	struct intel_plane_wm_parameters cur;
		1729	};
		1730
4560	Serge	1731	struct ilk_wm_maximums {
4104	Serge	1732	uint16_t pri;
		1733	uint16_t spr;
		1734	uint16_t cur;
		1735	uint16_t fbc;
		1736	};
		1737
		1738	/* used in computing the new watermarks state */
		1739	struct intel_wm_config {
		1740	unsigned int num_pipes_active;
		1741	bool sprites_enabled;
		1742	bool sprites_scaled;
		1743	};
		1744
		1745	/*
		1746	* For both WM_PIPE and WM_LP.
		1747	* mem_value must be in 0.1us units.
		1748	*/
4560	Serge	1749	static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	1750	uint32_t mem_value,
		1751	bool is_lp)
		1752	{
		1753	uint32_t method1, method2;
		1754
		1755	if (!params->active \|\| !params->pri.enabled)
		1756	return 0;
		1757
		1758	method1 = ilk_wm_method1(params->pixel_rate,
		1759	params->pri.bytes_per_pixel,
		1760	mem_value);
		1761
		1762	if (!is_lp)
		1763	return method1;
		1764
		1765	method2 = ilk_wm_method2(params->pixel_rate,
		1766	params->pipe_htotal,
		1767	params->pri.horiz_pixels,
		1768	params->pri.bytes_per_pixel,
		1769	mem_value);
		1770
		1771	return min(method1, method2);
		1772	}
		1773
		1774	/*
		1775	* For both WM_PIPE and WM_LP.
		1776	* mem_value must be in 0.1us units.
		1777	*/
4560	Serge	1778	static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	1779	uint32_t mem_value)
		1780	{
		1781	uint32_t method1, method2;
		1782
		1783	if (!params->active \|\| !params->spr.enabled)
		1784	return 0;
		1785
		1786	method1 = ilk_wm_method1(params->pixel_rate,
		1787	params->spr.bytes_per_pixel,
		1788	mem_value);
		1789	method2 = ilk_wm_method2(params->pixel_rate,
		1790	params->pipe_htotal,
		1791	params->spr.horiz_pixels,
		1792	params->spr.bytes_per_pixel,
		1793	mem_value);
		1794	return min(method1, method2);
		1795	}
		1796
		1797	/*
		1798	* For both WM_PIPE and WM_LP.
		1799	* mem_value must be in 0.1us units.
		1800	*/
4560	Serge	1801	static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	1802	uint32_t mem_value)
		1803	{
		1804	if (!params->active \|\| !params->cur.enabled)
		1805	return 0;
		1806
		1807	return ilk_wm_method2(params->pixel_rate,
		1808	params->pipe_htotal,
		1809	params->cur.horiz_pixels,
		1810	params->cur.bytes_per_pixel,
		1811	mem_value);
		1812	}
		1813
		1814	/* Only for WM_LP. */
4560	Serge	1815	static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	1816	uint32_t pri_val)
		1817	{
		1818	if (!params->active \|\| !params->pri.enabled)
		1819	return 0;
		1820
		1821	return ilk_wm_fbc(pri_val,
		1822	params->pri.horiz_pixels,
		1823	params->pri.bytes_per_pixel);
		1824	}
		1825
		1826	static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
		1827	{
4560	Serge	1828	if (INTEL_INFO(dev)->gen >= 8)
		1829	return 3072;
		1830	else if (INTEL_INFO(dev)->gen >= 7)
4104	Serge	1831	return 768;
		1832	else
		1833	return 512;
		1834	}
		1835
		1836	/* Calculate the maximum primary/sprite plane watermark */
		1837	static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
		1838	int level,
		1839	const struct intel_wm_config *config,
		1840	enum intel_ddb_partitioning ddb_partitioning,
		1841	bool is_sprite)
		1842	{
		1843	unsigned int fifo_size = ilk_display_fifo_size(dev);
		1844	unsigned int max;
		1845
		1846	/* if sprites aren't enabled, sprites get nothing */
		1847	if (is_sprite && !config->sprites_enabled)
		1848	return 0;
		1849
		1850	/* HSW allows LP1+ watermarks even with multiple pipes */
		1851	if (level == 0 \|\| config->num_pipes_active > 1) {
		1852	fifo_size /= INTEL_INFO(dev)->num_pipes;
		1853
		1854	/*
		1855	* For some reason the non self refresh
		1856	* FIFO size is only half of the self
		1857	* refresh FIFO size on ILK/SNB.
		1858	*/
		1859	if (INTEL_INFO(dev)->gen <= 6)
		1860	fifo_size /= 2;
		1861	}
		1862
		1863	if (config->sprites_enabled) {
		1864	/* level 0 is always calculated with 1:1 split */
		1865	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
		1866	if (is_sprite)
		1867	fifo_size *= 5;
		1868	fifo_size /= 6;
		1869	} else {
		1870	fifo_size /= 2;
		1871	}
		1872	}
		1873
		1874	/* clamp to max that the registers can hold */
4560	Serge	1875	if (INTEL_INFO(dev)->gen >= 8)
		1876	max = level == 0 ? 255 : 2047;
		1877	else if (INTEL_INFO(dev)->gen >= 7)
4104	Serge	1878	/* IVB/HSW primary/sprite plane watermarks */
		1879	max = level == 0 ? 127 : 1023;
		1880	else if (!is_sprite)
		1881	/* ILK/SNB primary plane watermarks */
		1882	max = level == 0 ? 127 : 511;
		1883	else
		1884	/* ILK/SNB sprite plane watermarks */
		1885	max = level == 0 ? 63 : 255;
		1886
		1887	return min(fifo_size, max);
		1888	}
		1889
		1890	/* Calculate the maximum cursor plane watermark */
		1891	static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
		1892	int level,
		1893	const struct intel_wm_config *config)
		1894	{
		1895	/* HSW LP1+ watermarks w/ multiple pipes */
		1896	if (level > 0 && config->num_pipes_active > 1)
		1897	return 64;
		1898
		1899	/* otherwise just report max that registers can hold */
		1900	if (INTEL_INFO(dev)->gen >= 7)
		1901	return level == 0 ? 63 : 255;
		1902	else
		1903	return level == 0 ? 31 : 63;
4539	Serge	1904	}
4104	Serge	1905
		1906	/* Calculate the maximum FBC watermark */
4560	Serge	1907	static unsigned int ilk_fbc_wm_max(struct drm_device *dev)
4104	Serge	1908	{
		1909	/* max that registers can hold */
4560	Serge	1910	if (INTEL_INFO(dev)->gen >= 8)
		1911	return 31;
		1912	else
4104	Serge	1913	return 15;
		1914	}
		1915
4560	Serge	1916	static void ilk_compute_wm_maximums(struct drm_device *dev,
4104	Serge	1917	int level,
		1918	const struct intel_wm_config *config,
		1919	enum intel_ddb_partitioning ddb_partitioning,
4560	Serge	1920	struct ilk_wm_maximums *max)
4104	Serge	1921	{
		1922	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
		1923	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
		1924	max->cur = ilk_cursor_wm_max(dev, level, config);
4560	Serge	1925	max->fbc = ilk_fbc_wm_max(dev);
4539	Serge	1926	}
4104	Serge	1927
4560	Serge	1928	static bool ilk_validate_wm_level(int level,
		1929	const struct ilk_wm_maximums *max,
4104	Serge	1930	struct intel_wm_level *result)
		1931	{
		1932	bool ret;
		1933
		1934	/* already determined to be invalid? */
		1935	if (!result->enable)
		1936	return false;
		1937
		1938	result->enable = result->pri_val <= max->pri &&
		1939	result->spr_val <= max->spr &&
		1940	result->cur_val <= max->cur;
		1941
		1942	ret = result->enable;
		1943
		1944	/*
		1945	* HACK until we can pre-compute everything,
		1946	* and thus fail gracefully if LP0 watermarks
		1947	* are exceeded...
		1948	*/
		1949	if (level == 0 && !result->enable) {
		1950	if (result->pri_val > max->pri)
		1951	DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
		1952	level, result->pri_val, max->pri);
		1953	if (result->spr_val > max->spr)
		1954	DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
		1955	level, result->spr_val, max->spr);
		1956	if (result->cur_val > max->cur)
		1957	DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
		1958	level, result->cur_val, max->cur);
		1959
		1960	result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
		1961	result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
		1962	result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
		1963	result->enable = true;
		1964	}
		1965
		1966	return ret;
		1967	}
		1968
		1969	static void ilk_compute_wm_level(struct drm_i915_private *dev_priv,
		1970	int level,
4560	Serge	1971	const struct ilk_pipe_wm_parameters *p,
4104	Serge	1972	struct intel_wm_level *result)
		1973	{
		1974	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
		1975	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
		1976	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
		1977
		1978	/* WM1+ latency values stored in 0.5us units */
		1979	if (level > 0) {
		1980	pri_latency *= 5;
		1981	spr_latency *= 5;
		1982	cur_latency *= 5;
		1983	}
		1984
		1985	result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
		1986	result->spr_val = ilk_compute_spr_wm(p, spr_latency);
		1987	result->cur_val = ilk_compute_cur_wm(p, cur_latency);
		1988	result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
		1989	result->enable = true;
		1990	}
		1991
		1992	static uint32_t
		1993	hsw_compute_linetime_wm(struct drm_device dev, struct drm_crtc crtc)
		1994	{
3031	serge	1995	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	1996	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1997	struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
		1998	u32 linetime, ips_linetime;
3031	serge	1999
4104	Serge	2000	if (!intel_crtc_active(crtc))
		2001	return 0;
3031	serge	2002
		2003	/* The WM are computed with base on how long it takes to fill a single
		2004	* row at the given clock rate, multiplied by 8.
		2005	* */
4560	Serge	2006	linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
		2007	mode->crtc_clock);
		2008	ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
4104	Serge	2009	intel_ddi_get_cdclk_freq(dev_priv));
3031	serge	2010
4104	Serge	2011	return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) \|
		2012	PIPE_WM_LINETIME_TIME(linetime);
		2013	}
		2014
		2015	static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2016	{
		2017	struct drm_i915_private *dev_priv = dev->dev_private;
		2018
4560	Serge	2019	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2020	uint64_t sskpd = I915_READ64(MCH_SSKPD);
		2021
		2022	wm[0] = (sskpd >> 56) & 0xFF;
		2023	if (wm[0] == 0)
		2024	wm[0] = sskpd & 0xF;
		2025	wm[1] = (sskpd >> 4) & 0xFF;
		2026	wm[2] = (sskpd >> 12) & 0xFF;
		2027	wm[3] = (sskpd >> 20) & 0x1FF;
		2028	wm[4] = (sskpd >> 32) & 0x1FF;
		2029	} else if (INTEL_INFO(dev)->gen >= 6) {
		2030	uint32_t sskpd = I915_READ(MCH_SSKPD);
		2031
		2032	wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		2033	wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		2034	wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		2035	wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
		2036	} else if (INTEL_INFO(dev)->gen >= 5) {
		2037	uint32_t mltr = I915_READ(MLTR_ILK);
		2038
		2039	/* ILK primary LP0 latency is 700 ns */
		2040	wm[0] = 7;
		2041	wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		2042	wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
		2043	}
		2044	}
		2045
		2046	static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2047	{
		2048	/* ILK sprite LP0 latency is 1300 ns */
		2049	if (INTEL_INFO(dev)->gen == 5)
		2050	wm[0] = 13;
		2051	}
		2052
		2053	static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2054	{
		2055	/* ILK cursor LP0 latency is 1300 ns */
		2056	if (INTEL_INFO(dev)->gen == 5)
		2057	wm[0] = 13;
		2058
		2059	/* WaDoubleCursorLP3Latency:ivb */
		2060	if (IS_IVYBRIDGE(dev))
		2061	wm[3] *= 2;
		2062	}
		2063
4560	Serge	2064	static int ilk_wm_max_level(const struct drm_device *dev)
		2065	{
		2066	/* how many WM levels are we expecting */
		2067	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2068	return 4;
		2069	else if (INTEL_INFO(dev)->gen >= 6)
		2070	return 3;
		2071	else
		2072	return 2;
		2073	}
		2074
4104	Serge	2075	static void intel_print_wm_latency(struct drm_device *dev,
		2076	const char *name,
		2077	const uint16_t wm[5])
		2078	{
4560	Serge	2079	int level, max_level = ilk_wm_max_level(dev);
4104	Serge	2080
		2081	for (level = 0; level <= max_level; level++) {
		2082	unsigned int latency = wm[level];
		2083
		2084	if (latency == 0) {
		2085	DRM_ERROR("%s WM%d latency not provided\n",
		2086	name, level);
		2087	continue;
		2088	}
		2089
		2090	/* WM1+ latency values in 0.5us units */
		2091	if (level > 0)
		2092	latency *= 5;
		2093
		2094	DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
		2095	name, level, wm[level],
		2096	latency / 10, latency % 10);
		2097	}
		2098	}
		2099
		2100	static void intel_setup_wm_latency(struct drm_device *dev)
		2101	{
		2102	struct drm_i915_private *dev_priv = dev->dev_private;
		2103
		2104	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
		2105
		2106	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
		2107	sizeof(dev_priv->wm.pri_latency));
		2108	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
		2109	sizeof(dev_priv->wm.pri_latency));
		2110
		2111	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
		2112	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
		2113
		2114	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2115	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2116	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
		2117	}
		2118
4560	Serge	2119	static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
		2120	struct ilk_pipe_wm_parameters *p,
		2121	struct intel_wm_config *config)
4104	Serge	2122	{
4560	Serge	2123	struct drm_device *dev = crtc->dev;
		2124	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2125	enum pipe pipe = intel_crtc->pipe;
4104	Serge	2126	struct drm_plane *plane;
		2127
		2128	p->active = intel_crtc_active(crtc);
4560	Serge	2129	if (p->active) {
		2130	p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal;
4104	Serge	2131	p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
		2132	p->pri.bytes_per_pixel = crtc->fb->bits_per_pixel / 8;
		2133	p->cur.bytes_per_pixel = 4;
4560	Serge	2134	p->pri.horiz_pixels = intel_crtc->config.pipe_src_w;
4104	Serge	2135	p->cur.horiz_pixels = 64;
		2136	/* TODO: for now, assume primary and cursor planes are always enabled. */
		2137	p->pri.enabled = true;
		2138	p->cur.enabled = true;
		2139	}
		2140
4560	Serge	2141	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
		2142	config->num_pipes_active += intel_crtc_active(crtc);
		2143
4104	Serge	2144	list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
		2145	struct intel_plane *intel_plane = to_intel_plane(plane);
		2146
4560	Serge	2147	if (intel_plane->pipe == pipe)
4104	Serge	2148	p->spr = intel_plane->wm;
		2149
4560	Serge	2150	config->sprites_enabled \|= intel_plane->wm.enabled;
		2151	config->sprites_scaled \|= intel_plane->wm.scaled;
4104	Serge	2152	}
4560	Serge	2153	}
4104	Serge	2154
4560	Serge	2155	/* Compute new watermarks for the pipe */
		2156	static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
		2157	const struct ilk_pipe_wm_parameters *params,
		2158	struct intel_pipe_wm *pipe_wm)
		2159	{
		2160	struct drm_device *dev = crtc->dev;
		2161	struct drm_i915_private *dev_priv = dev->dev_private;
		2162	int level, max_level = ilk_wm_max_level(dev);
		2163	/* LP0 watermark maximums depend on this pipe alone */
		2164	struct intel_wm_config config = {
		2165	.num_pipes_active = 1,
		2166	.sprites_enabled = params->spr.enabled,
		2167	.sprites_scaled = params->spr.scaled,
		2168	};
		2169	struct ilk_wm_maximums max;
4104	Serge	2170
4560	Serge	2171	/* LP0 watermarks always use 1/2 DDB partitioning */
		2172	ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
		2173
		2174	/* ILK/SNB: LP2+ watermarks only w/o sprites */
		2175	if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
		2176	max_level = 1;
		2177
		2178	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
		2179	if (params->spr.scaled)
		2180	max_level = 0;
		2181
		2182	for (level = 0; level <= max_level; level++)
		2183	ilk_compute_wm_level(dev_priv, level, params,
		2184	&pipe_wm->wm[level]);
		2185
		2186	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2187	pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
		2188
		2189	/* At least LP0 must be valid */
		2190	return ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]);
4104	Serge	2191	}
		2192
4560	Serge	2193	/*
		2194	* Merge the watermarks from all active pipes for a specific level.
		2195	*/
		2196	static void ilk_merge_wm_level(struct drm_device *dev,
		2197	int level,
		2198	struct intel_wm_level *ret_wm)
4104	Serge	2199	{
4560	Serge	2200	const struct intel_crtc *intel_crtc;
4104	Serge	2201
4560	Serge	2202	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
		2203	const struct intel_wm_level *wm =
		2204	&intel_crtc->wm.active.wm[level];
4104	Serge	2205
4560	Serge	2206	if (!wm->enable)
		2207	return;
4104	Serge	2208
4560	Serge	2209	ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
		2210	ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
		2211	ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
		2212	ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
		2213	}
		2214
		2215	ret_wm->enable = true;
		2216	}
		2217
		2218	/*
		2219	* Merge all low power watermarks for all active pipes.
		2220	*/
		2221	static void ilk_wm_merge(struct drm_device *dev,
		2222	const struct intel_wm_config *config,
		2223	const struct ilk_wm_maximums *max,
		2224	struct intel_pipe_wm *merged)
		2225	{
		2226	int level, max_level = ilk_wm_max_level(dev);
		2227
		2228	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
		2229	if ((INTEL_INFO(dev)->gen <= 6 \|\| IS_IVYBRIDGE(dev)) &&
		2230	config->num_pipes_active > 1)
		2231	return;
		2232
		2233	/* ILK: FBC WM must be disabled always */
		2234	merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
		2235
		2236	/* merge each WM1+ level */
4104	Serge	2237	for (level = 1; level <= max_level; level++) {
4560	Serge	2238	struct intel_wm_level *wm = &merged->wm[level];
		2239
		2240	ilk_merge_wm_level(dev, level, wm);
		2241
		2242	if (!ilk_validate_wm_level(level, max, wm))
		2243	break;
		2244
		2245	/*
		2246	* The spec says it is preferred to disable
		2247	* FBC WMs instead of disabling a WM level.
		2248	*/
		2249	if (wm->fbc_val > max->fbc) {
		2250	merged->fbc_wm_enabled = false;
		2251	wm->fbc_val = 0;
4104	Serge	2252	}
		2253	}
		2254
4560	Serge	2255	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
		2256	/*
		2257	* FIXME this is racy. FBC might get enabled later.
		2258	* What we should check here is whether FBC can be
		2259	* enabled sometime later.
		2260	*/
		2261	if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
		2262	for (level = 2; level <= max_level; level++) {
		2263	struct intel_wm_level *wm = &merged->wm[level];
		2264
		2265	wm->enable = false;
		2266	}
		2267	}
		2268	}
		2269
		2270	static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
		2271	{
		2272	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
		2273	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
		2274	}
		2275
		2276	/* The value we need to program into the WM_LPx latency field */
		2277	static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
		2278	{
		2279	struct drm_i915_private *dev_priv = dev->dev_private;
		2280
		2281	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2282	return 2 * level;
		2283	else
		2284	return dev_priv->wm.pri_latency[level];
		2285	}
		2286
		2287	static void ilk_compute_wm_results(struct drm_device *dev,
		2288	const struct intel_pipe_wm *merged,
		2289	enum intel_ddb_partitioning partitioning,
		2290	struct ilk_wm_values *results)
		2291	{
		2292	struct intel_crtc *intel_crtc;
		2293	int level, wm_lp;
		2294
		2295	results->enable_fbc_wm = merged->fbc_wm_enabled;
		2296	results->partitioning = partitioning;
		2297
		2298	/* LP1+ register values */
4104	Serge	2299	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2300	const struct intel_wm_level *r;
		2301
4560	Serge	2302	level = ilk_wm_lp_to_level(wm_lp, merged);
		2303
		2304	r = &merged->wm[level];
		2305	if (!r->enable)
4104	Serge	2306	break;
		2307
4560	Serge	2308	results->wm_lp[wm_lp - 1] = WM3_LP_EN \|
		2309	(ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) \|
		2310	(r->pri_val << WM1_LP_SR_SHIFT) \|
		2311	r->cur_val;
		2312
		2313	if (INTEL_INFO(dev)->gen >= 8)
		2314	results->wm_lp[wm_lp - 1] \|=
		2315	r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
		2316	else
		2317	results->wm_lp[wm_lp - 1] \|=
		2318	r->fbc_val << WM1_LP_FBC_SHIFT;
		2319
		2320	if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
		2321	WARN_ON(wm_lp != 1);
		2322	results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN \| r->spr_val;
		2323	} else
4104	Serge	2324	results->wm_lp_spr[wm_lp - 1] = r->spr_val;
		2325	}
		2326
4560	Serge	2327	/* LP0 register values */
		2328	list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head) {
		2329	enum pipe pipe = intel_crtc->pipe;
		2330	const struct intel_wm_level *r =
		2331	&intel_crtc->wm.active.wm[0];
4104	Serge	2332
4560	Serge	2333	if (WARN_ON(!r->enable))
		2334	continue;
		2335
		2336	results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
		2337
		2338	results->wm_pipe[pipe] =
		2339	(r->pri_val << WM0_PIPE_PLANE_SHIFT) \|
		2340	(r->spr_val << WM0_PIPE_SPRITE_SHIFT) \|
		2341	r->cur_val;
4104	Serge	2342	}
		2343	}
		2344
		2345	/* Find the result with the highest level enabled. Check for enable_fbc_wm in
		2346	* case both are at the same level. Prefer r1 in case they're the same. */
4560	Serge	2347	static struct intel_pipe_wm ilk_find_best_result(struct drm_device dev,
		2348	struct intel_pipe_wm *r1,
		2349	struct intel_pipe_wm *r2)
4104	Serge	2350	{
4560	Serge	2351	int level, max_level = ilk_wm_max_level(dev);
		2352	int level1 = 0, level2 = 0;
4104	Serge	2353
4560	Serge	2354	for (level = 1; level <= max_level; level++) {
		2355	if (r1->wm[level].enable)
		2356	level1 = level;
		2357	if (r2->wm[level].enable)
		2358	level2 = level;
4104	Serge	2359	}
		2360
4560	Serge	2361	if (level1 == level2) {
		2362	if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
4104	Serge	2363	return r2;
		2364	else
		2365	return r1;
4560	Serge	2366	} else if (level1 > level2) {
4104	Serge	2367	return r1;
		2368	} else {
		2369	return r2;
		2370	}
		2371	}
		2372
4560	Serge	2373	/* dirty bits used to track which watermarks need changes */
		2374	#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
		2375	#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
		2376	#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
		2377	#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) \| WM_DIRTY_LP(2) \| WM_DIRTY_LP(3))
		2378	#define WM_DIRTY_FBC (1 << 24)
		2379	#define WM_DIRTY_DDB (1 << 25)
		2380
		2381	static unsigned int ilk_compute_wm_dirty(struct drm_device *dev,
		2382	const struct ilk_wm_values *old,
		2383	const struct ilk_wm_values *new)
		2384	{
		2385	unsigned int dirty = 0;
		2386	enum pipe pipe;
		2387	int wm_lp;
		2388
		2389	for_each_pipe(pipe) {
		2390	if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
		2391	dirty \|= WM_DIRTY_LINETIME(pipe);
		2392	/* Must disable LP1+ watermarks too */
		2393	dirty \|= WM_DIRTY_LP_ALL;
		2394	}
		2395
		2396	if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
		2397	dirty \|= WM_DIRTY_PIPE(pipe);
		2398	/* Must disable LP1+ watermarks too */
		2399	dirty \|= WM_DIRTY_LP_ALL;
		2400	}
		2401	}
		2402
		2403	if (old->enable_fbc_wm != new->enable_fbc_wm) {
		2404	dirty \|= WM_DIRTY_FBC;
		2405	/* Must disable LP1+ watermarks too */
		2406	dirty \|= WM_DIRTY_LP_ALL;
		2407	}
		2408
		2409	if (old->partitioning != new->partitioning) {
		2410	dirty \|= WM_DIRTY_DDB;
		2411	/* Must disable LP1+ watermarks too */
		2412	dirty \|= WM_DIRTY_LP_ALL;
		2413	}
		2414
		2415	/* LP1+ watermarks already deemed dirty, no need to continue */
		2416	if (dirty & WM_DIRTY_LP_ALL)
		2417	return dirty;
		2418
		2419	/* Find the lowest numbered LP1+ watermark in need of an update... */
		2420	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2421	if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] \|\|
		2422	old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
		2423	break;
		2424	}
		2425
		2426	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
		2427	for (; wm_lp <= 3; wm_lp++)
		2428	dirty \|= WM_DIRTY_LP(wm_lp);
		2429
		2430	return dirty;
		2431	}
		2432
		2433	static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
		2434	unsigned int dirty)
		2435	{
		2436	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2437	bool changed = false;
		2438
		2439	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
		2440	previous->wm_lp[2] &= ~WM1_LP_SR_EN;
		2441	I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
		2442	changed = true;
		2443	}
		2444	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
		2445	previous->wm_lp[1] &= ~WM1_LP_SR_EN;
		2446	I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
		2447	changed = true;
		2448	}
		2449	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
		2450	previous->wm_lp[0] &= ~WM1_LP_SR_EN;
		2451	I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
		2452	changed = true;
		2453	}
		2454
		2455	/*
		2456	* Don't touch WM1S_LP_EN here.
		2457	* Doing so could cause underruns.
		2458	*/
		2459
		2460	return changed;
		2461	}
		2462
4104	Serge	2463	/*
		2464	* The spec says we shouldn't write when we don't need, because every write
		2465	* causes WMs to be re-evaluated, expending some power.
3031	serge	2466	*/
4560	Serge	2467	static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
		2468	struct ilk_wm_values *results)
4104	Serge	2469	{
4560	Serge	2470	struct drm_device *dev = dev_priv->dev;
		2471	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2472	unsigned int dirty;
4104	Serge	2473	uint32_t val;
3031	serge	2474
4560	Serge	2475	dirty = ilk_compute_wm_dirty(dev, previous, results);
		2476	if (!dirty)
4104	Serge	2477	return;
		2478
4560	Serge	2479	_ilk_disable_lp_wm(dev_priv, dirty);
4104	Serge	2480
4560	Serge	2481	if (dirty & WM_DIRTY_PIPE(PIPE_A))
4104	Serge	2482	I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
4560	Serge	2483	if (dirty & WM_DIRTY_PIPE(PIPE_B))
4104	Serge	2484	I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
4560	Serge	2485	if (dirty & WM_DIRTY_PIPE(PIPE_C))
4104	Serge	2486	I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
		2487
4560	Serge	2488	if (dirty & WM_DIRTY_LINETIME(PIPE_A))
4104	Serge	2489	I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
4560	Serge	2490	if (dirty & WM_DIRTY_LINETIME(PIPE_B))
4104	Serge	2491	I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
4560	Serge	2492	if (dirty & WM_DIRTY_LINETIME(PIPE_C))
4104	Serge	2493	I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
		2494
4560	Serge	2495	if (dirty & WM_DIRTY_DDB) {
		2496	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2497	val = I915_READ(WM_MISC);
4560	Serge	2498	if (results->partitioning == INTEL_DDB_PART_1_2)
4104	Serge	2499	val &= ~WM_MISC_DATA_PARTITION_5_6;
		2500	else
		2501	val \|= WM_MISC_DATA_PARTITION_5_6;
		2502	I915_WRITE(WM_MISC, val);
4560	Serge	2503	} else {
		2504	val = I915_READ(DISP_ARB_CTL2);
		2505	if (results->partitioning == INTEL_DDB_PART_1_2)
		2506	val &= ~DISP_DATA_PARTITION_5_6;
		2507	else
		2508	val \|= DISP_DATA_PARTITION_5_6;
		2509	I915_WRITE(DISP_ARB_CTL2, val);
		2510	}
4104	Serge	2511	}
		2512
4560	Serge	2513	if (dirty & WM_DIRTY_FBC) {
4104	Serge	2514	val = I915_READ(DISP_ARB_CTL);
		2515	if (results->enable_fbc_wm)
		2516	val &= ~DISP_FBC_WM_DIS;
		2517	else
		2518	val \|= DISP_FBC_WM_DIS;
		2519	I915_WRITE(DISP_ARB_CTL, val);
		2520	}
		2521
4560	Serge	2522	if (dirty & WM_DIRTY_LP(1) &&
		2523	previous->wm_lp_spr[0] != results->wm_lp_spr[0])
4104	Serge	2524	I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
4560	Serge	2525
		2526	if (INTEL_INFO(dev)->gen >= 7) {
		2527	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
4104	Serge	2528	I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
4560	Serge	2529	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
4104	Serge	2530	I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
4560	Serge	2531	}
4104	Serge	2532
4560	Serge	2533	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
4104	Serge	2534	I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
4560	Serge	2535	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
4104	Serge	2536	I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
4560	Serge	2537	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
4104	Serge	2538	I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
4560	Serge	2539
		2540	dev_priv->wm.hw = *results;
3031	serge	2541	}
		2542
4560	Serge	2543	static bool ilk_disable_lp_wm(struct drm_device *dev)
4104	Serge	2544	{
		2545	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2546
		2547	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
		2548	}
		2549
		2550	static void ilk_update_wm(struct drm_crtc *crtc)
		2551	{
		2552	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2553	struct drm_device *dev = crtc->dev;
		2554	struct drm_i915_private *dev_priv = dev->dev_private;
		2555	struct ilk_wm_maximums max;
		2556	struct ilk_pipe_wm_parameters params = {};
		2557	struct ilk_wm_values results = {};
4104	Serge	2558	enum intel_ddb_partitioning partitioning;
4560	Serge	2559	struct intel_pipe_wm pipe_wm = {};
		2560	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
		2561	struct intel_wm_config config = {};
4104	Serge	2562
4560	Serge	2563	ilk_compute_wm_parameters(crtc, ¶ms, &config);
4104	Serge	2564
4560	Serge	2565	intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm);
		2566
		2567	if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
		2568	return;
		2569
		2570	intel_crtc->wm.active = pipe_wm;
		2571
		2572	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
		2573	ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
		2574
		2575	/* 5/6 split only in single pipe config on IVB+ */
		2576	if (INTEL_INFO(dev)->gen >= 7 &&
		2577	config.num_pipes_active == 1 && config.sprites_enabled) {
		2578	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
		2579	ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
		2580
		2581	best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
4104	Serge	2582	} else {
4560	Serge	2583	best_lp_wm = &lp_wm_1_2;
4104	Serge	2584	}
		2585
4560	Serge	2586	partitioning = (best_lp_wm == &lp_wm_1_2) ?
4104	Serge	2587	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
		2588
4560	Serge	2589	ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
		2590
		2591	ilk_write_wm_values(dev_priv, &results);
4104	Serge	2592	}
		2593
4560	Serge	2594	static void ilk_update_sprite_wm(struct drm_plane *plane,
4104	Serge	2595	struct drm_crtc *crtc,
		2596	uint32_t sprite_width, int pixel_size,
		2597	bool enabled, bool scaled)
		2598	{
4560	Serge	2599	struct drm_device *dev = plane->dev;
4104	Serge	2600	struct intel_plane *intel_plane = to_intel_plane(plane);
		2601
		2602	intel_plane->wm.enabled = enabled;
		2603	intel_plane->wm.scaled = scaled;
		2604	intel_plane->wm.horiz_pixels = sprite_width;
		2605	intel_plane->wm.bytes_per_pixel = pixel_size;
		2606
4560	Serge	2607	/*
		2608	* IVB workaround: must disable low power watermarks for at least
		2609	* one frame before enabling scaling. LP watermarks can be re-enabled
		2610	* when scaling is disabled.
		2611	*
		2612	* WaCxSRDisabledForSpriteScaling:ivb
		2613	*/
		2614	if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
		2615	intel_wait_for_vblank(dev, intel_plane->pipe);
		2616
		2617	ilk_update_wm(crtc);
4104	Serge	2618	}
		2619
4560	Serge	2620	static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3031	serge	2621	{
4560	Serge	2622	struct drm_device *dev = crtc->dev;
		2623	struct drm_i915_private *dev_priv = dev->dev_private;
		2624	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		2625	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2626	struct intel_pipe_wm *active = &intel_crtc->wm.active;
		2627	enum pipe pipe = intel_crtc->pipe;
		2628	static const unsigned int wm0_pipe_reg[] = {
		2629	[PIPE_A] = WM0_PIPEA_ILK,
		2630	[PIPE_B] = WM0_PIPEB_ILK,
		2631	[PIPE_C] = WM0_PIPEC_IVB,
		2632	};
3031	serge	2633
4560	Serge	2634	hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
		2635	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2636	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3031	serge	2637
4560	Serge	2638	if (intel_crtc_active(crtc)) {
		2639	u32 tmp = hw->wm_pipe[pipe];
3031	serge	2640
4560	Serge	2641	/*
		2642	* For active pipes LP0 watermark is marked as
		2643	* enabled, and LP1+ watermaks as disabled since
		2644	* we can't really reverse compute them in case
		2645	* multiple pipes are active.
		2646	*/
		2647	active->wm[0].enable = true;
		2648	active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
		2649	active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
		2650	active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
		2651	active->linetime = hw->wm_linetime[pipe];
		2652	} else {
		2653	int level, max_level = ilk_wm_max_level(dev);
3031	serge	2654
4560	Serge	2655	/*
		2656	* For inactive pipes, all watermark levels
		2657	* should be marked as enabled but zeroed,
		2658	* which is what we'd compute them to.
		2659	*/
		2660	for (level = 0; level <= max_level; level++)
		2661	active->wm[level].enable = true;
3031	serge	2662	}
		2663	}
		2664
4560	Serge	2665	void ilk_wm_get_hw_state(struct drm_device *dev)
3031	serge	2666	{
		2667	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2668	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		2669	struct drm_crtc *crtc;
3031	serge	2670
4560	Serge	2671	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
		2672	ilk_pipe_wm_get_hw_state(crtc);
4104	Serge	2673
4560	Serge	2674	hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
		2675	hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
		2676	hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3031	serge	2677
4560	Serge	2678	hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
		2679	hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		2680	hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
3031	serge	2681
4560	Serge	2682	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2683	hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
		2684	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
		2685	else if (IS_IVYBRIDGE(dev))
		2686	hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
		2687	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3031	serge	2688
4560	Serge	2689	hw->enable_fbc_wm =
		2690	!(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3031	serge	2691	}
		2692
		2693	/**
		2694	* intel_update_watermarks - update FIFO watermark values based on current modes
		2695	*
		2696	* Calculate watermark values for the various WM regs based on current mode
		2697	* and plane configuration.
		2698	*
		2699	* There are several cases to deal with here:
		2700	* - normal (i.e. non-self-refresh)
		2701	* - self-refresh (SR) mode
		2702	* - lines are large relative to FIFO size (buffer can hold up to 2)
		2703	* - lines are small relative to FIFO size (buffer can hold more than 2
		2704	* lines), so need to account for TLB latency
		2705	*
		2706	* The normal calculation is:
		2707	* watermark = dotclock * bytes per pixel * latency
		2708	* where latency is platform & configuration dependent (we assume pessimal
		2709	* values here).
		2710	*
		2711	* The SR calculation is:
		2712	* watermark = (trunc(latency/line time)+1) * surface width *
		2713	* bytes per pixel
		2714	* where
		2715	* line time = htotal / dotclock
		2716	* surface width = hdisplay for normal plane and 64 for cursor
		2717	* and latency is assumed to be high, as above.
		2718	*
		2719	* The final value programmed to the register should always be rounded up,
		2720	* and include an extra 2 entries to account for clock crossings.
		2721	*
		2722	* We don't use the sprite, so we can ignore that. And on Crestline we have
		2723	* to set the non-SR watermarks to 8.
		2724	*/
4560	Serge	2725	void intel_update_watermarks(struct drm_crtc *crtc)
3031	serge	2726	{
4560	Serge	2727	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3031	serge	2728
		2729	if (dev_priv->display.update_wm)
4560	Serge	2730	dev_priv->display.update_wm(crtc);
3031	serge	2731	}
		2732
4104	Serge	2733	void intel_update_sprite_watermarks(struct drm_plane *plane,
		2734	struct drm_crtc *crtc,
		2735	uint32_t sprite_width, int pixel_size,
		2736	bool enabled, bool scaled)
3031	serge	2737	{
4104	Serge	2738	struct drm_i915_private *dev_priv = plane->dev->dev_private;
3031	serge	2739
		2740	if (dev_priv->display.update_sprite_wm)
4104	Serge	2741	dev_priv->display.update_sprite_wm(plane, crtc, sprite_width,
		2742	pixel_size, enabled, scaled);
3031	serge	2743	}
		2744
		2745	static struct drm_i915_gem_object *
		2746	intel_alloc_context_page(struct drm_device *dev)
		2747	{
		2748	struct drm_i915_gem_object *ctx;
		2749	int ret;
		2750
		2751	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2752
		2753	ctx = i915_gem_alloc_object(dev, 4096);
		2754	if (!ctx) {
		2755	DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
		2756	return NULL;
		2757	}
		2758
4104	Serge	2759	ret = i915_gem_obj_ggtt_pin(ctx, 4096, true, false);
3031	serge	2760	if (ret) {
		2761	DRM_ERROR("failed to pin power context: %d\n", ret);
		2762	goto err_unref;
		2763	}
		2764
		2765	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
		2766	if (ret) {
		2767	DRM_ERROR("failed to set-domain on power context: %d\n", ret);
		2768	goto err_unpin;
		2769	}
		2770
		2771	return ctx;
		2772
		2773	err_unpin:
		2774	i915_gem_object_unpin(ctx);
		2775	err_unref:
		2776	drm_gem_object_unreference(&ctx->base);
		2777	return NULL;
		2778	}
		2779
		2780	/**
		2781	* Lock protecting IPS related data structures
		2782	*/
		2783	DEFINE_SPINLOCK(mchdev_lock);
		2784
		2785	/* Global for IPS driver to get at the current i915 device. Protected by
		2786	* mchdev_lock. */
		2787	static struct drm_i915_private *i915_mch_dev;
		2788
		2789	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		2790	{
		2791	struct drm_i915_private *dev_priv = dev->dev_private;
		2792	u16 rgvswctl;
		2793
		2794	assert_spin_locked(&mchdev_lock);
		2795
		2796	rgvswctl = I915_READ16(MEMSWCTL);
		2797	if (rgvswctl & MEMCTL_CMD_STS) {
		2798	DRM_DEBUG("gpu busy, RCS change rejected\n");
		2799	return false; /* still busy with another command */
		2800	}
		2801
		2802	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		2803	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		2804	I915_WRITE16(MEMSWCTL, rgvswctl);
		2805	POSTING_READ16(MEMSWCTL);
		2806
		2807	rgvswctl \|= MEMCTL_CMD_STS;
		2808	I915_WRITE16(MEMSWCTL, rgvswctl);
		2809
		2810	return true;
		2811	}
		2812
		2813	static void ironlake_enable_drps(struct drm_device *dev)
		2814	{
		2815	struct drm_i915_private *dev_priv = dev->dev_private;
		2816	u32 rgvmodectl = I915_READ(MEMMODECTL);
		2817	u8 fmax, fmin, fstart, vstart;
		2818
		2819	spin_lock_irq(&mchdev_lock);
		2820
		2821	/* Enable temp reporting */
		2822	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		2823	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		2824
		2825	/* 100ms RC evaluation intervals */
		2826	I915_WRITE(RCUPEI, 100000);
		2827	I915_WRITE(RCDNEI, 100000);
		2828
		2829	/* Set max/min thresholds to 90ms and 80ms respectively */
		2830	I915_WRITE(RCBMAXAVG, 90000);
		2831	I915_WRITE(RCBMINAVG, 80000);
		2832
		2833	I915_WRITE(MEMIHYST, 1);
		2834
		2835	/* Set up min, max, and cur for interrupt handling */
		2836	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		2837	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		2838	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		2839	MEMMODE_FSTART_SHIFT;
		2840
		2841	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
		2842	PXVFREQ_PX_SHIFT;
		2843
		2844	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		2845	dev_priv->ips.fstart = fstart;
		2846
		2847	dev_priv->ips.max_delay = fstart;
		2848	dev_priv->ips.min_delay = fmin;
		2849	dev_priv->ips.cur_delay = fstart;
		2850
		2851	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		2852	fmax, fmin, fstart);
		2853
		2854	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		2855
		2856	/*
		2857	* Interrupts will be enabled in ironlake_irq_postinstall
		2858	*/
		2859
		2860	I915_WRITE(VIDSTART, vstart);
		2861	POSTING_READ(VIDSTART);
		2862
		2863	rgvmodectl \|= MEMMODE_SWMODE_EN;
		2864	I915_WRITE(MEMMODECTL, rgvmodectl);
		2865
		2866	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		2867	DRM_ERROR("stuck trying to change perf mode\n");
		2868	mdelay(1);
		2869
		2870	ironlake_set_drps(dev, fstart);
		2871
		2872	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
		2873	I915_READ(0x112e0);
		2874	dev_priv->ips.last_time1 = jiffies_to_msecs(GetTimerTicks());
		2875	dev_priv->ips.last_count2 = I915_READ(0x112f4);
3482	Serge	2876	getrawmonotonic(&dev_priv->ips.last_time2);
3031	serge	2877
		2878	spin_unlock_irq(&mchdev_lock);
		2879	}
		2880
		2881	static void ironlake_disable_drps(struct drm_device *dev)
		2882	{
		2883	struct drm_i915_private *dev_priv = dev->dev_private;
		2884	u16 rgvswctl;
		2885
		2886	spin_lock_irq(&mchdev_lock);
		2887
		2888	rgvswctl = I915_READ16(MEMSWCTL);
		2889
		2890	/* Ack interrupts, disable EFC interrupt */
		2891	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		2892	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		2893	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		2894	I915_WRITE(DEIIR, DE_PCU_EVENT);
		2895	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		2896
		2897	/* Go back to the starting frequency */
		2898	ironlake_set_drps(dev, dev_priv->ips.fstart);
		2899	mdelay(1);
		2900	rgvswctl \|= MEMCTL_CMD_STS;
		2901	I915_WRITE(MEMSWCTL, rgvswctl);
		2902	mdelay(1);
		2903
		2904	spin_unlock_irq(&mchdev_lock);
		2905	}
		2906
		2907	/* There's a funny hw issue where the hw returns all 0 when reading from
		2908	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		2909	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		2910	* all limits and the gpu stuck at whatever frequency it is at atm).
		2911	*/
4560	Serge	2912	static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3031	serge	2913	{
		2914	u32 limits;
		2915
		2916	/* Only set the down limit when we've reached the lowest level to avoid
		2917	* getting more interrupts, otherwise leave this clear. This prevents a
		2918	* race in the hw when coming out of rc6: There's a tiny window where
		2919	* the hw runs at the minimal clock before selecting the desired
		2920	* frequency, if the down threshold expires in that window we will not
		2921	* receive a down interrupt. */
4560	Serge	2922	limits = dev_priv->rps.max_delay << 24;
		2923	if (val <= dev_priv->rps.min_delay)
3031	serge	2924	limits \|= dev_priv->rps.min_delay << 16;
		2925
		2926	return limits;
		2927	}
		2928
4560	Serge	2929	static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
		2930	{
		2931	int new_power;
		2932
		2933	new_power = dev_priv->rps.power;
		2934	switch (dev_priv->rps.power) {
		2935	case LOW_POWER:
		2936	if (val > dev_priv->rps.rpe_delay + 1 && val > dev_priv->rps.cur_delay)
		2937	new_power = BETWEEN;
		2938	break;
		2939
		2940	case BETWEEN:
		2941	if (val <= dev_priv->rps.rpe_delay && val < dev_priv->rps.cur_delay)
		2942	new_power = LOW_POWER;
		2943	else if (val >= dev_priv->rps.rp0_delay && val > dev_priv->rps.cur_delay)
		2944	new_power = HIGH_POWER;
		2945	break;
		2946
		2947	case HIGH_POWER:
		2948	if (val < (dev_priv->rps.rp1_delay + dev_priv->rps.rp0_delay) >> 1 && val < dev_priv->rps.cur_delay)
		2949	new_power = BETWEEN;
		2950	break;
		2951	}
		2952	/* Max/min bins are special */
		2953	if (val == dev_priv->rps.min_delay)
		2954	new_power = LOW_POWER;
		2955	if (val == dev_priv->rps.max_delay)
		2956	new_power = HIGH_POWER;
		2957	if (new_power == dev_priv->rps.power)
		2958	return;
		2959
		2960	/* Note the units here are not exactly 1us, but 1280ns. */
		2961	switch (new_power) {
		2962	case LOW_POWER:
		2963	/* Upclock if more than 95% busy over 16ms */
		2964	I915_WRITE(GEN6_RP_UP_EI, 12500);
		2965	I915_WRITE(GEN6_RP_UP_THRESHOLD, 11800);
		2966
		2967	/* Downclock if less than 85% busy over 32ms */
		2968	I915_WRITE(GEN6_RP_DOWN_EI, 25000);
		2969	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 21250);
		2970
		2971	I915_WRITE(GEN6_RP_CONTROL,
		2972	GEN6_RP_MEDIA_TURBO \|
		2973	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		2974	GEN6_RP_MEDIA_IS_GFX \|
		2975	GEN6_RP_ENABLE \|
		2976	GEN6_RP_UP_BUSY_AVG \|
		2977	GEN6_RP_DOWN_IDLE_AVG);
		2978	break;
		2979
		2980	case BETWEEN:
		2981	/* Upclock if more than 90% busy over 13ms */
		2982	I915_WRITE(GEN6_RP_UP_EI, 10250);
		2983	I915_WRITE(GEN6_RP_UP_THRESHOLD, 9225);
		2984
		2985	/* Downclock if less than 75% busy over 32ms */
		2986	I915_WRITE(GEN6_RP_DOWN_EI, 25000);
		2987	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 18750);
		2988
		2989	I915_WRITE(GEN6_RP_CONTROL,
		2990	GEN6_RP_MEDIA_TURBO \|
		2991	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		2992	GEN6_RP_MEDIA_IS_GFX \|
		2993	GEN6_RP_ENABLE \|
		2994	GEN6_RP_UP_BUSY_AVG \|
		2995	GEN6_RP_DOWN_IDLE_AVG);
		2996	break;
		2997
		2998	case HIGH_POWER:
		2999	/* Upclock if more than 85% busy over 10ms */
		3000	I915_WRITE(GEN6_RP_UP_EI, 8000);
		3001	I915_WRITE(GEN6_RP_UP_THRESHOLD, 6800);
		3002
		3003	/* Downclock if less than 60% busy over 32ms */
		3004	I915_WRITE(GEN6_RP_DOWN_EI, 25000);
		3005	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 15000);
		3006
		3007	I915_WRITE(GEN6_RP_CONTROL,
		3008	GEN6_RP_MEDIA_TURBO \|
		3009	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3010	GEN6_RP_MEDIA_IS_GFX \|
		3011	GEN6_RP_ENABLE \|
		3012	GEN6_RP_UP_BUSY_AVG \|
		3013	GEN6_RP_DOWN_IDLE_AVG);
		3014	break;
		3015	}
		3016
		3017	dev_priv->rps.power = new_power;
		3018	dev_priv->rps.last_adj = 0;
		3019	}
		3020
3031	serge	3021	void gen6_set_rps(struct drm_device *dev, u8 val)
		3022	{
		3023	struct drm_i915_private *dev_priv = dev->dev_private;
		3024
3243	Serge	3025	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3026	WARN_ON(val > dev_priv->rps.max_delay);
		3027	WARN_ON(val < dev_priv->rps.min_delay);
		3028
		3029	if (val == dev_priv->rps.cur_delay)
		3030	return;
		3031
4560	Serge	3032	gen6_set_rps_thresholds(dev_priv, val);
		3033
3746	Serge	3034	if (IS_HASWELL(dev))
		3035	I915_WRITE(GEN6_RPNSWREQ,
		3036	HSW_FREQUENCY(val));
		3037	else
3031	serge	3038	I915_WRITE(GEN6_RPNSWREQ,
		3039	GEN6_FREQUENCY(val) \|
		3040	GEN6_OFFSET(0) \|
		3041	GEN6_AGGRESSIVE_TURBO);
		3042
		3043	/* Make sure we continue to get interrupts
		3044	* until we hit the minimum or maximum frequencies.
		3045	*/
4560	Serge	3046	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
		3047	gen6_rps_limits(dev_priv, val));
3031	serge	3048
		3049	POSTING_READ(GEN6_RPNSWREQ);
		3050
		3051	dev_priv->rps.cur_delay = val;
		3052
		3053	trace_intel_gpu_freq_change(val * 50);
		3054	}
		3055
4560	Serge	3056	void gen6_rps_idle(struct drm_i915_private *dev_priv)
3031	serge	3057	{
4560	Serge	3058	struct drm_device *dev = dev_priv->dev;
4104	Serge	3059
4560	Serge	3060	mutex_lock(&dev_priv->rps.hw_lock);
		3061	if (dev_priv->rps.enabled) {
		3062	if (IS_VALLEYVIEW(dev))
		3063	valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
		3064	else
		3065	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
		3066	dev_priv->rps.last_adj = 0;
		3067	}
		3068	mutex_unlock(&dev_priv->rps.hw_lock);
		3069	}
4104	Serge	3070
4560	Serge	3071	void gen6_rps_boost(struct drm_i915_private *dev_priv)
		3072	{
		3073	struct drm_device *dev = dev_priv->dev;
4104	Serge	3074
4560	Serge	3075	mutex_lock(&dev_priv->rps.hw_lock);
		3076	if (dev_priv->rps.enabled) {
		3077	if (IS_VALLEYVIEW(dev))
		3078	valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_delay);
		3079	else
		3080	gen6_set_rps(dev_priv->dev, dev_priv->rps.max_delay);
		3081	dev_priv->rps.last_adj = 0;
		3082	}
		3083	mutex_unlock(&dev_priv->rps.hw_lock);
4104	Serge	3084	}
		3085
		3086	void valleyview_set_rps(struct drm_device *dev, u8 val)
		3087	{
3031	serge	3088	struct drm_i915_private *dev_priv = dev->dev_private;
		3089
4104	Serge	3090	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		3091	WARN_ON(val > dev_priv->rps.max_delay);
		3092	WARN_ON(val < dev_priv->rps.min_delay);
		3093
		3094	DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
4560	Serge	3095	vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay),
4104	Serge	3096	dev_priv->rps.cur_delay,
4560	Serge	3097	vlv_gpu_freq(dev_priv, val), val);
4104	Serge	3098
		3099	if (val == dev_priv->rps.cur_delay)
		3100	return;
		3101
		3102	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
		3103
		3104	dev_priv->rps.cur_delay = val;
		3105
4560	Serge	3106	trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val));
4104	Serge	3107	}
		3108
		3109	static void gen6_disable_rps_interrupts(struct drm_device *dev)
		3110	{
		3111	struct drm_i915_private *dev_priv = dev->dev_private;
		3112
3031	serge	3113	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
4104	Serge	3114	I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & ~GEN6_PM_RPS_EVENTS);
3031	serge	3115	/* Complete PM interrupt masking here doesn't race with the rps work
		3116	* item again unmasking PM interrupts because that is using a different
		3117	* register (PMIMR) to mask PM interrupts. The only risk is in leaving
		3118	* stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
		3119
4104	Serge	3120	spin_lock_irq(&dev_priv->irq_lock);
3031	serge	3121	dev_priv->rps.pm_iir = 0;
4104	Serge	3122	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	3123
4104	Serge	3124	I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
3031	serge	3125	}
		3126
4104	Serge	3127	static void gen6_disable_rps(struct drm_device *dev)
		3128	{
		3129	struct drm_i915_private *dev_priv = dev->dev_private;
		3130
		3131	I915_WRITE(GEN6_RC_CONTROL, 0);
		3132	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		3133
		3134	gen6_disable_rps_interrupts(dev);
		3135	}
		3136
		3137	static void valleyview_disable_rps(struct drm_device *dev)
		3138	{
		3139	struct drm_i915_private *dev_priv = dev->dev_private;
		3140
		3141	I915_WRITE(GEN6_RC_CONTROL, 0);
		3142
		3143	gen6_disable_rps_interrupts(dev);
		3144
		3145	if (dev_priv->vlv_pctx) {
		3146	drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
		3147	dev_priv->vlv_pctx = NULL;
		3148	}
		3149	}
		3150
4560	Serge	3151	static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
		3152	{
		3153	if (IS_GEN6(dev))
		3154	DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
		3155
		3156	if (IS_HASWELL(dev))
		3157	DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
		3158
		3159	DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
		3160	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		3161	(mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		3162	(mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
		3163	}
		3164
3031	serge	3165	int intel_enable_rc6(const struct drm_device *dev)
		3166	{
4104	Serge	3167	/* No RC6 before Ironlake */
		3168	if (INTEL_INFO(dev)->gen < 5)
		3169	return 0;
		3170
3031	serge	3171	/* Respect the kernel parameter if it is set */
		3172	if (i915_enable_rc6 >= 0)
		3173	return i915_enable_rc6;
		3174
3120	serge	3175	/* Disable RC6 on Ironlake */
		3176	if (INTEL_INFO(dev)->gen == 5)
		3177	return 0;
3031	serge	3178
4560	Serge	3179	if (IS_HASWELL(dev))
3031	serge	3180	return INTEL_RC6_ENABLE;
		3181
		3182	/* snb/ivb have more than one rc6 state. */
4560	Serge	3183	if (INTEL_INFO(dev)->gen == 6)
3031	serge	3184	return INTEL_RC6_ENABLE;
		3185
		3186	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
		3187	}
		3188
4104	Serge	3189	static void gen6_enable_rps_interrupts(struct drm_device *dev)
		3190	{
		3191	struct drm_i915_private *dev_priv = dev->dev_private;
		3192	u32 enabled_intrs;
		3193
		3194	spin_lock_irq(&dev_priv->irq_lock);
		3195	WARN_ON(dev_priv->rps.pm_iir);
		3196	snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
		3197	I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
		3198	spin_unlock_irq(&dev_priv->irq_lock);
		3199
		3200	/* only unmask PM interrupts we need. Mask all others. */
		3201	enabled_intrs = GEN6_PM_RPS_EVENTS;
		3202
		3203	/* IVB and SNB hard hangs on looping batchbuffer
		3204	* if GEN6_PM_UP_EI_EXPIRED is masked.
		3205	*/
		3206	if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
		3207	enabled_intrs \|= GEN6_PM_RP_UP_EI_EXPIRED;
		3208
		3209	I915_WRITE(GEN6_PMINTRMSK, ~enabled_intrs);
		3210	}
		3211
4560	Serge	3212	static void gen8_enable_rps(struct drm_device *dev)
		3213	{
		3214	struct drm_i915_private *dev_priv = dev->dev_private;
		3215	struct intel_ring_buffer *ring;
		3216	uint32_t rc6_mask = 0, rp_state_cap;
		3217	int unused;
		3218
		3219	/* 1a: Software RC state - RC0 */
		3220	I915_WRITE(GEN6_RC_STATE, 0);
		3221
		3222	/* 1c & 1d: Get forcewake during program sequence. Although the driver
		3223	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
		3224	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
		3225
		3226	/* 2a: Disable RC states. */
		3227	I915_WRITE(GEN6_RC_CONTROL, 0);
		3228
		3229	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		3230
		3231	/* 2b: Program RC6 thresholds.*/
		3232	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		3233	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		3234	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		3235	for_each_ring(ring, dev_priv, unused)
		3236	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		3237	I915_WRITE(GEN6_RC_SLEEP, 0);
		3238	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
		3239
		3240	/* 3: Enable RC6 */
		3241	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		3242	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
		3243	DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
		3244	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		3245	GEN6_RC_CTL_EI_MODE(1) \|
		3246	rc6_mask);
		3247
		3248	/* 4 Program defaults and thresholds for RPS*/
		3249	I915_WRITE(GEN6_RPNSWREQ, HSW_FREQUENCY(10)); /* Request 500 MHz */
		3250	I915_WRITE(GEN6_RC_VIDEO_FREQ, HSW_FREQUENCY(12)); /* Request 600 MHz */
		3251	/* NB: Docs say 1s, and 1000000 - which aren't equivalent */
		3252	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
		3253
		3254	/* Docs recommend 900MHz, and 300 MHz respectively */
		3255	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
		3256	dev_priv->rps.max_delay << 24 \|
		3257	dev_priv->rps.min_delay << 16);
		3258
		3259	I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
		3260	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
		3261	I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
		3262	I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
		3263
		3264	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		3265
		3266	/* 5: Enable RPS */
		3267	I915_WRITE(GEN6_RP_CONTROL,
		3268	GEN6_RP_MEDIA_TURBO \|
		3269	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3270	GEN6_RP_MEDIA_IS_GFX \|
		3271	GEN6_RP_ENABLE \|
		3272	GEN6_RP_UP_BUSY_AVG \|
		3273	GEN6_RP_DOWN_IDLE_AVG);
		3274
		3275	/* 6: Ring frequency + overclocking (our driver does this later */
		3276
		3277	gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8);
		3278
		3279	gen6_enable_rps_interrupts(dev);
		3280
		3281	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
		3282	}
		3283
3031	serge	3284	static void gen6_enable_rps(struct drm_device *dev)
		3285	{
		3286	struct drm_i915_private *dev_priv = dev->dev_private;
		3287	struct intel_ring_buffer *ring;
		3288	u32 rp_state_cap;
		3289	u32 gt_perf_status;
3243	Serge	3290	u32 rc6vids, pcu_mbox, rc6_mask = 0;
3031	serge	3291	u32 gtfifodbg;
		3292	int rc6_mode;
3243	Serge	3293	int i, ret;
3031	serge	3294
3243	Serge	3295	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3296
		3297	/* Here begins a magic sequence of register writes to enable
		3298	* auto-downclocking.
		3299	*
		3300	* Perhaps there might be some value in exposing these to
		3301	* userspace...
		3302	*/
		3303	I915_WRITE(GEN6_RC_STATE, 0);
		3304
		3305	/* Clear the DBG now so we don't confuse earlier errors */
		3306	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		3307	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		3308	I915_WRITE(GTFIFODBG, gtfifodbg);
		3309	}
		3310
4560	Serge	3311	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
3031	serge	3312
		3313	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		3314	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
		3315
3746	Serge	3316	/* In units of 50MHz */
		3317	dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff;
4560	Serge	3318	dev_priv->rps.min_delay = (rp_state_cap >> 16) & 0xff;
		3319	dev_priv->rps.rp1_delay = (rp_state_cap >> 8) & 0xff;
		3320	dev_priv->rps.rp0_delay = (rp_state_cap >> 0) & 0xff;
		3321	dev_priv->rps.rpe_delay = dev_priv->rps.rp1_delay;
3031	serge	3322	dev_priv->rps.cur_delay = 0;
		3323
		3324	/* disable the counters and set deterministic thresholds */
		3325	I915_WRITE(GEN6_RC_CONTROL, 0);
		3326
		3327	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		3328	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		3329	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		3330	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		3331	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		3332
		3333	for_each_ring(ring, dev_priv, i)
		3334	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		3335
		3336	I915_WRITE(GEN6_RC_SLEEP, 0);
		3337	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4560	Serge	3338	if (IS_IVYBRIDGE(dev))
4104	Serge	3339	I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
		3340	else
3031	serge	3341	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	3342	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	3343	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		3344
		3345	/* Check if we are enabling RC6 */
		3346	rc6_mode = intel_enable_rc6(dev_priv->dev);
		3347	if (rc6_mode & INTEL_RC6_ENABLE)
		3348	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		3349
		3350	/* We don't use those on Haswell */
		3351	if (!IS_HASWELL(dev)) {
		3352	if (rc6_mode & INTEL_RC6p_ENABLE)
		3353	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		3354
		3355	if (rc6_mode & INTEL_RC6pp_ENABLE)
		3356	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		3357	}
		3358
4560	Serge	3359	intel_print_rc6_info(dev, rc6_mask);
3031	serge	3360
		3361	I915_WRITE(GEN6_RC_CONTROL,
		3362	rc6_mask \|
		3363	GEN6_RC_CTL_EI_MODE(1) \|
		3364	GEN6_RC_CTL_HW_ENABLE);
		3365
4560	Serge	3366	/* Power down if completely idle for over 50ms */
		3367	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
3031	serge	3368	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		3369
3243	Serge	3370	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
		3371	if (!ret) {
		3372	pcu_mbox = 0;
		3373	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
3746	Serge	3374	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		3375	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
		3376	(dev_priv->rps.max_delay & 0xff) * 50,
		3377	(pcu_mbox & 0xff) * 50);
		3378	dev_priv->rps.hw_max = pcu_mbox & 0xff;
3031	serge	3379	}
3243	Serge	3380	} else {
		3381	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		3382	}
3031	serge	3383
4560	Serge	3384	dev_priv->rps.power = HIGH_POWER; /* force a reset */
		3385	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
3031	serge	3386
4104	Serge	3387	gen6_enable_rps_interrupts(dev);
3031	serge	3388
3243	Serge	3389	rc6vids = 0;
		3390	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		3391	if (IS_GEN6(dev) && ret) {
		3392	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		3393	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		3394	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		3395	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		3396	rc6vids &= 0xffff00;
		3397	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		3398	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		3399	if (ret)
		3400	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		3401	}
		3402
4560	Serge	3403	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
3031	serge	3404	}
		3405
4104	Serge	3406	void gen6_update_ring_freq(struct drm_device *dev)
3031	serge	3407	{
		3408	struct drm_i915_private *dev_priv = dev->dev_private;
		3409	int min_freq = 15;
3746	Serge	3410	unsigned int gpu_freq;
		3411	unsigned int max_ia_freq, min_ring_freq;
3031	serge	3412	int scaling_factor = 180;
4560	Serge	3413	struct cpufreq_policy *policy;
3031	serge	3414
3243	Serge	3415	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3416
		3417	max_ia_freq = cpufreq_quick_get_max(0);
		3418	/*
		3419	* Default to measured freq if none found, PCU will ensure we don't go
		3420	* over
		3421	*/
		3422	max_ia_freq = tsc_khz;
		3423
		3424	/* Convert from kHz to MHz */
		3425	max_ia_freq /= 1000;
		3426
4560	Serge	3427	min_ring_freq = I915_READ(DCLK) & 0xf;
		3428	/* convert DDR frequency from units of 266.6MHz to bandwidth */
		3429	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3746	Serge	3430
3031	serge	3431	/*
		3432	* For each potential GPU frequency, load a ring frequency we'd like
		3433	* to use for memory access. We do this by specifying the IA frequency
		3434	* the PCU should use as a reference to determine the ring frequency.
		3435	*/
		3436	for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
		3437	gpu_freq--) {
		3438	int diff = dev_priv->rps.max_delay - gpu_freq;
3746	Serge	3439	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	3440
4560	Serge	3441	if (INTEL_INFO(dev)->gen >= 8) {
		3442	/* max(2 * GT, DDR). NB: GT is 50MHz units */
		3443	ring_freq = max(min_ring_freq, gpu_freq);
		3444	} else if (IS_HASWELL(dev)) {
		3445	ring_freq = mult_frac(gpu_freq, 5, 4);
3746	Serge	3446	ring_freq = max(min_ring_freq, ring_freq);
		3447	/* leave ia_freq as the default, chosen by cpufreq */
		3448	} else {
		3449	/* On older processors, there is no separate ring
		3450	* clock domain, so in order to boost the bandwidth
		3451	* of the ring, we need to upclock the CPU (ia_freq).
		3452	*
		3453	* For GPU frequencies less than 750MHz,
		3454	* just use the lowest ring freq.
3031	serge	3455	*/
		3456	if (gpu_freq < min_freq)
		3457	ia_freq = 800;
		3458	else
		3459	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		3460	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	3461	}
3031	serge	3462
3243	Serge	3463	sandybridge_pcode_write(dev_priv,
		3464	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	3465	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		3466	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		3467	gpu_freq);
3031	serge	3468	}
		3469	}
		3470
4104	Serge	3471	int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
		3472	{
		3473	u32 val, rp0;
		3474
		3475	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		3476
		3477	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
		3478	/* Clamp to max */
		3479	rp0 = min_t(u32, rp0, 0xea);
		3480
		3481	return rp0;
		3482	}
		3483
		3484	static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		3485	{
		3486	u32 val, rpe;
		3487
		3488	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
		3489	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
		3490	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
		3491	rpe \|= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
		3492
		3493	return rpe;
		3494	}
		3495
		3496	int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
		3497	{
		3498	return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
		3499	}
		3500
		3501	static void valleyview_setup_pctx(struct drm_device *dev)
		3502	{
		3503	struct drm_i915_private *dev_priv = dev->dev_private;
		3504	struct drm_i915_gem_object *pctx;
		3505	unsigned long pctx_paddr;
		3506	u32 pcbr;
		3507	int pctx_size = 24*1024;
		3508
		3509	pcbr = I915_READ(VLV_PCBR);
		3510	if (pcbr) {
		3511	/* BIOS set it up already, grab the pre-alloc'd space */
		3512	int pcbr_offset;
		3513
		3514	pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
		3515	pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
		3516	pcbr_offset,
		3517	I915_GTT_OFFSET_NONE,
		3518	pctx_size);
		3519	goto out;
		3520	}
		3521
		3522	/*
		3523	* From the Gunit register HAS:
		3524	* The Gfx driver is expected to program this register and ensure
		3525	* proper allocation within Gfx stolen memory. For example, this
		3526	* register should be programmed such than the PCBR range does not
		3527	* overlap with other ranges, such as the frame buffer, protected
		3528	* memory, or any other relevant ranges.
		3529	*/
		3530	pctx = i915_gem_object_create_stolen(dev, pctx_size);
		3531	if (!pctx) {
		3532	DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
		3533	return;
		3534	}
		3535
		3536	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
		3537	I915_WRITE(VLV_PCBR, pctx_paddr);
		3538
		3539	out:
		3540	dev_priv->vlv_pctx = pctx;
		3541	}
		3542
		3543	static void valleyview_enable_rps(struct drm_device *dev)
		3544	{
		3545	struct drm_i915_private *dev_priv = dev->dev_private;
		3546	struct intel_ring_buffer *ring;
4560	Serge	3547	u32 gtfifodbg, val, rc6_mode = 0;
4104	Serge	3548	int i;
		3549
		3550	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		3551
		3552	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4560	Serge	3553	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		3554	gtfifodbg);
4104	Serge	3555	I915_WRITE(GTFIFODBG, gtfifodbg);
		3556	}
		3557
		3558	valleyview_setup_pctx(dev);
		3559
4560	Serge	3560	/* If VLV, Forcewake all wells, else re-direct to regular path */
		3561	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
4104	Serge	3562
		3563	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		3564	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		3565	I915_WRITE(GEN6_RP_UP_EI, 66000);
		3566	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		3567
		3568	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		3569
		3570	I915_WRITE(GEN6_RP_CONTROL,
		3571	GEN6_RP_MEDIA_TURBO \|
		3572	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3573	GEN6_RP_MEDIA_IS_GFX \|
		3574	GEN6_RP_ENABLE \|
		3575	GEN6_RP_UP_BUSY_AVG \|
		3576	GEN6_RP_DOWN_IDLE_CONT);
		3577
		3578	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
		3579	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		3580	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		3581
		3582	for_each_ring(ring, dev_priv, i)
		3583	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		3584
4560	Serge	3585	I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4104	Serge	3586
		3587	/* allows RC6 residency counter to work */
4560	Serge	3588	I915_WRITE(VLV_COUNTER_CONTROL,
		3589	_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH \|
		3590	VLV_MEDIA_RC6_COUNT_EN \|
		3591	VLV_RENDER_RC6_COUNT_EN));
		3592	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		3593	rc6_mode = GEN7_RC_CTL_TO_MODE \| VLV_RC_CTL_CTX_RST_PARALLEL;
4104	Serge	3594
4560	Serge	3595	intel_print_rc6_info(dev, rc6_mode);
		3596
		3597	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		3598
4104	Serge	3599	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		3600
		3601	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
		3602	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		3603
		3604	dev_priv->rps.cur_delay = (val >> 8) & 0xff;
		3605	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
4560	Serge	3606	vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay),
4104	Serge	3607	dev_priv->rps.cur_delay);
		3608
		3609	dev_priv->rps.max_delay = valleyview_rps_max_freq(dev_priv);
		3610	dev_priv->rps.hw_max = dev_priv->rps.max_delay;
		3611	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
4560	Serge	3612	vlv_gpu_freq(dev_priv, dev_priv->rps.max_delay),
4104	Serge	3613	dev_priv->rps.max_delay);
		3614
		3615	dev_priv->rps.rpe_delay = valleyview_rps_rpe_freq(dev_priv);
		3616	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
4560	Serge	3617	vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay),
4104	Serge	3618	dev_priv->rps.rpe_delay);
		3619
		3620	dev_priv->rps.min_delay = valleyview_rps_min_freq(dev_priv);
		3621	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
4560	Serge	3622	vlv_gpu_freq(dev_priv, dev_priv->rps.min_delay),
4104	Serge	3623	dev_priv->rps.min_delay);
		3624
		3625	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
4560	Serge	3626	vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay),
4104	Serge	3627	dev_priv->rps.rpe_delay);
		3628
		3629	valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay);
		3630
		3631	gen6_enable_rps_interrupts(dev);
		3632
4560	Serge	3633	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	3634	}
		3635
3031	serge	3636	void ironlake_teardown_rc6(struct drm_device *dev)
		3637	{
		3638	struct drm_i915_private *dev_priv = dev->dev_private;
		3639
3243	Serge	3640	if (dev_priv->ips.renderctx) {
		3641	i915_gem_object_unpin(dev_priv->ips.renderctx);
		3642	drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
		3643	dev_priv->ips.renderctx = NULL;
3031	serge	3644	}
		3645
3243	Serge	3646	if (dev_priv->ips.pwrctx) {
		3647	i915_gem_object_unpin(dev_priv->ips.pwrctx);
		3648	drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
		3649	dev_priv->ips.pwrctx = NULL;
3031	serge	3650	}
		3651	}
		3652
		3653	static void ironlake_disable_rc6(struct drm_device *dev)
		3654	{
		3655	struct drm_i915_private *dev_priv = dev->dev_private;
		3656
		3657	if (I915_READ(PWRCTXA)) {
		3658	/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
		3659	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) \| RCX_SW_EXIT);
		3660	wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
		3661	50);
		3662
		3663	I915_WRITE(PWRCTXA, 0);
		3664	POSTING_READ(PWRCTXA);
		3665
		3666	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		3667	POSTING_READ(RSTDBYCTL);
		3668	}
		3669	}
		3670
		3671	static int ironlake_setup_rc6(struct drm_device *dev)
		3672	{
		3673	struct drm_i915_private *dev_priv = dev->dev_private;
		3674
3243	Serge	3675	if (dev_priv->ips.renderctx == NULL)
		3676	dev_priv->ips.renderctx = intel_alloc_context_page(dev);
		3677	if (!dev_priv->ips.renderctx)
3031	serge	3678	return -ENOMEM;
		3679
3243	Serge	3680	if (dev_priv->ips.pwrctx == NULL)
		3681	dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
		3682	if (!dev_priv->ips.pwrctx) {
3031	serge	3683	ironlake_teardown_rc6(dev);
		3684	return -ENOMEM;
		3685	}
		3686
		3687	return 0;
		3688	}
		3689
		3690	static void ironlake_enable_rc6(struct drm_device *dev)
		3691	{
		3692	struct drm_i915_private *dev_priv = dev->dev_private;
		3693	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
3243	Serge	3694	bool was_interruptible;
3031	serge	3695	int ret;
		3696
		3697	/* rc6 disabled by default due to repeated reports of hanging during
		3698	* boot and resume.
		3699	*/
		3700	if (!intel_enable_rc6(dev))
		3701	return;
		3702
		3703	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		3704
		3705	ret = ironlake_setup_rc6(dev);
		3706	if (ret)
		3707	return;
		3708
3243	Serge	3709	was_interruptible = dev_priv->mm.interruptible;
		3710	dev_priv->mm.interruptible = false;
		3711
3031	serge	3712	/*
		3713	* GPU can automatically power down the render unit if given a page
		3714	* to save state.
		3715	*/
		3716	ret = intel_ring_begin(ring, 6);
		3717	if (ret) {
		3718	ironlake_teardown_rc6(dev);
3243	Serge	3719	dev_priv->mm.interruptible = was_interruptible;
3031	serge	3720	return;
		3721	}
		3722
		3723	intel_ring_emit(ring, MI_SUSPEND_FLUSH \| MI_SUSPEND_FLUSH_EN);
		3724	intel_ring_emit(ring, MI_SET_CONTEXT);
4104	Serge	3725	intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) \|
3031	serge	3726	MI_MM_SPACE_GTT \|
		3727	MI_SAVE_EXT_STATE_EN \|
		3728	MI_RESTORE_EXT_STATE_EN \|
		3729	MI_RESTORE_INHIBIT);
		3730	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
		3731	intel_ring_emit(ring, MI_NOOP);
		3732	intel_ring_emit(ring, MI_FLUSH);
		3733	intel_ring_advance(ring);
		3734
		3735	/*
		3736	* Wait for the command parser to advance past MI_SET_CONTEXT. The HW
		3737	* does an implicit flush, combined with MI_FLUSH above, it should be
		3738	* safe to assume that renderctx is valid
		3739	*/
3243	Serge	3740	ret = intel_ring_idle(ring);
		3741	dev_priv->mm.interruptible = was_interruptible;
3031	serge	3742	if (ret) {
3746	Serge	3743	DRM_ERROR("failed to enable ironlake power savings\n");
3031	serge	3744	ironlake_teardown_rc6(dev);
		3745	return;
		3746	}
		3747
4104	Serge	3748	I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) \| PWRCTX_EN);
3031	serge	3749	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
4560	Serge	3750
		3751	intel_print_rc6_info(dev, INTEL_RC6_ENABLE);
3031	serge	3752	}
		3753
		3754	static unsigned long intel_pxfreq(u32 vidfreq)
		3755	{
		3756	unsigned long freq;
		3757	int div = (vidfreq & 0x3f0000) >> 16;
		3758	int post = (vidfreq & 0x3000) >> 12;
		3759	int pre = (vidfreq & 0x7);
		3760
		3761	if (!pre)
		3762	return 0;
		3763
		3764	freq = ((div * 133333) / ((1<
		3765
		3766	return freq;
		3767	}
		3768
		3769	static const struct cparams {
		3770	u16 i;
		3771	u16 t;
		3772	u16 m;
		3773	u16 c;
		3774	} cparams[] = {
		3775	{ 1, 1333, 301, 28664 },
		3776	{ 1, 1066, 294, 24460 },
		3777	{ 1, 800, 294, 25192 },
		3778	{ 0, 1333, 276, 27605 },
		3779	{ 0, 1066, 276, 27605 },
		3780	{ 0, 800, 231, 23784 },
		3781	};
		3782
		3783	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		3784	{
		3785	u64 total_count, diff, ret;
		3786	u32 count1, count2, count3, m = 0, c = 0;
		3787	unsigned long now = jiffies_to_msecs(GetTimerTicks()), diff1;
		3788	int i;
		3789
		3790	assert_spin_locked(&mchdev_lock);
		3791
		3792	diff1 = now - dev_priv->ips.last_time1;
		3793
		3794	/* Prevent division-by-zero if we are asking too fast.
		3795	* Also, we don't get interesting results if we are polling
		3796	* faster than once in 10ms, so just return the saved value
		3797	* in such cases.
		3798	*/
		3799	if (diff1 <= 10)
		3800	return dev_priv->ips.chipset_power;
		3801
		3802	count1 = I915_READ(DMIEC);
		3803	count2 = I915_READ(DDREC);
		3804	count3 = I915_READ(CSIEC);
		3805
		3806	total_count = count1 + count2 + count3;
		3807
		3808	/* FIXME: handle per-counter overflow */
		3809	if (total_count < dev_priv->ips.last_count1) {
		3810	diff = ~0UL - dev_priv->ips.last_count1;
		3811	diff += total_count;
		3812	} else {
		3813	diff = total_count - dev_priv->ips.last_count1;
		3814	}
		3815
		3816	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		3817	if (cparams[i].i == dev_priv->ips.c_m &&
		3818	cparams[i].t == dev_priv->ips.r_t) {
		3819	m = cparams[i].m;
		3820	c = cparams[i].c;
		3821	break;
		3822	}
		3823	}
		3824
		3825	diff = div_u64(diff, diff1);
		3826	ret = ((m * diff) + c);
		3827	ret = div_u64(ret, 10);
		3828
		3829	dev_priv->ips.last_count1 = total_count;
		3830	dev_priv->ips.last_time1 = now;
		3831
		3832	dev_priv->ips.chipset_power = ret;
		3833
		3834	return ret;
		3835	}
		3836
		3837	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		3838	{
		3839	unsigned long val;
		3840
		3841	if (dev_priv->info->gen != 5)
		3842	return 0;
		3843
		3844	spin_lock_irq(&mchdev_lock);
		3845
		3846	val = __i915_chipset_val(dev_priv);
		3847
		3848	spin_unlock_irq(&mchdev_lock);
		3849
		3850	return val;
		3851	}
		3852
		3853	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		3854	{
		3855	unsigned long m, x, b;
		3856	u32 tsfs;
		3857
		3858	tsfs = I915_READ(TSFS);
		3859
		3860	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		3861	x = I915_READ8(TR1);
		3862
		3863	b = tsfs & TSFS_INTR_MASK;
		3864
		3865	return ((m * x) / 127) - b;
		3866	}
		3867
		3868	static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		3869	{
		3870	static const struct v_table {
		3871	u16 vd; /* in .1 mil */
		3872	u16 vm; /* in .1 mil */
		3873	} v_table[] = {
		3874	{ 0, 0, },
		3875	{ 375, 0, },
		3876	{ 500, 0, },
		3877	{ 625, 0, },
		3878	{ 750, 0, },
		3879	{ 875, 0, },
		3880	{ 1000, 0, },
		3881	{ 1125, 0, },
		3882	{ 4125, 3000, },
		3883	{ 4125, 3000, },
		3884	{ 4125, 3000, },
		3885	{ 4125, 3000, },
		3886	{ 4125, 3000, },
		3887	{ 4125, 3000, },
		3888	{ 4125, 3000, },
		3889	{ 4125, 3000, },
		3890	{ 4125, 3000, },
		3891	{ 4125, 3000, },
		3892	{ 4125, 3000, },
		3893	{ 4125, 3000, },
		3894	{ 4125, 3000, },
		3895	{ 4125, 3000, },
		3896	{ 4125, 3000, },
		3897	{ 4125, 3000, },
		3898	{ 4125, 3000, },
		3899	{ 4125, 3000, },
		3900	{ 4125, 3000, },
		3901	{ 4125, 3000, },
		3902	{ 4125, 3000, },
		3903	{ 4125, 3000, },
		3904	{ 4125, 3000, },
		3905	{ 4125, 3000, },
		3906	{ 4250, 3125, },
		3907	{ 4375, 3250, },
		3908	{ 4500, 3375, },
		3909	{ 4625, 3500, },
		3910	{ 4750, 3625, },
		3911	{ 4875, 3750, },
		3912	{ 5000, 3875, },
		3913	{ 5125, 4000, },
		3914	{ 5250, 4125, },
		3915	{ 5375, 4250, },
		3916	{ 5500, 4375, },
		3917	{ 5625, 4500, },
		3918	{ 5750, 4625, },
		3919	{ 5875, 4750, },
		3920	{ 6000, 4875, },
		3921	{ 6125, 5000, },
		3922	{ 6250, 5125, },
		3923	{ 6375, 5250, },
		3924	{ 6500, 5375, },
		3925	{ 6625, 5500, },
		3926	{ 6750, 5625, },
		3927	{ 6875, 5750, },
		3928	{ 7000, 5875, },
		3929	{ 7125, 6000, },
		3930	{ 7250, 6125, },
		3931	{ 7375, 6250, },
		3932	{ 7500, 6375, },
		3933	{ 7625, 6500, },
		3934	{ 7750, 6625, },
		3935	{ 7875, 6750, },
		3936	{ 8000, 6875, },
		3937	{ 8125, 7000, },
		3938	{ 8250, 7125, },
		3939	{ 8375, 7250, },
		3940	{ 8500, 7375, },
		3941	{ 8625, 7500, },
		3942	{ 8750, 7625, },
		3943	{ 8875, 7750, },
		3944	{ 9000, 7875, },
		3945	{ 9125, 8000, },
		3946	{ 9250, 8125, },
		3947	{ 9375, 8250, },
		3948	{ 9500, 8375, },
		3949	{ 9625, 8500, },
		3950	{ 9750, 8625, },
		3951	{ 9875, 8750, },
		3952	{ 10000, 8875, },
		3953	{ 10125, 9000, },
		3954	{ 10250, 9125, },
		3955	{ 10375, 9250, },
		3956	{ 10500, 9375, },
		3957	{ 10625, 9500, },
		3958	{ 10750, 9625, },
		3959	{ 10875, 9750, },
		3960	{ 11000, 9875, },
		3961	{ 11125, 10000, },
		3962	{ 11250, 10125, },
		3963	{ 11375, 10250, },
		3964	{ 11500, 10375, },
		3965	{ 11625, 10500, },
		3966	{ 11750, 10625, },
		3967	{ 11875, 10750, },
		3968	{ 12000, 10875, },
		3969	{ 12125, 11000, },
		3970	{ 12250, 11125, },
		3971	{ 12375, 11250, },
		3972	{ 12500, 11375, },
		3973	{ 12625, 11500, },
		3974	{ 12750, 11625, },
		3975	{ 12875, 11750, },
		3976	{ 13000, 11875, },
		3977	{ 13125, 12000, },
		3978	{ 13250, 12125, },
		3979	{ 13375, 12250, },
		3980	{ 13500, 12375, },
		3981	{ 13625, 12500, },
		3982	{ 13750, 12625, },
		3983	{ 13875, 12750, },
		3984	{ 14000, 12875, },
		3985	{ 14125, 13000, },
		3986	{ 14250, 13125, },
		3987	{ 14375, 13250, },
		3988	{ 14500, 13375, },
		3989	{ 14625, 13500, },
		3990	{ 14750, 13625, },
		3991	{ 14875, 13750, },
		3992	{ 15000, 13875, },
		3993	{ 15125, 14000, },
		3994	{ 15250, 14125, },
		3995	{ 15375, 14250, },
		3996	{ 15500, 14375, },
		3997	{ 15625, 14500, },
		3998	{ 15750, 14625, },
		3999	{ 15875, 14750, },
		4000	{ 16000, 14875, },
		4001	{ 16125, 15000, },
		4002	};
		4003	if (dev_priv->info->is_mobile)
		4004	return v_table[pxvid].vm;
		4005	else
		4006	return v_table[pxvid].vd;
		4007	}
		4008
		4009	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		4010	{
		4011	struct timespec now, diff1;
		4012	u64 diff;
		4013	unsigned long diffms;
		4014	u32 count;
		4015
		4016	assert_spin_locked(&mchdev_lock);
		4017
		4018	getrawmonotonic(&now);
		4019	diff1 = timespec_sub(now, dev_priv->ips.last_time2);
		4020
		4021	/* Don't divide by 0 */
		4022	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
		4023	if (!diffms)
		4024	return;
		4025
		4026	count = I915_READ(GFXEC);
		4027
		4028	if (count < dev_priv->ips.last_count2) {
		4029	diff = ~0UL - dev_priv->ips.last_count2;
		4030	diff += count;
		4031	} else {
		4032	diff = count - dev_priv->ips.last_count2;
		4033	}
		4034
		4035	dev_priv->ips.last_count2 = count;
		4036	dev_priv->ips.last_time2 = now;
		4037
		4038	/* More magic constants... */
		4039	diff = diff * 1181;
		4040	diff = div_u64(diff, diffms * 10);
		4041	dev_priv->ips.gfx_power = diff;
		4042	}
		4043
		4044	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		4045	{
		4046	if (dev_priv->info->gen != 5)
		4047	return;
		4048
		4049	spin_lock_irq(&mchdev_lock);
		4050
		4051	__i915_update_gfx_val(dev_priv);
		4052
		4053	spin_unlock_irq(&mchdev_lock);
		4054	}
		4055
		4056	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		4057	{
		4058	unsigned long t, corr, state1, corr2, state2;
		4059	u32 pxvid, ext_v;
		4060
		4061	assert_spin_locked(&mchdev_lock);
		4062
		4063	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
		4064	pxvid = (pxvid >> 24) & 0x7f;
		4065	ext_v = pvid_to_extvid(dev_priv, pxvid);
		4066
		4067	state1 = ext_v;
		4068
		4069	t = i915_mch_val(dev_priv);
		4070
		4071	/* Revel in the empirically derived constants */
		4072
		4073	/* Correction factor in 1/100000 units */
		4074	if (t > 80)
		4075	corr = ((t * 2349) + 135940);
		4076	else if (t >= 50)
		4077	corr = ((t * 964) + 29317);
		4078	else /* < 50 */
		4079	corr = ((t * 301) + 1004);
		4080
		4081	corr = corr * ((150142 * state1) / 10000 - 78642);
		4082	corr /= 100000;
		4083	corr2 = (corr * dev_priv->ips.corr);
		4084
		4085	state2 = (corr2 * state1) / 10000;
		4086	state2 /= 100; /* convert to mW */
		4087
		4088	__i915_update_gfx_val(dev_priv);
		4089
		4090	return dev_priv->ips.gfx_power + state2;
		4091	}
		4092
		4093	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		4094	{
		4095	unsigned long val;
		4096
		4097	if (dev_priv->info->gen != 5)
		4098	return 0;
		4099
		4100	spin_lock_irq(&mchdev_lock);
		4101
		4102	val = __i915_gfx_val(dev_priv);
		4103
		4104	spin_unlock_irq(&mchdev_lock);
		4105
		4106	return val;
		4107	}
		4108
		4109	/**
		4110	* i915_read_mch_val - return value for IPS use
		4111	*
		4112	* Calculate and return a value for the IPS driver to use when deciding whether
		4113	* we have thermal and power headroom to increase CPU or GPU power budget.
		4114	*/
		4115	unsigned long i915_read_mch_val(void)
		4116	{
		4117	struct drm_i915_private *dev_priv;
		4118	unsigned long chipset_val, graphics_val, ret = 0;
		4119
		4120	spin_lock_irq(&mchdev_lock);
		4121	if (!i915_mch_dev)
		4122	goto out_unlock;
		4123	dev_priv = i915_mch_dev;
		4124
		4125	chipset_val = __i915_chipset_val(dev_priv);
		4126	graphics_val = __i915_gfx_val(dev_priv);
		4127
		4128	ret = chipset_val + graphics_val;
		4129
		4130	out_unlock:
		4131	spin_unlock_irq(&mchdev_lock);
		4132
		4133	return ret;
		4134	}
		4135	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		4136
		4137	/**
		4138	* i915_gpu_raise - raise GPU frequency limit
		4139	*
		4140	* Raise the limit; IPS indicates we have thermal headroom.
		4141	*/
		4142	bool i915_gpu_raise(void)
		4143	{
		4144	struct drm_i915_private *dev_priv;
		4145	bool ret = true;
		4146
		4147	spin_lock_irq(&mchdev_lock);
		4148	if (!i915_mch_dev) {
		4149	ret = false;
		4150	goto out_unlock;
		4151	}
		4152	dev_priv = i915_mch_dev;
		4153
		4154	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		4155	dev_priv->ips.max_delay--;
		4156
		4157	out_unlock:
		4158	spin_unlock_irq(&mchdev_lock);
		4159
		4160	return ret;
		4161	}
		4162	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		4163
		4164	/**
		4165	* i915_gpu_lower - lower GPU frequency limit
		4166	*
		4167	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		4168	* frequency maximum.
		4169	*/
		4170	bool i915_gpu_lower(void)
		4171	{
		4172	struct drm_i915_private *dev_priv;
		4173	bool ret = true;
		4174
		4175	spin_lock_irq(&mchdev_lock);
		4176	if (!i915_mch_dev) {
		4177	ret = false;
		4178	goto out_unlock;
		4179	}
		4180	dev_priv = i915_mch_dev;
		4181
		4182	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		4183	dev_priv->ips.max_delay++;
		4184
		4185	out_unlock:
		4186	spin_unlock_irq(&mchdev_lock);
		4187
		4188	return ret;
		4189	}
		4190	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		4191
		4192	/**
		4193	* i915_gpu_busy - indicate GPU business to IPS
		4194	*
		4195	* Tell the IPS driver whether or not the GPU is busy.
		4196	*/
		4197	bool i915_gpu_busy(void)
		4198	{
		4199	struct drm_i915_private *dev_priv;
		4200	struct intel_ring_buffer *ring;
		4201	bool ret = false;
		4202	int i;
		4203
		4204	spin_lock_irq(&mchdev_lock);
		4205	if (!i915_mch_dev)
		4206	goto out_unlock;
		4207	dev_priv = i915_mch_dev;
		4208
		4209	for_each_ring(ring, dev_priv, i)
		4210	ret \|= !list_empty(&ring->request_list);
		4211
		4212	out_unlock:
		4213	spin_unlock_irq(&mchdev_lock);
		4214
		4215	return ret;
		4216	}
		4217	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		4218
		4219	/**
		4220	* i915_gpu_turbo_disable - disable graphics turbo
		4221	*
		4222	* Disable graphics turbo by resetting the max frequency and setting the
		4223	* current frequency to the default.
		4224	*/
		4225	bool i915_gpu_turbo_disable(void)
		4226	{
		4227	struct drm_i915_private *dev_priv;
		4228	bool ret = true;
		4229
		4230	spin_lock_irq(&mchdev_lock);
		4231	if (!i915_mch_dev) {
		4232	ret = false;
		4233	goto out_unlock;
		4234	}
		4235	dev_priv = i915_mch_dev;
		4236
		4237	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		4238
		4239	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		4240	ret = false;
		4241
		4242	out_unlock:
		4243	spin_unlock_irq(&mchdev_lock);
		4244
		4245	return ret;
		4246	}
		4247	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		4248
		4249	/**
		4250	* Tells the intel_ips driver that the i915 driver is now loaded, if
		4251	* IPS got loaded first.
		4252	*
		4253	* This awkward dance is so that neither module has to depend on the
		4254	* other in order for IPS to do the appropriate communication of
		4255	* GPU turbo limits to i915.
		4256	*/
		4257	static void
		4258	ips_ping_for_i915_load(void)
		4259	{
		4260	void (*link)(void);
		4261
		4262	// link = symbol_get(ips_link_to_i915_driver);
		4263	// if (link) {
		4264	// link();
		4265	// symbol_put(ips_link_to_i915_driver);
		4266	// }
		4267	}
		4268
		4269	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		4270	{
		4271	/* We only register the i915 ips part with intel-ips once everything is
		4272	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		4273	spin_lock_irq(&mchdev_lock);
		4274	i915_mch_dev = dev_priv;
		4275	spin_unlock_irq(&mchdev_lock);
		4276
		4277	ips_ping_for_i915_load();
		4278	}
		4279
		4280	void intel_gpu_ips_teardown(void)
		4281	{
		4282	spin_lock_irq(&mchdev_lock);
		4283	i915_mch_dev = NULL;
		4284	spin_unlock_irq(&mchdev_lock);
		4285	}
		4286	static void intel_init_emon(struct drm_device *dev)
		4287	{
		4288	struct drm_i915_private *dev_priv = dev->dev_private;
		4289	u32 lcfuse;
		4290	u8 pxw[16];
		4291	int i;
		4292
		4293	/* Disable to program */
		4294	I915_WRITE(ECR, 0);
		4295	POSTING_READ(ECR);
		4296
		4297	/* Program energy weights for various events */
		4298	I915_WRITE(SDEW, 0x15040d00);
		4299	I915_WRITE(CSIEW0, 0x007f0000);
		4300	I915_WRITE(CSIEW1, 0x1e220004);
		4301	I915_WRITE(CSIEW2, 0x04000004);
		4302
		4303	for (i = 0; i < 5; i++)
		4304	I915_WRITE(PEW + (i * 4), 0);
		4305	for (i = 0; i < 3; i++)
		4306	I915_WRITE(DEW + (i * 4), 0);
		4307
		4308	/* Program P-state weights to account for frequency power adjustment */
		4309	for (i = 0; i < 16; i++) {
		4310	u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
		4311	unsigned long freq = intel_pxfreq(pxvidfreq);
		4312	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		4313	PXVFREQ_PX_SHIFT;
		4314	unsigned long val;
		4315
		4316	val = vid * vid;
		4317	val *= (freq / 1000);
		4318	val *= 255;
		4319	val /= (127127900);
		4320	if (val > 0xff)
		4321	DRM_ERROR("bad pxval: %ld\n", val);
		4322	pxw[i] = val;
		4323	}
		4324	/* Render standby states get 0 weight */
		4325	pxw[14] = 0;
		4326	pxw[15] = 0;
		4327
		4328	for (i = 0; i < 4; i++) {
		4329	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		4330	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
		4331	I915_WRITE(PXW + (i * 4), val);
		4332	}
		4333
		4334	/* Adjust magic regs to magic values (more experimental results) */
		4335	I915_WRITE(OGW0, 0);
		4336	I915_WRITE(OGW1, 0);
		4337	I915_WRITE(EG0, 0x00007f00);
		4338	I915_WRITE(EG1, 0x0000000e);
		4339	I915_WRITE(EG2, 0x000e0000);
		4340	I915_WRITE(EG3, 0x68000300);
		4341	I915_WRITE(EG4, 0x42000000);
		4342	I915_WRITE(EG5, 0x00140031);
		4343	I915_WRITE(EG6, 0);
		4344	I915_WRITE(EG7, 0);
		4345
		4346	for (i = 0; i < 8; i++)
		4347	I915_WRITE(PXWL + (i * 4), 0);
		4348
		4349	/* Enable PMON + select events */
		4350	I915_WRITE(ECR, 0x80000019);
		4351
		4352	lcfuse = I915_READ(LCFUSE02);
		4353
		4354	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		4355	}
		4356
		4357	void intel_disable_gt_powersave(struct drm_device *dev)
		4358	{
3243	Serge	4359	struct drm_i915_private *dev_priv = dev->dev_private;
		4360
4104	Serge	4361	/* Interrupts should be disabled already to avoid re-arming. */
		4362	WARN_ON(dev->irq_enabled);
		4363
3031	serge	4364	if (IS_IRONLAKE_M(dev)) {
		4365	ironlake_disable_drps(dev);
		4366	ironlake_disable_rc6(dev);
4293	Serge	4367	} else if (INTEL_INFO(dev)->gen >= 6) {
		4368	cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
		4369	cancel_work_sync(&dev_priv->rps.work);
3482	Serge	4370	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	4371	if (IS_VALLEYVIEW(dev))
		4372	valleyview_disable_rps(dev);
		4373	else
3031	serge	4374	gen6_disable_rps(dev);
4560	Serge	4375	dev_priv->rps.enabled = false;
3480	Serge	4376	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	4377	}
		4378	}
		4379
3482	Serge	4380	static void intel_gen6_powersave_work(struct work_struct *work)
		4381	{
		4382	struct drm_i915_private *dev_priv =
		4383	container_of(work, struct drm_i915_private,
		4384	rps.delayed_resume_work.work);
		4385	struct drm_device *dev = dev_priv->dev;
		4386
		4387	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	4388
		4389	if (IS_VALLEYVIEW(dev)) {
		4390	valleyview_enable_rps(dev);
4560	Serge	4391	} else if (IS_BROADWELL(dev)) {
		4392	gen8_enable_rps(dev);
		4393	gen6_update_ring_freq(dev);
4104	Serge	4394	} else {
3482	Serge	4395	gen6_enable_rps(dev);
		4396	gen6_update_ring_freq(dev);
4104	Serge	4397	}
4560	Serge	4398	dev_priv->rps.enabled = true;
3482	Serge	4399	mutex_unlock(&dev_priv->rps.hw_lock);
		4400	}
		4401
3031	serge	4402	void intel_enable_gt_powersave(struct drm_device *dev)
		4403	{
3243	Serge	4404	struct drm_i915_private *dev_priv = dev->dev_private;
		4405
3031	serge	4406	if (IS_IRONLAKE_M(dev)) {
		4407	ironlake_enable_drps(dev);
		4408	ironlake_enable_rc6(dev);
		4409	intel_init_emon(dev);
4104	Serge	4410	} else if (IS_GEN6(dev) \|\| IS_GEN7(dev)) {
3243	Serge	4411	/*
		4412	* PCU communication is slow and this doesn't need to be
		4413	* done at any specific time, so do this out of our fast path
		4414	* to make resume and init faster.
		4415	*/
3482	Serge	4416	schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		4417	round_jiffies_up_relative(HZ));
3031	serge	4418	}
		4419	}
		4420
3243	Serge	4421	static void ibx_init_clock_gating(struct drm_device *dev)
		4422	{
		4423	struct drm_i915_private *dev_priv = dev->dev_private;
		4424
		4425	/*
		4426	* On Ibex Peak and Cougar Point, we need to disable clock
		4427	* gating for the panel power sequencer or it will fail to
		4428	* start up when no ports are active.
		4429	*/
		4430	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		4431	}
		4432
4104	Serge	4433	static void g4x_disable_trickle_feed(struct drm_device *dev)
		4434	{
		4435	struct drm_i915_private *dev_priv = dev->dev_private;
		4436	int pipe;
		4437
		4438	for_each_pipe(pipe) {
		4439	I915_WRITE(DSPCNTR(pipe),
		4440	I915_READ(DSPCNTR(pipe)) \|
		4441	DISPPLANE_TRICKLE_FEED_DISABLE);
4560	Serge	4442	intel_flush_primary_plane(dev_priv, pipe);
4104	Serge	4443	}
		4444	}
		4445
4560	Serge	4446	static void ilk_init_lp_watermarks(struct drm_device *dev)
		4447	{
		4448	struct drm_i915_private *dev_priv = dev->dev_private;
		4449
		4450	I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
		4451	I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
		4452	I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
		4453
		4454	/*
		4455	* Don't touch WM1S_LP_EN here.
		4456	* Doing so could cause underruns.
		4457	*/
		4458	}
		4459
3031	serge	4460	static void ironlake_init_clock_gating(struct drm_device *dev)
		4461	{
		4462	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	4463	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	4464
4104	Serge	4465	/*
		4466	* Required for FBC
		4467	* WaFbcDisableDpfcClockGating:ilk
		4468	*/
3243	Serge	4469	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		4470	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		4471	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	4472
		4473	I915_WRITE(PCH_3DCGDIS0,
		4474	MARIUNIT_CLOCK_GATE_DISABLE \|
		4475	SVSMUNIT_CLOCK_GATE_DISABLE);
		4476	I915_WRITE(PCH_3DCGDIS1,
		4477	VFMUNIT_CLOCK_GATE_DISABLE);
		4478
		4479	/*
		4480	* According to the spec the following bits should be set in
		4481	* order to enable memory self-refresh
		4482	* The bit 22/21 of 0x42004
		4483	* The bit 5 of 0x42020
		4484	* The bit 15 of 0x45000
		4485	*/
		4486	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4487	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4488	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	4489	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	4490	I915_WRITE(DISP_ARB_CTL,
		4491	(I915_READ(DISP_ARB_CTL) \|
		4492	DISP_FBC_WM_DIS));
		4493
4560	Serge	4494	ilk_init_lp_watermarks(dev);
		4495
3031	serge	4496	/*
		4497	* Based on the document from hardware guys the following bits
		4498	* should be set unconditionally in order to enable FBC.
		4499	* The bit 22 of 0x42000
		4500	* The bit 22 of 0x42004
		4501	* The bit 7,8,9 of 0x42020.
		4502	*/
		4503	if (IS_IRONLAKE_M(dev)) {
4104	Serge	4504	/* WaFbcAsynchFlipDisableFbcQueue:ilk */
3031	serge	4505	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		4506	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		4507	ILK_FBCQ_DIS);
		4508	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4509	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4510	ILK_DPARB_GATE);
		4511	}
		4512
3243	Serge	4513	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		4514
3031	serge	4515	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4516	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4517	ILK_ELPIN_409_SELECT);
		4518	I915_WRITE(_3D_CHICKEN2,
		4519	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		4520	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	4521
4104	Serge	4522	/* WaDisableRenderCachePipelinedFlush:ilk */
3243	Serge	4523	I915_WRITE(CACHE_MODE_0,
		4524	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		4525
4104	Serge	4526	g4x_disable_trickle_feed(dev);
		4527
3243	Serge	4528	ibx_init_clock_gating(dev);
3031	serge	4529	}
		4530
3243	Serge	4531	static void cpt_init_clock_gating(struct drm_device *dev)
		4532	{
		4533	struct drm_i915_private *dev_priv = dev->dev_private;
		4534	int pipe;
3746	Serge	4535	uint32_t val;
3243	Serge	4536
		4537	/*
		4538	* On Ibex Peak and Cougar Point, we need to disable clock
		4539	* gating for the panel power sequencer or it will fail to
		4540	* start up when no ports are active.
		4541	*/
4280	Serge	4542	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE \|
		4543	PCH_DPLUNIT_CLOCK_GATE_DISABLE \|
		4544	PCH_CPUNIT_CLOCK_GATE_DISABLE);
3243	Serge	4545	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		4546	DPLS_EDP_PPS_FIX_DIS);
		4547	/* The below fixes the weird display corruption, a few pixels shifted
		4548	* downward, on (only) LVDS of some HP laptops with IVY.
		4549	*/
3746	Serge	4550	for_each_pipe(pipe) {
		4551	val = I915_READ(TRANS_CHICKEN2(pipe));
		4552	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		4553	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4104	Serge	4554	if (dev_priv->vbt.fdi_rx_polarity_inverted)
3746	Serge	4555	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		4556	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		4557	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		4558	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		4559	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		4560	}
3243	Serge	4561	/* WADP0ClockGatingDisable */
		4562	for_each_pipe(pipe) {
		4563	I915_WRITE(TRANS_CHICKEN1(pipe),
		4564	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		4565	}
		4566	}
		4567
3480	Serge	4568	static void gen6_check_mch_setup(struct drm_device *dev)
		4569	{
		4570	struct drm_i915_private *dev_priv = dev->dev_private;
		4571	uint32_t tmp;
		4572
		4573	tmp = I915_READ(MCH_SSKPD);
		4574	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) {
		4575	DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp);
		4576	DRM_INFO("This can cause pipe underruns and display issues.\n");
		4577	DRM_INFO("Please upgrade your BIOS to fix this.\n");
		4578	}
		4579	}
		4580
3031	serge	4581	static void gen6_init_clock_gating(struct drm_device *dev)
		4582	{
		4583	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	4584	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	4585
3243	Serge	4586	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	4587
		4588	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4589	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4590	ILK_ELPIN_409_SELECT);
		4591
4104	Serge	4592	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3243	Serge	4593	I915_WRITE(_3D_CHICKEN,
		4594	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		4595
4104	Serge	4596	/* WaSetupGtModeTdRowDispatch:snb */
3243	Serge	4597	if (IS_SNB_GT1(dev))
		4598	I915_WRITE(GEN6_GT_MODE,
		4599	_MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
		4600
4560	Serge	4601	ilk_init_lp_watermarks(dev);
3031	serge	4602
		4603	I915_WRITE(CACHE_MODE_0,
		4604	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		4605
		4606	I915_WRITE(GEN6_UCGCTL1,
		4607	I915_READ(GEN6_UCGCTL1) \|
		4608	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		4609	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		4610
		4611	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		4612	* gating disable must be set. Failure to set it results in
		4613	* flickering pixels due to Z write ordering failures after
		4614	* some amount of runtime in the Mesa "fire" demo, and Unigine
		4615	* Sanctuary and Tropics, and apparently anything else with
		4616	* alpha test or pixel discard.
		4617	*
		4618	* According to the spec, bit 11 (RCCUNIT) must also be set,
		4619	* but we didn't debug actual testcases to find it out.
		4620	*
4104	Serge	4621	* Also apply WaDisableVDSUnitClockGating:snb and
		4622	* WaDisableRCPBUnitClockGating:snb.
3031	serge	4623	*/
		4624	I915_WRITE(GEN6_UCGCTL2,
		4625	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		4626	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		4627	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		4628
		4629	/* Bspec says we need to always set all mask bits. */
		4630	I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) \|
		4631	_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
		4632
		4633	/*
		4634	* According to the spec the following bits should be
		4635	* set in order to enable memory self-refresh and fbc:
		4636	* The bit21 and bit22 of 0x42000
		4637	* The bit21 and bit22 of 0x42004
		4638	* The bit5 and bit7 of 0x42020
		4639	* The bit14 of 0x70180
		4640	* The bit14 of 0x71180
4104	Serge	4641	*
		4642	* WaFbcAsynchFlipDisableFbcQueue:snb
3031	serge	4643	*/
		4644	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		4645	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		4646	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		4647	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		4648	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		4649	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	4650	I915_WRITE(ILK_DSPCLK_GATE_D,
		4651	I915_READ(ILK_DSPCLK_GATE_D) \|
		4652	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		4653	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	4654
4104	Serge	4655	g4x_disable_trickle_feed(dev);
3031	serge	4656
		4657	/* The default value should be 0x200 according to docs, but the two
		4658	* platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
		4659	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
		4660	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
3243	Serge	4661
		4662	cpt_init_clock_gating(dev);
3480	Serge	4663
		4664	gen6_check_mch_setup(dev);
3031	serge	4665	}
		4666
		4667	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		4668	{
		4669	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		4670
		4671	reg &= ~GEN7_FF_SCHED_MASK;
		4672	reg \|= GEN7_FF_TS_SCHED_HW;
		4673	reg \|= GEN7_FF_VS_SCHED_HW;
		4674	reg \|= GEN7_FF_DS_SCHED_HW;
		4675
3480	Serge	4676	if (IS_HASWELL(dev_priv->dev))
		4677	reg &= ~GEN7_FF_VS_REF_CNT_FFME;
		4678
3031	serge	4679	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		4680	}
		4681
3243	Serge	4682	static void lpt_init_clock_gating(struct drm_device *dev)
		4683	{
		4684	struct drm_i915_private *dev_priv = dev->dev_private;
		4685
		4686	/*
		4687	* TODO: this bit should only be enabled when really needed, then
		4688	* disabled when not needed anymore in order to save power.
		4689	*/
		4690	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
		4691	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		4692	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		4693	PCH_LP_PARTITION_LEVEL_DISABLE);
4104	Serge	4694
		4695	/* WADPOClockGatingDisable:hsw */
		4696	I915_WRITE(_TRANSA_CHICKEN1,
		4697	I915_READ(_TRANSA_CHICKEN1) \|
		4698	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3243	Serge	4699	}
		4700
4104	Serge	4701	static void lpt_suspend_hw(struct drm_device *dev)
		4702	{
		4703	struct drm_i915_private *dev_priv = dev->dev_private;
		4704
		4705	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
		4706	uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
		4707
		4708	val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		4709	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
		4710	}
		4711	}
		4712
4560	Serge	4713	static void gen8_init_clock_gating(struct drm_device *dev)
3031	serge	4714	{
		4715	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4716	enum pipe i;
3031	serge	4717
		4718	I915_WRITE(WM3_LP_ILK, 0);
		4719	I915_WRITE(WM2_LP_ILK, 0);
		4720	I915_WRITE(WM1_LP_ILK, 0);
		4721
4560	Serge	4722	/* FIXME(BDW): Check all the w/a, some might only apply to
		4723	* pre-production hw. */
		4724
		4725	WARN(!i915_preliminary_hw_support,
		4726	"GEN8_CENTROID_PIXEL_OPT_DIS not be needed for production\n");
		4727	I915_WRITE(HALF_SLICE_CHICKEN3,
		4728	_MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS));
		4729	I915_WRITE(HALF_SLICE_CHICKEN3,
		4730	_MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
		4731	I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE));
		4732
		4733	I915_WRITE(_3D_CHICKEN3,
		4734	_3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2));
		4735
		4736	I915_WRITE(COMMON_SLICE_CHICKEN2,
		4737	_MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE));
		4738
		4739	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		4740	_MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE));
		4741
		4742	/* WaSwitchSolVfFArbitrationPriority:bdw */
		4743	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		4744
		4745	/* WaPsrDPAMaskVBlankInSRD:bdw */
		4746	I915_WRITE(CHICKEN_PAR1_1,
		4747	I915_READ(CHICKEN_PAR1_1) \| DPA_MASK_VBLANK_SRD);
		4748
		4749	/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
		4750	for_each_pipe(i) {
		4751	I915_WRITE(CHICKEN_PIPESL_1(i),
		4752	I915_READ(CHICKEN_PIPESL_1(i) \|
		4753	DPRS_MASK_VBLANK_SRD));
		4754	}
		4755
		4756	/* Use Force Non-Coherent whenever executing a 3D context. This is a
		4757	* workaround for for a possible hang in the unlikely event a TLB
		4758	* invalidation occurs during a PSD flush.
		4759	*/
		4760	I915_WRITE(HDC_CHICKEN0,
		4761	I915_READ(HDC_CHICKEN0) \|
		4762	_MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT));
		4763
		4764	/* WaVSRefCountFullforceMissDisable:bdw */
		4765	/* WaDSRefCountFullforceMissDisable:bdw */
		4766	I915_WRITE(GEN7_FF_THREAD_MODE,
		4767	I915_READ(GEN7_FF_THREAD_MODE) &
		4768	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
		4769	}
		4770
		4771	static void haswell_init_clock_gating(struct drm_device *dev)
		4772	{
		4773	struct drm_i915_private *dev_priv = dev->dev_private;
		4774
		4775	ilk_init_lp_watermarks(dev);
		4776
3031	serge	4777	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	4778	* This implements the WaDisableRCZUnitClockGating:hsw workaround.
3031	serge	4779	*/
		4780	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
		4781
4104	Serge	4782	/* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */
3031	serge	4783	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		4784	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		4785
4104	Serge	4786	/* WaApplyL3ControlAndL3ChickenMode:hsw */
3031	serge	4787	I915_WRITE(GEN7_L3CNTLREG1,
		4788	GEN7_WA_FOR_GEN7_L3_CONTROL);
		4789	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		4790	GEN7_WA_L3_CHICKEN_MODE);
		4791
4104	Serge	4792	/* L3 caching of data atomics doesn't work -- disable it. */
		4793	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
		4794	I915_WRITE(HSW_ROW_CHICKEN3,
		4795	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
		4796
		4797	/* This is required by WaCatErrorRejectionIssue:hsw */
3031	serge	4798	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		4799	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		4800	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		4801
4104	Serge	4802	/* WaVSRefCountFullforceMissDisable:hsw */
3031	serge	4803	gen7_setup_fixed_func_scheduler(dev_priv);
		4804
4104	Serge	4805	/* WaDisable4x2SubspanOptimization:hsw */
3031	serge	4806	I915_WRITE(CACHE_MODE_1,
		4807	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		4808
4104	Serge	4809	/* WaSwitchSolVfFArbitrationPriority:hsw */
3746	Serge	4810	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		4811
4104	Serge	4812	/* WaRsPkgCStateDisplayPMReq:hsw */
		4813	I915_WRITE(CHICKEN_PAR1_1,
		4814	I915_READ(CHICKEN_PAR1_1) \| FORCE_ARB_IDLE_PLANES);
3031	serge	4815
3243	Serge	4816	lpt_init_clock_gating(dev);
3031	serge	4817	}
		4818
		4819	static void ivybridge_init_clock_gating(struct drm_device *dev)
		4820	{
		4821	struct drm_i915_private *dev_priv = dev->dev_private;
		4822	uint32_t snpcr;
		4823
4560	Serge	4824	ilk_init_lp_watermarks(dev);
3031	serge	4825
3243	Serge	4826	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	4827
4104	Serge	4828	/* WaDisableEarlyCull:ivb */
3243	Serge	4829	I915_WRITE(_3D_CHICKEN3,
		4830	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		4831
4104	Serge	4832	/* WaDisableBackToBackFlipFix:ivb */
3031	serge	4833	I915_WRITE(IVB_CHICKEN3,
		4834	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		4835	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		4836
4104	Serge	4837	/* WaDisablePSDDualDispatchEnable:ivb */
3243	Serge	4838	if (IS_IVB_GT1(dev))
		4839	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		4840	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		4841	else
		4842	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
		4843	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		4844
4104	Serge	4845	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
3031	serge	4846	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		4847	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		4848
4104	Serge	4849	/* WaApplyL3ControlAndL3ChickenMode:ivb */
3031	serge	4850	I915_WRITE(GEN7_L3CNTLREG1,
		4851	GEN7_WA_FOR_GEN7_L3_CONTROL);
		4852	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		4853	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	4854	if (IS_IVB_GT1(dev))
		4855	I915_WRITE(GEN7_ROW_CHICKEN2,
		4856	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		4857	else
		4858	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		4859	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3031	serge	4860
3243	Serge	4861
4104	Serge	4862	/* WaForceL3Serialization:ivb */
3243	Serge	4863	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		4864	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		4865
3031	serge	4866	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		4867	* gating disable must be set. Failure to set it results in
		4868	* flickering pixels due to Z write ordering failures after
		4869	* some amount of runtime in the Mesa "fire" demo, and Unigine
		4870	* Sanctuary and Tropics, and apparently anything else with
		4871	* alpha test or pixel discard.
		4872	*
		4873	* According to the spec, bit 11 (RCCUNIT) must also be set,
		4874	* but we didn't debug actual testcases to find it out.
		4875	*
		4876	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	4877	* This implements the WaDisableRCZUnitClockGating:ivb workaround.
3031	serge	4878	*/
		4879	I915_WRITE(GEN6_UCGCTL2,
		4880	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		4881	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		4882
4104	Serge	4883	/* This is required by WaCatErrorRejectionIssue:ivb */
3031	serge	4884	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		4885	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		4886	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		4887
4104	Serge	4888	g4x_disable_trickle_feed(dev);
3031	serge	4889
4104	Serge	4890	/* WaVSRefCountFullforceMissDisable:ivb */
3031	serge	4891	gen7_setup_fixed_func_scheduler(dev_priv);
		4892
4104	Serge	4893	/* WaDisable4x2SubspanOptimization:ivb */
3031	serge	4894	I915_WRITE(CACHE_MODE_1,
		4895	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		4896
		4897	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		4898	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		4899	snpcr \|= GEN6_MBC_SNPCR_MED;
		4900	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	4901
3746	Serge	4902	if (!HAS_PCH_NOP(dev))
3243	Serge	4903	cpt_init_clock_gating(dev);
3480	Serge	4904
		4905	gen6_check_mch_setup(dev);
3031	serge	4906	}
		4907
		4908	static void valleyview_init_clock_gating(struct drm_device *dev)
		4909	{
		4910	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	4911	u32 val;
3031	serge	4912
4560	Serge	4913	mutex_lock(&dev_priv->rps.hw_lock);
		4914	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		4915	mutex_unlock(&dev_priv->rps.hw_lock);
		4916	switch ((val >> 6) & 3) {
		4917	case 0:
		4918	dev_priv->mem_freq = 800;
		4919	break;
		4920	case 1:
		4921	dev_priv->mem_freq = 1066;
		4922	break;
		4923	case 2:
		4924	dev_priv->mem_freq = 1333;
		4925	break;
		4926	case 3:
		4927	dev_priv->mem_freq = 1333;
		4928	break;
		4929	}
		4930	DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
		4931
4104	Serge	4932	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	4933
4104	Serge	4934	/* WaDisableEarlyCull:vlv */
3243	Serge	4935	I915_WRITE(_3D_CHICKEN3,
		4936	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		4937
4104	Serge	4938	/* WaDisableBackToBackFlipFix:vlv */
3031	serge	4939	I915_WRITE(IVB_CHICKEN3,
		4940	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		4941	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		4942
4104	Serge	4943	/* WaDisablePSDDualDispatchEnable:vlv */
3243	Serge	4944	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	4945	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		4946	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	4947
4104	Serge	4948	/* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */
3031	serge	4949	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		4950	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		4951
4104	Serge	4952	/* WaApplyL3ControlAndL3ChickenMode:vlv */
3243	Serge	4953	I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) \| GEN7_L3AGDIS);
3031	serge	4954	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
		4955
4104	Serge	4956	/* WaForceL3Serialization:vlv */
3243	Serge	4957	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		4958	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		4959
4104	Serge	4960	/* WaDisableDopClockGating:vlv */
3243	Serge	4961	I915_WRITE(GEN7_ROW_CHICKEN2,
		4962	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		4963
4104	Serge	4964	/* This is required by WaCatErrorRejectionIssue:vlv */
3031	serge	4965	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		4966	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		4967	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		4968
		4969	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		4970	* gating disable must be set. Failure to set it results in
		4971	* flickering pixels due to Z write ordering failures after
		4972	* some amount of runtime in the Mesa "fire" demo, and Unigine
		4973	* Sanctuary and Tropics, and apparently anything else with
		4974	* alpha test or pixel discard.
		4975	*
		4976	* According to the spec, bit 11 (RCCUNIT) must also be set,
		4977	* but we didn't debug actual testcases to find it out.
		4978	*
		4979	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	4980	* This implements the WaDisableRCZUnitClockGating:vlv workaround.
3031	serge	4981	*
4104	Serge	4982	* Also apply WaDisableVDSUnitClockGating:vlv and
		4983	* WaDisableRCPBUnitClockGating:vlv.
3031	serge	4984	*/
		4985	I915_WRITE(GEN6_UCGCTL2,
		4986	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		4987	GEN7_TDLUNIT_CLOCK_GATE_DISABLE \|
		4988	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		4989	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		4990	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		4991
		4992	I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
		4993
4104	Serge	4994	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
3031	serge	4995
		4996	I915_WRITE(CACHE_MODE_1,
		4997	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		4998
		4999	/*
4104	Serge	5000	* WaDisableVLVClockGating_VBIIssue:vlv
3243	Serge	5001	* Disable clock gating on th GCFG unit to prevent a delay
		5002	* in the reporting of vblank events.
		5003	*/
3746	Serge	5004	I915_WRITE(VLV_GUNIT_CLOCK_GATE, 0xffffffff);
		5005
		5006	/* Conservative clock gating settings for now */
		5007	I915_WRITE(0x9400, 0xffffffff);
		5008	I915_WRITE(0x9404, 0xffffffff);
		5009	I915_WRITE(0x9408, 0xffffffff);
		5010	I915_WRITE(0x940c, 0xffffffff);
		5011	I915_WRITE(0x9410, 0xffffffff);
		5012	I915_WRITE(0x9414, 0xffffffff);
		5013	I915_WRITE(0x9418, 0xffffffff);
3031	serge	5014	}
		5015
		5016	static void g4x_init_clock_gating(struct drm_device *dev)
		5017	{
		5018	struct drm_i915_private *dev_priv = dev->dev_private;
		5019	uint32_t dspclk_gate;
		5020
		5021	I915_WRITE(RENCLK_GATE_D1, 0);
		5022	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		5023	GS_UNIT_CLOCK_GATE_DISABLE \|
		5024	CL_UNIT_CLOCK_GATE_DISABLE);
		5025	I915_WRITE(RAMCLK_GATE_D, 0);
		5026	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		5027	OVRUNIT_CLOCK_GATE_DISABLE \|
		5028	OVCUNIT_CLOCK_GATE_DISABLE;
		5029	if (IS_GM45(dev))
		5030	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		5031	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	5032
		5033	/* WaDisableRenderCachePipelinedFlush */
		5034	I915_WRITE(CACHE_MODE_0,
		5035	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4104	Serge	5036
		5037	g4x_disable_trickle_feed(dev);
3031	serge	5038	}
		5039
		5040	static void crestline_init_clock_gating(struct drm_device *dev)
		5041	{
		5042	struct drm_i915_private *dev_priv = dev->dev_private;
		5043
		5044	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		5045	I915_WRITE(RENCLK_GATE_D2, 0);
		5046	I915_WRITE(DSPCLK_GATE_D, 0);
		5047	I915_WRITE(RAMCLK_GATE_D, 0);
		5048	I915_WRITE16(DEUC, 0);
4104	Serge	5049	I915_WRITE(MI_ARB_STATE,
		5050	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	5051	}
		5052
		5053	static void broadwater_init_clock_gating(struct drm_device *dev)
		5054	{
		5055	struct drm_i915_private *dev_priv = dev->dev_private;
		5056
		5057	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		5058	I965_RCC_CLOCK_GATE_DISABLE \|
		5059	I965_RCPB_CLOCK_GATE_DISABLE \|
		5060	I965_ISC_CLOCK_GATE_DISABLE \|
		5061	I965_FBC_CLOCK_GATE_DISABLE);
		5062	I915_WRITE(RENCLK_GATE_D2, 0);
4104	Serge	5063	I915_WRITE(MI_ARB_STATE,
		5064	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
3031	serge	5065	}
		5066
		5067	static void gen3_init_clock_gating(struct drm_device *dev)
		5068	{
		5069	struct drm_i915_private *dev_priv = dev->dev_private;
		5070	u32 dstate = I915_READ(D_STATE);
		5071
		5072	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		5073	DSTATE_DOT_CLOCK_GATING;
		5074	I915_WRITE(D_STATE, dstate);
		5075
		5076	if (IS_PINEVIEW(dev))
		5077	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		5078
		5079	/* IIR "flip pending" means done if this bit is set */
		5080	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
		5081	}
		5082
		5083	static void i85x_init_clock_gating(struct drm_device *dev)
		5084	{
		5085	struct drm_i915_private *dev_priv = dev->dev_private;
		5086
		5087	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
		5088	}
		5089
		5090	static void i830_init_clock_gating(struct drm_device *dev)
		5091	{
		5092	struct drm_i915_private *dev_priv = dev->dev_private;
		5093
		5094	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
		5095	}
		5096
		5097	void intel_init_clock_gating(struct drm_device *dev)
		5098	{
		5099	struct drm_i915_private *dev_priv = dev->dev_private;
		5100
		5101	dev_priv->display.init_clock_gating(dev);
		5102	}
		5103
4104	Serge	5104	void intel_suspend_hw(struct drm_device *dev)
		5105	{
		5106	if (HAS_PCH_LPT(dev))
		5107	lpt_suspend_hw(dev);
		5108	}
		5109
4560	Serge	5110	#define for_each_power_well(i, power_well, domain_mask, power_domains) \
		5111	for (i = 0; \
		5112	i < (power_domains)->power_well_count && \
		5113	((power_well) = &(power_domains)->power_wells[i]); \
		5114	i++) \
		5115	if ((power_well)->domains & (domain_mask))
		5116
		5117	#define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \
		5118	for (i = (power_domains)->power_well_count - 1; \
		5119	i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\
		5120	i--) \
		5121	if ((power_well)->domains & (domain_mask))
		5122
3746	Serge	5123	/**
		5124	* We should only use the power well if we explicitly asked the hardware to
		5125	* enable it, so check if it's enabled and also check if we've requested it to
		5126	* be enabled.
		5127	*/
4560	Serge	5128	static bool hsw_power_well_enabled(struct drm_device *dev,
		5129	struct i915_power_well *power_well)
		5130	{
		5131	struct drm_i915_private *dev_priv = dev->dev_private;
		5132
		5133	return I915_READ(HSW_PWR_WELL_DRIVER) ==
		5134	(HSW_PWR_WELL_ENABLE_REQUEST \| HSW_PWR_WELL_STATE_ENABLED);
		5135	}
		5136
		5137	bool intel_display_power_enabled_sw(struct drm_device *dev,
		5138	enum intel_display_power_domain domain)
		5139	{
		5140	struct drm_i915_private *dev_priv = dev->dev_private;
		5141	struct i915_power_domains *power_domains;
		5142
		5143	power_domains = &dev_priv->power_domains;
		5144
		5145	return power_domains->domain_use_count[domain];
		5146	}
		5147
4104	Serge	5148	bool intel_display_power_enabled(struct drm_device *dev,
		5149	enum intel_display_power_domain domain)
3746	Serge	5150	{
		5151	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	5152	struct i915_power_domains *power_domains;
		5153	struct i915_power_well *power_well;
		5154	bool is_enabled;
		5155	int i;
3746	Serge	5156
4560	Serge	5157	power_domains = &dev_priv->power_domains;
4104	Serge	5158
4560	Serge	5159	is_enabled = true;
		5160
		5161	mutex_lock(&power_domains->lock);
		5162	for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
		5163	if (power_well->always_on)
		5164	continue;
		5165
		5166	if (!power_well->is_enabled(dev, power_well)) {
		5167	is_enabled = false;
		5168	break;
		5169	}
4104	Serge	5170	}
4560	Serge	5171	mutex_unlock(&power_domains->lock);
		5172
		5173	return is_enabled;
3746	Serge	5174	}
		5175
4560	Serge	5176	static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
3031	serge	5177	{
4560	Serge	5178	struct drm_device *dev = dev_priv->dev;
		5179	unsigned long irqflags;
		5180
		5181	/*
		5182	* After we re-enable the power well, if we touch VGA register 0x3d5
		5183	* we'll get unclaimed register interrupts. This stops after we write
		5184	* anything to the VGA MSR register. The vgacon module uses this
		5185	* register all the time, so if we unbind our driver and, as a
		5186	* consequence, bind vgacon, we'll get stuck in an infinite loop at
		5187	* console_unlock(). So make here we touch the VGA MSR register, making
		5188	* sure vgacon can keep working normally without triggering interrupts
		5189	* and error messages.
		5190	*/
		5191	// vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
		5192	outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
		5193	// vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
		5194
		5195	if (IS_BROADWELL(dev)) {
		5196	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		5197	I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_B),
		5198	dev_priv->de_irq_mask[PIPE_B]);
		5199	I915_WRITE(GEN8_DE_PIPE_IER(PIPE_B),
		5200	~dev_priv->de_irq_mask[PIPE_B] \|
		5201	GEN8_PIPE_VBLANK);
		5202	I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_C),
		5203	dev_priv->de_irq_mask[PIPE_C]);
		5204	I915_WRITE(GEN8_DE_PIPE_IER(PIPE_C),
		5205	~dev_priv->de_irq_mask[PIPE_C] \|
		5206	GEN8_PIPE_VBLANK);
		5207	POSTING_READ(GEN8_DE_PIPE_IER(PIPE_C));
		5208	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
		5209	}
		5210	}
		5211
		5212	static void hsw_power_well_post_disable(struct drm_i915_private *dev_priv)
		5213	{
		5214	struct drm_device *dev = dev_priv->dev;
		5215	enum pipe p;
		5216	unsigned long irqflags;
		5217
		5218	/*
		5219	* After this, the registers on the pipes that are part of the power
		5220	* well will become zero, so we have to adjust our counters according to
		5221	* that.
		5222	*
		5223	* FIXME: Should we do this in general in drm_vblank_post_modeset?
		5224	*/
		5225	// spin_lock_irqsave(&dev->vbl_lock, irqflags);
		5226	// for_each_pipe(p)
		5227	// if (p != PIPE_A)
		5228	// dev->vblank[p].last = 0;
		5229	// spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
		5230	}
		5231
		5232	static void hsw_set_power_well(struct drm_device *dev,
		5233	struct i915_power_well *power_well, bool enable)
		5234	{
3031	serge	5235	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	5236	bool is_enabled, enable_requested;
		5237	uint32_t tmp;
3031	serge	5238
4560	Serge	5239	WARN_ON(dev_priv->pc8.enabled);
		5240
3480	Serge	5241	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
4104	Serge	5242	is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
		5243	enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
3031	serge	5244
3480	Serge	5245	if (enable) {
		5246	if (!enable_requested)
4104	Serge	5247	I915_WRITE(HSW_PWR_WELL_DRIVER,
		5248	HSW_PWR_WELL_ENABLE_REQUEST);
3031	serge	5249
3480	Serge	5250	if (!is_enabled) {
		5251	DRM_DEBUG_KMS("Enabling power well\n");
		5252	if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
4104	Serge	5253	HSW_PWR_WELL_STATE_ENABLED), 20))
3480	Serge	5254	DRM_ERROR("Timeout enabling power well\n");
3031	serge	5255	}
4560	Serge	5256
		5257	hsw_power_well_post_enable(dev_priv);
3480	Serge	5258	} else {
		5259	if (enable_requested) {
		5260	I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
4104	Serge	5261	POSTING_READ(HSW_PWR_WELL_DRIVER);
3480	Serge	5262	DRM_DEBUG_KMS("Requesting to disable the power well\n");
4104	Serge	5263
4560	Serge	5264	hsw_power_well_post_disable(dev_priv);
4104	Serge	5265	}
		5266	}
		5267	}
		5268
4560	Serge	5269	static void __intel_power_well_get(struct drm_device *dev,
		5270	struct i915_power_well *power_well)
		5271	{
		5272	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	5273
4560	Serge	5274	if (!power_well->count++ && power_well->set) {
		5275	hsw_disable_package_c8(dev_priv);
		5276	power_well->set(dev, power_well, true);
		5277	}
		5278	}
		5279
		5280	static void __intel_power_well_put(struct drm_device *dev,
		5281	struct i915_power_well *power_well)
		5282	{
		5283	struct drm_i915_private *dev_priv = dev->dev_private;
		5284
		5285	WARN_ON(!power_well->count);
		5286
		5287	if (!--power_well->count && power_well->set &&
		5288	i915_disable_power_well) {
		5289	power_well->set(dev, power_well, false);
		5290	hsw_enable_package_c8(dev_priv);
		5291	}
		5292	}
		5293
		5294	void intel_display_power_get(struct drm_device *dev,
		5295	enum intel_display_power_domain domain)
		5296	{
		5297	struct drm_i915_private *dev_priv = dev->dev_private;
		5298	struct i915_power_domains *power_domains;
		5299	struct i915_power_well *power_well;
		5300	int i;
		5301
		5302	power_domains = &dev_priv->power_domains;
		5303
		5304	mutex_lock(&power_domains->lock);
		5305
		5306	for_each_power_well(i, power_well, BIT(domain), power_domains)
		5307	__intel_power_well_get(dev, power_well);
		5308
		5309	power_domains->domain_use_count[domain]++;
		5310
		5311	mutex_unlock(&power_domains->lock);
		5312	}
		5313
		5314	void intel_display_power_put(struct drm_device *dev,
		5315	enum intel_display_power_domain domain)
		5316	{
		5317	struct drm_i915_private *dev_priv = dev->dev_private;
		5318	struct i915_power_domains *power_domains;
		5319	struct i915_power_well *power_well;
		5320	int i;
		5321
		5322	power_domains = &dev_priv->power_domains;
		5323
		5324	mutex_lock(&power_domains->lock);
		5325
		5326	WARN_ON(!power_domains->domain_use_count[domain]);
		5327	power_domains->domain_use_count[domain]--;
		5328
		5329	for_each_power_well_rev(i, power_well, BIT(domain), power_domains)
		5330	__intel_power_well_put(dev, power_well);
		5331
		5332	mutex_unlock(&power_domains->lock);
		5333	}
		5334
		5335	static struct i915_power_domains *hsw_pwr;
		5336
4104	Serge	5337	/* Display audio driver power well request */
		5338	void i915_request_power_well(void)
		5339	{
4560	Serge	5340	struct drm_i915_private *dev_priv;
		5341
4104	Serge	5342	if (WARN_ON(!hsw_pwr))
		5343	return;
		5344
4560	Serge	5345	dev_priv = container_of(hsw_pwr, struct drm_i915_private,
		5346	power_domains);
		5347	intel_display_power_get(dev_priv->dev, POWER_DOMAIN_AUDIO);
4104	Serge	5348	}
		5349	EXPORT_SYMBOL_GPL(i915_request_power_well);
		5350
		5351	/* Display audio driver power well release */
		5352	void i915_release_power_well(void)
		5353	{
4560	Serge	5354	struct drm_i915_private *dev_priv;
		5355
4104	Serge	5356	if (WARN_ON(!hsw_pwr))
		5357	return;
		5358
4560	Serge	5359	dev_priv = container_of(hsw_pwr, struct drm_i915_private,
		5360	power_domains);
		5361	intel_display_power_put(dev_priv->dev, POWER_DOMAIN_AUDIO);
4104	Serge	5362	}
		5363	EXPORT_SYMBOL_GPL(i915_release_power_well);
		5364
4560	Serge	5365	static struct i915_power_well i9xx_always_on_power_well[] = {
		5366	{
		5367	.name = "always-on",
		5368	.always_on = 1,
		5369	.domains = POWER_DOMAIN_MASK,
		5370	},
		5371	};
		5372
		5373	static struct i915_power_well hsw_power_wells[] = {
		5374	{
		5375	.name = "always-on",
		5376	.always_on = 1,
		5377	.domains = HSW_ALWAYS_ON_POWER_DOMAINS,
		5378	},
		5379	{
		5380	.name = "display",
		5381	.domains = POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS,
		5382	.is_enabled = hsw_power_well_enabled,
		5383	.set = hsw_set_power_well,
		5384	},
		5385	};
		5386
		5387	static struct i915_power_well bdw_power_wells[] = {
		5388	{
		5389	.name = "always-on",
		5390	.always_on = 1,
		5391	.domains = BDW_ALWAYS_ON_POWER_DOMAINS,
		5392	},
		5393	{
		5394	.name = "display",
		5395	.domains = POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS,
		5396	.is_enabled = hsw_power_well_enabled,
		5397	.set = hsw_set_power_well,
		5398	},
		5399	};
		5400
		5401	#define set_power_wells(power_domains, __power_wells) ({ \
		5402	(power_domains)->power_wells = (__power_wells); \
		5403	(power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \
		5404	})
		5405
		5406	int intel_power_domains_init(struct drm_device *dev)
4104	Serge	5407	{
		5408	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	5409	struct i915_power_domains *power_domains = &dev_priv->power_domains;
4104	Serge	5410
4560	Serge	5411	mutex_init(&power_domains->lock);
4104	Serge	5412
4560	Serge	5413	/*
		5414	* The enabling order will be from lower to higher indexed wells,
		5415	* the disabling order is reversed.
		5416	*/
		5417	if (IS_HASWELL(dev)) {
		5418	set_power_wells(power_domains, hsw_power_wells);
		5419	hsw_pwr = power_domains;
		5420	} else if (IS_BROADWELL(dev)) {
		5421	set_power_wells(power_domains, bdw_power_wells);
		5422	hsw_pwr = power_domains;
		5423	} else {
		5424	set_power_wells(power_domains, i9xx_always_on_power_well);
		5425	}
4104	Serge	5426
		5427	return 0;
		5428	}
		5429
4560	Serge	5430	void intel_power_domains_remove(struct drm_device *dev)
4104	Serge	5431	{
		5432	hsw_pwr = NULL;
		5433	}
		5434
4560	Serge	5435	static void intel_power_domains_resume(struct drm_device *dev)
4104	Serge	5436	{
		5437	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	5438	struct i915_power_domains *power_domains = &dev_priv->power_domains;
		5439	struct i915_power_well *power_well;
		5440	int i;
4104	Serge	5441
4560	Serge	5442	mutex_lock(&power_domains->lock);
		5443	for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
		5444	if (power_well->set)
		5445	power_well->set(dev, power_well, power_well->count > 0);
3031	serge	5446	}
4560	Serge	5447	mutex_unlock(&power_domains->lock);
3480	Serge	5448	}
3031	serge	5449
3480	Serge	5450	/*
		5451	* Starting with Haswell, we have a "Power Down Well" that can be turned off
		5452	* when not needed anymore. We have 4 registers that can request the power well
		5453	* to be enabled, and it will only be disabled if none of the registers is
		5454	* requesting it to be enabled.
		5455	*/
4560	Serge	5456	void intel_power_domains_init_hw(struct drm_device *dev)
3480	Serge	5457	{
		5458	struct drm_i915_private *dev_priv = dev->dev_private;
		5459
4560	Serge	5460	/* For now, we need the power well to be always enabled. */
		5461	intel_display_set_init_power(dev, true);
		5462	intel_power_domains_resume(dev);
		5463
		5464	if (!(IS_HASWELL(dev) \|\| IS_BROADWELL(dev)))
3480	Serge	5465	return;
		5466
		5467	/* We're taking over the BIOS, so clear any requests made by it since
		5468	* the driver is in charge now. */
4104	Serge	5469	if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
3480	Serge	5470	I915_WRITE(HSW_PWR_WELL_BIOS, 0);
3031	serge	5471	}
		5472
4104	Serge	5473	/* Disables PC8 so we can use the GMBUS and DP AUX interrupts. */
		5474	void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
		5475	{
		5476	hsw_disable_package_c8(dev_priv);
		5477	}
		5478
		5479	void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
		5480	{
		5481	hsw_enable_package_c8(dev_priv);
		5482	}
		5483
4560	Serge	5484	void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
		5485	{
		5486	struct drm_device *dev = dev_priv->dev;
		5487	struct device *device = &dev->pdev->dev;
		5488
		5489	return;
		5490	}
		5491
		5492	void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
		5493	{
		5494	struct drm_device *dev = dev_priv->dev;
		5495	struct device *device = &dev->pdev->dev;
		5496
		5497	return;
		5498
		5499	}
		5500
		5501	void intel_init_runtime_pm(struct drm_i915_private *dev_priv)
		5502	{
		5503	struct drm_device *dev = dev_priv->dev;
		5504	struct device *device = &dev->pdev->dev;
		5505
		5506	dev_priv->pm.suspended = false;
		5507
		5508	return;
		5509
		5510	}
		5511
		5512	void intel_fini_runtime_pm(struct drm_i915_private *dev_priv)
		5513	{
		5514	struct drm_device *dev = dev_priv->dev;
		5515	struct device *device = &dev->pdev->dev;
		5516
		5517	return;
		5518
		5519	}
		5520
3031	serge	5521	/* Set up chip specific power management-related functions */
		5522	void intel_init_pm(struct drm_device *dev)
		5523	{
		5524	struct drm_i915_private *dev_priv = dev->dev_private;
		5525
4560	Serge	5526	if (HAS_FBC(dev)) {
		5527	if (INTEL_INFO(dev)->gen >= 7) {
3031	serge	5528	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
4560	Serge	5529	dev_priv->display.enable_fbc = gen7_enable_fbc;
3031	serge	5530	dev_priv->display.disable_fbc = ironlake_disable_fbc;
4560	Serge	5531	} else if (INTEL_INFO(dev)->gen >= 5) {
		5532	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
		5533	dev_priv->display.enable_fbc = ironlake_enable_fbc;
		5534	dev_priv->display.disable_fbc = ironlake_disable_fbc;
3031	serge	5535	} else if (IS_GM45(dev)) {
		5536	dev_priv->display.fbc_enabled = g4x_fbc_enabled;
		5537	dev_priv->display.enable_fbc = g4x_enable_fbc;
		5538	dev_priv->display.disable_fbc = g4x_disable_fbc;
4560	Serge	5539	} else {
3031	serge	5540	dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
		5541	dev_priv->display.enable_fbc = i8xx_enable_fbc;
		5542	dev_priv->display.disable_fbc = i8xx_disable_fbc;
4560	Serge	5543
		5544	/* This value was pulled out of someone's hat */
		5545	I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT);
3031	serge	5546	}
		5547	}
		5548
		5549	/* For cxsr */
		5550	if (IS_PINEVIEW(dev))
		5551	i915_pineview_get_mem_freq(dev);
		5552	else if (IS_GEN5(dev))
		5553	i915_ironlake_get_mem_freq(dev);
		5554
		5555	/* For FIFO watermark updates */
		5556	if (HAS_PCH_SPLIT(dev)) {
4104	Serge	5557	intel_setup_wm_latency(dev);
		5558
4560	Serge	5559	if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
		5560	dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) \|\|
		5561	(!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
		5562	dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
		5563	dev_priv->display.update_wm = ilk_update_wm;
		5564	dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
3031	serge	5565	} else {
		5566	DRM_DEBUG_KMS("Failed to read display plane latency. "
		5567	"Disable CxSR\n");
		5568	}
4560	Serge	5569
		5570	if (IS_GEN5(dev))
		5571	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		5572	else if (IS_GEN6(dev))
3031	serge	5573	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
4560	Serge	5574	else if (IS_IVYBRIDGE(dev))
3031	serge	5575	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
4560	Serge	5576	else if (IS_HASWELL(dev))
3031	serge	5577	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
4560	Serge	5578	else if (INTEL_INFO(dev)->gen == 8)
		5579	dev_priv->display.init_clock_gating = gen8_init_clock_gating;
3031	serge	5580	} else if (IS_VALLEYVIEW(dev)) {
		5581	dev_priv->display.update_wm = valleyview_update_wm;
		5582	dev_priv->display.init_clock_gating =
		5583	valleyview_init_clock_gating;
		5584	} else if (IS_PINEVIEW(dev)) {
		5585	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		5586	dev_priv->is_ddr3,
		5587	dev_priv->fsb_freq,
		5588	dev_priv->mem_freq)) {
		5589	DRM_INFO("failed to find known CxSR latency "
		5590	"(found ddr%s fsb freq %d, mem freq %d), "
		5591	"disabling CxSR\n",
		5592	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		5593	dev_priv->fsb_freq, dev_priv->mem_freq);
		5594	/* Disable CxSR and never update its watermark again */
		5595	pineview_disable_cxsr(dev);
		5596	dev_priv->display.update_wm = NULL;
		5597	} else
		5598	dev_priv->display.update_wm = pineview_update_wm;
		5599	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		5600	} else if (IS_G4X(dev)) {
		5601	dev_priv->display.update_wm = g4x_update_wm;
		5602	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		5603	} else if (IS_GEN4(dev)) {
		5604	dev_priv->display.update_wm = i965_update_wm;
		5605	if (IS_CRESTLINE(dev))
		5606	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		5607	else if (IS_BROADWATER(dev))
		5608	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		5609	} else if (IS_GEN3(dev)) {
		5610	dev_priv->display.update_wm = i9xx_update_wm;
		5611	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		5612	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
4560	Serge	5613	} else if (IS_GEN2(dev)) {
		5614	if (INTEL_INFO(dev)->num_pipes == 1) {
		5615	dev_priv->display.update_wm = i845_update_wm;
		5616	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		5617	} else {
		5618	dev_priv->display.update_wm = i9xx_update_wm;
3031	serge	5619	dev_priv->display.get_fifo_size = i830_get_fifo_size;
4560	Serge	5620	}
		5621
		5622	if (IS_I85X(dev) \|\| IS_I865G(dev))
3031	serge	5623	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
4560	Serge	5624	else
		5625	dev_priv->display.init_clock_gating = i830_init_clock_gating;
3031	serge	5626	} else {
4560	Serge	5627	DRM_ERROR("unexpected fall-through in intel_init_pm\n");
3031	serge	5628	}
		5629	}
		5630
3243	Serge	5631	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u8 mbox, u32 val)
		5632	{
		5633	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	5634
3243	Serge	5635	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		5636	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		5637	return -EAGAIN;
		5638	}
3031	serge	5639
3243	Serge	5640	I915_WRITE(GEN6_PCODE_DATA, *val);
		5641	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		5642
		5643	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		5644	500)) {
		5645	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		5646	return -ETIMEDOUT;
3031	serge	5647	}
3243	Serge	5648
		5649	*val = I915_READ(GEN6_PCODE_DATA);
		5650	I915_WRITE(GEN6_PCODE_DATA, 0);
		5651
		5652	return 0;
		5653	}
		5654
		5655	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
		5656	{
		5657	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		5658
		5659	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		5660	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		5661	return -EAGAIN;
3031	serge	5662	}
3243	Serge	5663
		5664	I915_WRITE(GEN6_PCODE_DATA, val);
		5665	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		5666
		5667	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		5668	500)) {
		5669	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		5670	return -ETIMEDOUT;
3031	serge	5671	}
3243	Serge	5672
		5673	I915_WRITE(GEN6_PCODE_DATA, 0);
		5674
		5675	return 0;
3031	serge	5676	}
3746	Serge	5677
4560	Serge	5678	int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val)
3746	Serge	5679	{
4560	Serge	5680	int div;
3746	Serge	5681
4560	Serge	5682	/* 4 x czclk */
		5683	switch (dev_priv->mem_freq) {
4104	Serge	5684	case 800:
4560	Serge	5685	div = 10;
4104	Serge	5686	break;
		5687	case 1066:
4560	Serge	5688	div = 12;
4104	Serge	5689	break;
		5690	case 1333:
4560	Serge	5691	div = 16;
4104	Serge	5692	break;
		5693	default:
		5694	return -1;
		5695	}
3746	Serge	5696
4560	Serge	5697	return DIV_ROUND_CLOSEST(dev_priv->mem_freq * (val + 6 - 0xbd), 4 * div);
4104	Serge	5698	}
3746	Serge	5699
4560	Serge	5700	int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val)
4104	Serge	5701	{
4560	Serge	5702	int mul;
3746	Serge	5703
4560	Serge	5704	/* 4 x czclk */
		5705	switch (dev_priv->mem_freq) {
4104	Serge	5706	case 800:
4560	Serge	5707	mul = 10;
4104	Serge	5708	break;
		5709	case 1066:
4560	Serge	5710	mul = 12;
4104	Serge	5711	break;
		5712	case 1333:
4560	Serge	5713	mul = 16;
4104	Serge	5714	break;
		5715	default:
		5716	return -1;
3746	Serge	5717	}
		5718
4560	Serge	5719	return DIV_ROUND_CLOSEST(4 * mul * val, dev_priv->mem_freq) + 0xbd - 6;
3746	Serge	5720	}
		5721
4560	Serge	5722	void intel_pm_setup(struct drm_device *dev)
3746	Serge	5723	{
4104	Serge	5724	struct drm_i915_private *dev_priv = dev->dev_private;
		5725
4560	Serge	5726	mutex_init(&dev_priv->rps.hw_lock);
		5727
		5728	mutex_init(&dev_priv->pc8.lock);
		5729	dev_priv->pc8.requirements_met = false;
		5730	dev_priv->pc8.gpu_idle = false;
		5731	dev_priv->pc8.irqs_disabled = false;
		5732	dev_priv->pc8.enabled = false;
		5733	dev_priv->pc8.disable_count = 2; /* requirements_met + gpu_idle */
		5734	INIT_DELAYED_WORK(&dev_priv->pc8.enable_work, hsw_enable_pc8_work);
4104	Serge	5735	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		5736	intel_gen6_powersave_work);
3746	Serge	5737	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 4560