WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	#include
		32	//#include "../../../platform/x86/intel_ips.h"
		33	#include
		34
4560	Serge	35	#include
		36
3031	serge	37	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		38
		39	#define assert_spin_locked(x)
		40
		41	void getrawmonotonic(struct timespec *ts);
		42
4560	Serge	43	static inline void outb(u8 v, u16 port)
		44	{
		45	asm volatile("outb %0,%1" : : "a" (v), "dN" (port));
		46	}
		47	static inline u8 inb(u16 port)
		48	{
		49	u8 v;
		50	asm volatile("inb %1,%0" : "=a" (v) : "dN" (port));
		51	return v;
		52	}
3031	serge	53
5060	serge	54	union ktime {
		55	s64 tv64;
		56	};
3031	serge	57
5060	serge	58	typedef union ktime ktime_t; /* Kill this */
		59
		60	#define ktime_to_ns(kt) ((kt).tv64)
		61
		62	static inline u64 ktime_get_raw_ns(void)
		63	{
		64	return 0; //ktime_to_ns(ktime_get_raw());
		65	}
4560	Serge	66	/**
		67	* RC6 is a special power stage which allows the GPU to enter an very
		68	* low-voltage mode when idle, using down to 0V while at this stage. This
		69	* stage is entered automatically when the GPU is idle when RC6 support is
		70	* enabled, and as soon as new workload arises GPU wakes up automatically as well.
		71	*
		72	* There are different RC6 modes available in Intel GPU, which differentiate
		73	* among each other with the latency required to enter and leave RC6 and
		74	* voltage consumed by the GPU in different states.
		75	*
		76	* The combination of the following flags define which states GPU is allowed
		77	* to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
		78	* RC6pp is deepest RC6. Their support by hardware varies according to the
		79	* GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
		80	* which brings the most power savings; deeper states save more power, but
		81	* require higher latency to switch to and wake up.
		82	*/
		83	#define INTEL_RC6_ENABLE (1<<0)
		84	#define INTEL_RC6p_ENABLE (1<<1)
		85	#define INTEL_RC6pp_ENABLE (1<<2)
		86
3031	serge	87	/* FBC, or Frame Buffer Compression, is a technique employed to compress the
		88	* framebuffer contents in-memory, aiming at reducing the required bandwidth
		89	* during in-memory transfers and, therefore, reduce the power packet.
		90	*
		91	* The benefits of FBC are mostly visible with solid backgrounds and
		92	* variation-less patterns.
		93	*
		94	* FBC-related functionality can be enabled by the means of the
		95	* i915.i915_enable_fbc parameter
		96	*/
		97
		98	static void i8xx_disable_fbc(struct drm_device *dev)
		99	{
		100	struct drm_i915_private *dev_priv = dev->dev_private;
		101	u32 fbc_ctl;
		102
		103	/* Disable compression */
		104	fbc_ctl = I915_READ(FBC_CONTROL);
		105	if ((fbc_ctl & FBC_CTL_EN) == 0)
		106	return;
		107
		108	fbc_ctl &= ~FBC_CTL_EN;
		109	I915_WRITE(FBC_CONTROL, fbc_ctl);
		110
		111	/* Wait for compressing bit to clear */
		112	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
		113	DRM_DEBUG_KMS("FBC idle timed out\n");
		114	return;
		115	}
		116
		117	DRM_DEBUG_KMS("disabled FBC\n");
		118	}
		119
4560	Serge	120	static void i8xx_enable_fbc(struct drm_crtc *crtc)
3031	serge	121	{
		122	struct drm_device *dev = crtc->dev;
		123	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	124	struct drm_framebuffer *fb = crtc->primary->fb;
		125	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
3031	serge	126	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		127	int cfb_pitch;
5060	serge	128	int i;
4560	Serge	129	u32 fbc_ctl;
3031	serge	130
4104	Serge	131	cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE;
3031	serge	132	if (fb->pitches[0] < cfb_pitch)
		133	cfb_pitch = fb->pitches[0];
		134
4560	Serge	135	/* FBC_CTL wants 32B or 64B units */
		136	if (IS_GEN2(dev))
		137	cfb_pitch = (cfb_pitch / 32) - 1;
		138	else
3031	serge	139	cfb_pitch = (cfb_pitch / 64) - 1;
		140
		141	/* Clear old tags */
		142	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
		143	I915_WRITE(FBC_TAG + (i * 4), 0);
		144
4560	Serge	145	if (IS_GEN4(dev)) {
		146	u32 fbc_ctl2;
		147
3031	serge	148	/* Set it up... */
		149	fbc_ctl2 = FBC_CTL_FENCE_DBL \| FBC_CTL_IDLE_IMM \| FBC_CTL_CPU_FENCE;
5060	serge	150	fbc_ctl2 \|= FBC_CTL_PLANE(intel_crtc->plane);
3031	serge	151	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
		152	I915_WRITE(FBC_FENCE_OFF, crtc->y);
4560	Serge	153	}
3031	serge	154
		155	/* enable it... */
4560	Serge	156	fbc_ctl = I915_READ(FBC_CONTROL);
		157	fbc_ctl &= 0x3fff << FBC_CTL_INTERVAL_SHIFT;
		158	fbc_ctl \|= FBC_CTL_EN \| FBC_CTL_PERIODIC;
3031	serge	159	if (IS_I945GM(dev))
		160	fbc_ctl \|= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
		161	fbc_ctl \|= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
		162	fbc_ctl \|= obj->fence_reg;
		163	I915_WRITE(FBC_CONTROL, fbc_ctl);
		164
5060	serge	165	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n",
4104	Serge	166	cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
3031	serge	167	}
		168
		169	static bool i8xx_fbc_enabled(struct drm_device *dev)
		170	{
		171	struct drm_i915_private *dev_priv = dev->dev_private;
		172
		173	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
		174	}
		175
4560	Serge	176	static void g4x_enable_fbc(struct drm_crtc *crtc)
3031	serge	177	{
		178	struct drm_device *dev = crtc->dev;
		179	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	180	struct drm_framebuffer *fb = crtc->primary->fb;
		181	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
3031	serge	182	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		183	u32 dpfc_ctl;
		184
5060	serge	185	dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) \| DPFC_SR_EN;
		186	if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
		187	dpfc_ctl \|= DPFC_CTL_LIMIT_2X;
		188	else
		189	dpfc_ctl \|= DPFC_CTL_LIMIT_1X;
3031	serge	190	dpfc_ctl \|= DPFC_CTL_FENCE_EN \| obj->fence_reg;
		191
		192	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
		193
		194	/* enable it... */
5060	serge	195	I915_WRITE(DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
3031	serge	196
4104	Serge	197	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
3031	serge	198	}
		199
		200	static void g4x_disable_fbc(struct drm_device *dev)
		201	{
		202	struct drm_i915_private *dev_priv = dev->dev_private;
		203	u32 dpfc_ctl;
		204
		205	/* Disable compression */
		206	dpfc_ctl = I915_READ(DPFC_CONTROL);
		207	if (dpfc_ctl & DPFC_CTL_EN) {
		208	dpfc_ctl &= ~DPFC_CTL_EN;
		209	I915_WRITE(DPFC_CONTROL, dpfc_ctl);
		210
		211	DRM_DEBUG_KMS("disabled FBC\n");
		212	}
		213	}
		214
		215	static bool g4x_fbc_enabled(struct drm_device *dev)
		216	{
		217	struct drm_i915_private *dev_priv = dev->dev_private;
		218
		219	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
		220	}
		221
		222	static void sandybridge_blit_fbc_update(struct drm_device *dev)
		223	{
		224	struct drm_i915_private *dev_priv = dev->dev_private;
		225	u32 blt_ecoskpd;
		226
		227	/* Make sure blitter notifies FBC of writes */
4560	Serge	228
		229	/* Blitter is part of Media powerwell on VLV. No impact of
		230	* his param in other platforms for now */
		231	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_MEDIA);
		232
3031	serge	233	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
		234	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY <<
		235	GEN6_BLITTER_LOCK_SHIFT;
		236	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		237	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY;
		238	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		239	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
		240	GEN6_BLITTER_LOCK_SHIFT);
		241	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		242	POSTING_READ(GEN6_BLITTER_ECOSKPD);
4560	Serge	243
		244	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_MEDIA);
3031	serge	245	}
		246
4560	Serge	247	static void ironlake_enable_fbc(struct drm_crtc *crtc)
3031	serge	248	{
		249	struct drm_device *dev = crtc->dev;
		250	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	251	struct drm_framebuffer *fb = crtc->primary->fb;
		252	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
3031	serge	253	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		254	u32 dpfc_ctl;
		255
5060	serge	256	dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane);
		257	if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
		258	dev_priv->fbc.threshold++;
		259
		260	switch (dev_priv->fbc.threshold) {
		261	case 4:
		262	case 3:
		263	dpfc_ctl \|= DPFC_CTL_LIMIT_4X;
		264	break;
		265	case 2:
		266	dpfc_ctl \|= DPFC_CTL_LIMIT_2X;
		267	break;
		268	case 1:
		269	dpfc_ctl \|= DPFC_CTL_LIMIT_1X;
		270	break;
		271	}
4560	Serge	272	dpfc_ctl \|= DPFC_CTL_FENCE_EN;
		273	if (IS_GEN5(dev))
		274	dpfc_ctl \|= obj->fence_reg;
3031	serge	275
		276	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
4104	Serge	277	I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) \| ILK_FBC_RT_VALID);
3031	serge	278	/* enable it... */
		279	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		280
		281	if (IS_GEN6(dev)) {
		282	I915_WRITE(SNB_DPFC_CTL_SA,
		283	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		284	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		285	sandybridge_blit_fbc_update(dev);
		286	}
		287
4104	Serge	288	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
3031	serge	289	}
		290
		291	static void ironlake_disable_fbc(struct drm_device *dev)
		292	{
		293	struct drm_i915_private *dev_priv = dev->dev_private;
		294	u32 dpfc_ctl;
		295
		296	/* Disable compression */
		297	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		298	if (dpfc_ctl & DPFC_CTL_EN) {
		299	dpfc_ctl &= ~DPFC_CTL_EN;
		300	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
		301
		302	DRM_DEBUG_KMS("disabled FBC\n");
		303	}
		304	}
		305
		306	static bool ironlake_fbc_enabled(struct drm_device *dev)
		307	{
		308	struct drm_i915_private *dev_priv = dev->dev_private;
		309
		310	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
		311	}
		312
4560	Serge	313	static void gen7_enable_fbc(struct drm_crtc *crtc)
4104	Serge	314	{
		315	struct drm_device *dev = crtc->dev;
		316	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	317	struct drm_framebuffer *fb = crtc->primary->fb;
		318	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
4104	Serge	319	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5060	serge	320	u32 dpfc_ctl;
4104	Serge	321
5060	serge	322	dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane);
		323	if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
		324	dev_priv->fbc.threshold++;
4104	Serge	325
5060	serge	326	switch (dev_priv->fbc.threshold) {
		327	case 4:
		328	case 3:
		329	dpfc_ctl \|= DPFC_CTL_LIMIT_4X;
		330	break;
		331	case 2:
		332	dpfc_ctl \|= DPFC_CTL_LIMIT_2X;
		333	break;
		334	case 1:
		335	dpfc_ctl \|= DPFC_CTL_LIMIT_1X;
		336	break;
		337	}
4104	Serge	338
5060	serge	339	dpfc_ctl \|= IVB_DPFC_CTL_FENCE_EN;
		340
		341	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		342
4104	Serge	343	if (IS_IVYBRIDGE(dev)) {
		344	/* WaFbcAsynchFlipDisableFbcQueue:ivb */
5060	serge	345	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		346	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		347	ILK_FBCQ_DIS);
4104	Serge	348	} else {
5060	serge	349	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
		350	I915_WRITE(CHICKEN_PIPESL_1(intel_crtc->pipe),
		351	I915_READ(CHICKEN_PIPESL_1(intel_crtc->pipe)) \|
		352	HSW_FBCQ_DIS);
4104	Serge	353	}
		354
		355	I915_WRITE(SNB_DPFC_CTL_SA,
		356	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		357	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		358
		359	sandybridge_blit_fbc_update(dev);
		360
4560	Serge	361	DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
4104	Serge	362	}
		363
3031	serge	364	bool intel_fbc_enabled(struct drm_device *dev)
		365	{
		366	struct drm_i915_private *dev_priv = dev->dev_private;
		367
		368	if (!dev_priv->display.fbc_enabled)
		369	return false;
		370
		371	return dev_priv->display.fbc_enabled(dev);
		372	}
		373
		374	static void intel_fbc_work_fn(struct work_struct *__work)
		375	{
		376	struct intel_fbc_work *work =
		377	container_of(to_delayed_work(__work),
		378	struct intel_fbc_work, work);
		379	struct drm_device *dev = work->crtc->dev;
		380	struct drm_i915_private *dev_priv = dev->dev_private;
		381
		382	mutex_lock(&dev->struct_mutex);
4104	Serge	383	if (work == dev_priv->fbc.fbc_work) {
3031	serge	384	/* Double check that we haven't switched fb without cancelling
		385	* the prior work.
		386	*/
5060	serge	387	if (work->crtc->primary->fb == work->fb) {
4560	Serge	388	dev_priv->display.enable_fbc(work->crtc);
3031	serge	389
4104	Serge	390	dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane;
5060	serge	391	dev_priv->fbc.fb_id = work->crtc->primary->fb->base.id;
4104	Serge	392	dev_priv->fbc.y = work->crtc->y;
3031	serge	393	}
		394
4104	Serge	395	dev_priv->fbc.fbc_work = NULL;
3031	serge	396	}
		397	mutex_unlock(&dev->struct_mutex);
		398
		399	kfree(work);
		400	}
		401
		402	static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
		403	{
4104	Serge	404	if (dev_priv->fbc.fbc_work == NULL)
3031	serge	405	return;
		406
		407	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
		408
		409	/* Synchronisation is provided by struct_mutex and checking of
4104	Serge	410	* dev_priv->fbc.fbc_work, so we can perform the cancellation
3031	serge	411	* entirely asynchronously.
		412	*/
4293	Serge	413	if (cancel_delayed_work(&dev_priv->fbc.fbc_work->work))
3031	serge	414	/* tasklet was killed before being run, clean up */
4293	Serge	415	kfree(dev_priv->fbc.fbc_work);
3031	serge	416
		417	/* Mark the work as no longer wanted so that if it does
		418	* wake-up (because the work was already running and waiting
		419	* for our mutex), it will discover that is no longer
		420	* necessary to run.
		421	*/
4104	Serge	422	dev_priv->fbc.fbc_work = NULL;
3031	serge	423	}
		424
4560	Serge	425	static void intel_enable_fbc(struct drm_crtc *crtc)
3031	serge	426	{
		427	struct intel_fbc_work *work;
		428	struct drm_device *dev = crtc->dev;
		429	struct drm_i915_private *dev_priv = dev->dev_private;
		430
3482	Serge	431	if (!dev_priv->display.enable_fbc)
3031	serge	432	return;
3482	Serge	433
3031	serge	434	intel_cancel_fbc_work(dev_priv);
		435
4560	Serge	436	work = kzalloc(sizeof(*work), GFP_KERNEL);
3031	serge	437	if (work == NULL) {
4104	Serge	438	DRM_ERROR("Failed to allocate FBC work structure\n");
4560	Serge	439	dev_priv->display.enable_fbc(crtc);
3031	serge	440	return;
		441	}
		442
		443	work->crtc = crtc;
5060	serge	444	work->fb = crtc->primary->fb;
3031	serge	445	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
		446
4104	Serge	447	dev_priv->fbc.fbc_work = work;
3031	serge	448
		449	/* Delay the actual enabling to let pageflipping cease and the
		450	* display to settle before starting the compression. Note that
		451	* this delay also serves a second purpose: it allows for a
		452	* vblank to pass after disabling the FBC before we attempt
		453	* to modify the control registers.
		454	*
		455	* A more complicated solution would involve tracking vblanks
		456	* following the termination of the page-flipping sequence
		457	* and indeed performing the enable as a co-routine and not
		458	* waiting synchronously upon the vblank.
4104	Serge	459	*
		460	* WaFbcWaitForVBlankBeforeEnable:ilk,snb
3031	serge	461	*/
		462	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
		463	}
		464
		465	void intel_disable_fbc(struct drm_device *dev)
		466	{
		467	struct drm_i915_private *dev_priv = dev->dev_private;
		468
3482	Serge	469	intel_cancel_fbc_work(dev_priv);
3031	serge	470
3482	Serge	471	if (!dev_priv->display.disable_fbc)
		472	return;
3031	serge	473
3482	Serge	474	dev_priv->display.disable_fbc(dev);
4104	Serge	475	dev_priv->fbc.plane = -1;
3031	serge	476	}
		477
4104	Serge	478	static bool set_no_fbc_reason(struct drm_i915_private *dev_priv,
		479	enum no_fbc_reason reason)
		480	{
		481	if (dev_priv->fbc.no_fbc_reason == reason)
		482	return false;
		483
		484	dev_priv->fbc.no_fbc_reason = reason;
		485	return true;
		486	}
		487
3031	serge	488	/**
		489	* intel_update_fbc - enable/disable FBC as needed
		490	* @dev: the drm_device
		491	*
		492	* Set up the framebuffer compression hardware at mode set time. We
		493	* enable it if possible:
		494	* - plane A only (on pre-965)
		495	* - no pixel mulitply/line duplication
		496	* - no alpha buffer discard
		497	* - no dual wide
4104	Serge	498	* - framebuffer <= max_hdisplay in width, max_vdisplay in height
3031	serge	499	*
		500	* We can't assume that any compression will take place (worst case),
		501	* so the compressed buffer has to be the same size as the uncompressed
		502	* one. It also must reside (along with the line length buffer) in
		503	* stolen memory.
		504	*
		505	* We need to enable/disable FBC on a global basis.
		506	*/
		507	void intel_update_fbc(struct drm_device *dev)
		508	{
		509	struct drm_i915_private *dev_priv = dev->dev_private;
		510	struct drm_crtc crtc = NULL, tmp_crtc;
		511	struct intel_crtc *intel_crtc;
		512	struct drm_framebuffer *fb;
		513	struct drm_i915_gem_object *obj;
4560	Serge	514	const struct drm_display_mode *adjusted_mode;
		515	unsigned int max_width, max_height;
3031	serge	516
4560	Serge	517	if (!HAS_FBC(dev)) {
4104	Serge	518	set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED);
3031	serge	519	return;
4104	Serge	520	}
3031	serge	521
5060	serge	522	if (!i915.powersave) {
4104	Serge	523	if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
		524	DRM_DEBUG_KMS("fbc disabled per module param\n");
3031	serge	525	return;
4104	Serge	526	}
3031	serge	527
		528	/*
		529	* If FBC is already on, we just have to verify that we can
		530	* keep it that way...
		531	* Need to disable if:
		532	* - more than one pipe is active
		533	* - changing FBC params (stride, fence, mode)
		534	* - new fb is too large to fit in compressed buffer
		535	* - going to an unsupported config (interlace, pixel multiply, etc.)
		536	*/
5060	serge	537	for_each_crtc(dev, tmp_crtc) {
3243	Serge	538	if (intel_crtc_active(tmp_crtc) &&
4560	Serge	539	to_intel_crtc(tmp_crtc)->primary_enabled) {
3031	serge	540	if (crtc) {
4104	Serge	541	if (set_no_fbc_reason(dev_priv, FBC_MULTIPLE_PIPES))
3031	serge	542	DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
		543	goto out_disable;
		544	}
		545	crtc = tmp_crtc;
		546	}
		547	}
		548
5060	serge	549	if (!crtc \|\| crtc->primary->fb == NULL) {
4104	Serge	550	if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT))
3031	serge	551	DRM_DEBUG_KMS("no output, disabling\n");
		552	goto out_disable;
		553	}
		554
		555	intel_crtc = to_intel_crtc(crtc);
5060	serge	556	fb = crtc->primary->fb;
		557	obj = intel_fb_obj(fb);
4560	Serge	558	adjusted_mode = &intel_crtc->config.adjusted_mode;
3031	serge	559
5060	serge	560	if (i915.enable_fbc < 0) {
4104	Serge	561	if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT))
		562	DRM_DEBUG_KMS("disabled per chip default\n");
		563	goto out_disable;
3031	serge	564	}
5060	serge	565	if (!i915.enable_fbc) {
4104	Serge	566	if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
3031	serge	567	DRM_DEBUG_KMS("fbc disabled per module param\n");
		568	goto out_disable;
		569	}
4560	Serge	570	if ((adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) \|\|
		571	(adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)) {
4104	Serge	572	if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
3031	serge	573	DRM_DEBUG_KMS("mode incompatible with compression, "
		574	"disabling\n");
		575	goto out_disable;
		576	}
4104	Serge	577
5060	serge	578	if (INTEL_INFO(dev)->gen >= 8 \|\| IS_HASWELL(dev)) {
4560	Serge	579	max_width = 4096;
5060	serge	580	max_height = 4096;
		581	} else if (IS_G4X(dev) \|\| INTEL_INFO(dev)->gen >= 5) {
		582	max_width = 4096;
4560	Serge	583	max_height = 2048;
4104	Serge	584	} else {
4560	Serge	585	max_width = 2048;
		586	max_height = 1536;
4104	Serge	587	}
4560	Serge	588	if (intel_crtc->config.pipe_src_w > max_width \|\|
		589	intel_crtc->config.pipe_src_h > max_height) {
4104	Serge	590	if (set_no_fbc_reason(dev_priv, FBC_MODE_TOO_LARGE))
3031	serge	591	DRM_DEBUG_KMS("mode too large for compression, disabling\n");
		592	goto out_disable;
		593	}
5060	serge	594	if ((INTEL_INFO(dev)->gen < 4 \|\| HAS_DDI(dev)) &&
4560	Serge	595	intel_crtc->plane != PLANE_A) {
4104	Serge	596	if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE))
4560	Serge	597	DRM_DEBUG_KMS("plane not A, disabling compression\n");
3031	serge	598	goto out_disable;
		599	}
		600
		601	/* The use of a CPU fence is mandatory in order to detect writes
		602	* by the CPU to the scanout and trigger updates to the FBC.
		603	*/
		604	if (obj->tiling_mode != I915_TILING_X \|\|
		605	obj->fence_reg == I915_FENCE_REG_NONE) {
4104	Serge	606	if (set_no_fbc_reason(dev_priv, FBC_NOT_TILED))
3031	serge	607	DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
		608	goto out_disable;
		609	}
		610
		611	/* If the kernel debugger is active, always disable compression */
		612	if (in_dbg_master())
		613	goto out_disable;
		614
5060	serge	615	if (i915_gem_stolen_setup_compression(dev, obj->base.size,
		616	drm_format_plane_cpp(fb->pixel_format, 0))) {
4104	Serge	617	if (set_no_fbc_reason(dev_priv, FBC_STOLEN_TOO_SMALL))
3480	Serge	618	DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
		619	goto out_disable;
		620	}
		621
3031	serge	622	/* If the scanout has not changed, don't modify the FBC settings.
		623	* Note that we make the fundamental assumption that the fb->obj
		624	* cannot be unpinned (and have its GTT offset and fence revoked)
		625	* without first being decoupled from the scanout and FBC disabled.
		626	*/
4104	Serge	627	if (dev_priv->fbc.plane == intel_crtc->plane &&
		628	dev_priv->fbc.fb_id == fb->base.id &&
		629	dev_priv->fbc.y == crtc->y)
3031	serge	630	return;
		631
		632	if (intel_fbc_enabled(dev)) {
		633	/* We update FBC along two paths, after changing fb/crtc
		634	* configuration (modeswitching) and after page-flipping
		635	* finishes. For the latter, we know that not only did
		636	* we disable the FBC at the start of the page-flip
		637	* sequence, but also more than one vblank has passed.
		638	*
		639	* For the former case of modeswitching, it is possible
		640	* to switch between two FBC valid configurations
		641	* instantaneously so we do need to disable the FBC
		642	* before we can modify its control registers. We also
		643	* have to wait for the next vblank for that to take
		644	* effect. However, since we delay enabling FBC we can
		645	* assume that a vblank has passed since disabling and
		646	* that we can safely alter the registers in the deferred
		647	* callback.
		648	*
		649	* In the scenario that we go from a valid to invalid
		650	* and then back to valid FBC configuration we have
		651	* no strict enforcement that a vblank occurred since
		652	* disabling the FBC. However, along all current pipe
		653	* disabling paths we do need to wait for a vblank at
		654	* some point. And we wait before enabling FBC anyway.
		655	*/
		656	DRM_DEBUG_KMS("disabling active FBC for update\n");
		657	intel_disable_fbc(dev);
		658	}
		659
4560	Serge	660	intel_enable_fbc(crtc);
4104	Serge	661	dev_priv->fbc.no_fbc_reason = FBC_OK;
3031	serge	662	return;
		663
		664	out_disable:
		665	/* Multiple disables should be harmless */
		666	if (intel_fbc_enabled(dev)) {
		667	DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
		668	intel_disable_fbc(dev);
		669	}
3480	Serge	670	i915_gem_stolen_cleanup_compression(dev);
3031	serge	671	}
		672
		673	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		674	{
5060	serge	675	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	676	u32 tmp;
		677
		678	tmp = I915_READ(CLKCFG);
		679
		680	switch (tmp & CLKCFG_FSB_MASK) {
		681	case CLKCFG_FSB_533:
		682	dev_priv->fsb_freq = 533; /* 1334 /
		683	break;
		684	case CLKCFG_FSB_800:
		685	dev_priv->fsb_freq = 800; /* 2004 /
		686	break;
		687	case CLKCFG_FSB_667:
		688	dev_priv->fsb_freq = 667; /* 1674 /
		689	break;
		690	case CLKCFG_FSB_400:
		691	dev_priv->fsb_freq = 400; /* 1004 /
		692	break;
		693	}
		694
		695	switch (tmp & CLKCFG_MEM_MASK) {
		696	case CLKCFG_MEM_533:
		697	dev_priv->mem_freq = 533;
		698	break;
		699	case CLKCFG_MEM_667:
		700	dev_priv->mem_freq = 667;
		701	break;
		702	case CLKCFG_MEM_800:
		703	dev_priv->mem_freq = 800;
		704	break;
		705	}
		706
		707	/* detect pineview DDR3 setting */
		708	tmp = I915_READ(CSHRDDR3CTL);
		709	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		710	}
		711
		712	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		713	{
5060	serge	714	struct drm_i915_private *dev_priv = dev->dev_private;
3031	serge	715	u16 ddrpll, csipll;
		716
		717	ddrpll = I915_READ16(DDRMPLL1);
		718	csipll = I915_READ16(CSIPLL0);
		719
		720	switch (ddrpll & 0xff) {
		721	case 0xc:
		722	dev_priv->mem_freq = 800;
		723	break;
		724	case 0x10:
		725	dev_priv->mem_freq = 1066;
		726	break;
		727	case 0x14:
		728	dev_priv->mem_freq = 1333;
		729	break;
		730	case 0x18:
		731	dev_priv->mem_freq = 1600;
		732	break;
		733	default:
		734	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		735	ddrpll & 0xff);
		736	dev_priv->mem_freq = 0;
		737	break;
		738	}
		739
		740	dev_priv->ips.r_t = dev_priv->mem_freq;
		741
		742	switch (csipll & 0x3ff) {
		743	case 0x00c:
		744	dev_priv->fsb_freq = 3200;
		745	break;
		746	case 0x00e:
		747	dev_priv->fsb_freq = 3733;
		748	break;
		749	case 0x010:
		750	dev_priv->fsb_freq = 4266;
		751	break;
		752	case 0x012:
		753	dev_priv->fsb_freq = 4800;
		754	break;
		755	case 0x014:
		756	dev_priv->fsb_freq = 5333;
		757	break;
		758	case 0x016:
		759	dev_priv->fsb_freq = 5866;
		760	break;
		761	case 0x018:
		762	dev_priv->fsb_freq = 6400;
		763	break;
		764	default:
		765	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		766	csipll & 0x3ff);
		767	dev_priv->fsb_freq = 0;
		768	break;
		769	}
		770
		771	if (dev_priv->fsb_freq == 3200) {
		772	dev_priv->ips.c_m = 0;
		773	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		774	dev_priv->ips.c_m = 1;
		775	} else {
		776	dev_priv->ips.c_m = 2;
		777	}
		778	}
		779
		780	static const struct cxsr_latency cxsr_latency_table[] = {
		781	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		782	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		783	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		784	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		785	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		786
		787	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		788	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		789	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		790	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		791	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		792
		793	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		794	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		795	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		796	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		797	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		798
		799	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		800	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		801	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		802	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		803	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		804
		805	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		806	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		807	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		808	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		809	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		810
		811	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		812	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		813	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		814	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		815	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		816	};
		817
		818	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		819	int is_ddr3,
		820	int fsb,
		821	int mem)
		822	{
		823	const struct cxsr_latency *latency;
		824	int i;
		825
		826	if (fsb == 0 \|\| mem == 0)
		827	return NULL;
		828
		829	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		830	latency = &cxsr_latency_table[i];
		831	if (is_desktop == latency->is_desktop &&
		832	is_ddr3 == latency->is_ddr3 &&
		833	fsb == latency->fsb_freq && mem == latency->mem_freq)
		834	return latency;
		835	}
		836
		837	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		838
		839	return NULL;
		840	}
		841
5060	serge	842	void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
3031	serge	843	{
5060	serge	844	struct drm_device *dev = dev_priv->dev;
		845	u32 val;
3031	serge	846
5060	serge	847	if (IS_VALLEYVIEW(dev)) {
		848	I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
		849	} else if (IS_G4X(dev) \|\| IS_CRESTLINE(dev)) {
		850	I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
		851	} else if (IS_PINEVIEW(dev)) {
		852	val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
		853	val \|= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
		854	I915_WRITE(DSPFW3, val);
		855	} else if (IS_I945G(dev) \|\| IS_I945GM(dev)) {
		856	val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
		857	_MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
		858	I915_WRITE(FW_BLC_SELF, val);
		859	} else if (IS_I915GM(dev)) {
		860	val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
		861	_MASKED_BIT_DISABLE(INSTPM_SELF_EN);
		862	I915_WRITE(INSTPM, val);
		863	} else {
		864	return;
		865	}
		866
		867	DRM_DEBUG_KMS("memory self-refresh is %s\n",
		868	enable ? "enabled" : "disabled");
3031	serge	869	}
		870
		871	/*
		872	* Latency for FIFO fetches is dependent on several factors:
		873	* - memory configuration (speed, channels)
		874	* - chipset
		875	* - current MCH state
		876	* It can be fairly high in some situations, so here we assume a fairly
		877	* pessimal value. It's a tradeoff between extra memory fetches (if we
		878	* set this value too high, the FIFO will fetch frequently to stay full)
		879	* and power consumption (set it too low to save power and we might see
		880	* FIFO underruns and display "flicker").
		881	*
		882	* A value of 5us seems to be a good balance; safe for very low end
		883	* platforms but not overly aggressive on lower latency configs.
		884	*/
		885	static const int latency_ns = 5000;
		886
		887	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		888	{
		889	struct drm_i915_private *dev_priv = dev->dev_private;
		890	uint32_t dsparb = I915_READ(DSPARB);
		891	int size;
		892
		893	size = dsparb & 0x7f;
		894	if (plane)
		895	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		896
		897	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		898	plane ? "B" : "A", size);
		899
		900	return size;
		901	}
		902
4560	Serge	903	static int i830_get_fifo_size(struct drm_device *dev, int plane)
3031	serge	904	{
		905	struct drm_i915_private *dev_priv = dev->dev_private;
		906	uint32_t dsparb = I915_READ(DSPARB);
		907	int size;
		908
		909	size = dsparb & 0x1ff;
		910	if (plane)
		911	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		912	size >>= 1; /* Convert to cachelines */
		913
		914	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		915	plane ? "B" : "A", size);
		916
		917	return size;
		918	}
		919
		920	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		921	{
		922	struct drm_i915_private *dev_priv = dev->dev_private;
		923	uint32_t dsparb = I915_READ(DSPARB);
		924	int size;
		925
		926	size = dsparb & 0x7f;
		927	size >>= 2; /* Convert to cachelines */
		928
		929	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		930	plane ? "B" : "A",
		931	size);
		932
		933	return size;
		934	}
		935
		936	/* Pineview has different values for various configs */
		937	static const struct intel_watermark_params pineview_display_wm = {
5060	serge	938	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		939	.max_wm = PINEVIEW_MAX_WM,
		940	.default_wm = PINEVIEW_DFT_WM,
		941	.guard_size = PINEVIEW_GUARD_WM,
		942	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	943	};
		944	static const struct intel_watermark_params pineview_display_hplloff_wm = {
5060	serge	945	.fifo_size = PINEVIEW_DISPLAY_FIFO,
		946	.max_wm = PINEVIEW_MAX_WM,
		947	.default_wm = PINEVIEW_DFT_HPLLOFF_WM,
		948	.guard_size = PINEVIEW_GUARD_WM,
		949	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	950	};
		951	static const struct intel_watermark_params pineview_cursor_wm = {
5060	serge	952	.fifo_size = PINEVIEW_CURSOR_FIFO,
		953	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		954	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		955	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		956	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	957	};
		958	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
5060	serge	959	.fifo_size = PINEVIEW_CURSOR_FIFO,
		960	.max_wm = PINEVIEW_CURSOR_MAX_WM,
		961	.default_wm = PINEVIEW_CURSOR_DFT_WM,
		962	.guard_size = PINEVIEW_CURSOR_GUARD_WM,
		963	.cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
3031	serge	964	};
		965	static const struct intel_watermark_params g4x_wm_info = {
5060	serge	966	.fifo_size = G4X_FIFO_SIZE,
		967	.max_wm = G4X_MAX_WM,
		968	.default_wm = G4X_MAX_WM,
		969	.guard_size = 2,
		970	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	971	};
		972	static const struct intel_watermark_params g4x_cursor_wm_info = {
5060	serge	973	.fifo_size = I965_CURSOR_FIFO,
		974	.max_wm = I965_CURSOR_MAX_WM,
		975	.default_wm = I965_CURSOR_DFT_WM,
		976	.guard_size = 2,
		977	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	978	};
		979	static const struct intel_watermark_params valleyview_wm_info = {
5060	serge	980	.fifo_size = VALLEYVIEW_FIFO_SIZE,
		981	.max_wm = VALLEYVIEW_MAX_WM,
		982	.default_wm = VALLEYVIEW_MAX_WM,
		983	.guard_size = 2,
		984	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	985	};
		986	static const struct intel_watermark_params valleyview_cursor_wm_info = {
5060	serge	987	.fifo_size = I965_CURSOR_FIFO,
		988	.max_wm = VALLEYVIEW_CURSOR_MAX_WM,
		989	.default_wm = I965_CURSOR_DFT_WM,
		990	.guard_size = 2,
		991	.cacheline_size = G4X_FIFO_LINE_SIZE,
3031	serge	992	};
		993	static const struct intel_watermark_params i965_cursor_wm_info = {
5060	serge	994	.fifo_size = I965_CURSOR_FIFO,
		995	.max_wm = I965_CURSOR_MAX_WM,
		996	.default_wm = I965_CURSOR_DFT_WM,
		997	.guard_size = 2,
		998	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	999	};
		1000	static const struct intel_watermark_params i945_wm_info = {
5060	serge	1001	.fifo_size = I945_FIFO_SIZE,
		1002	.max_wm = I915_MAX_WM,
		1003	.default_wm = 1,
		1004	.guard_size = 2,
		1005	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	1006	};
		1007	static const struct intel_watermark_params i915_wm_info = {
5060	serge	1008	.fifo_size = I915_FIFO_SIZE,
		1009	.max_wm = I915_MAX_WM,
		1010	.default_wm = 1,
		1011	.guard_size = 2,
		1012	.cacheline_size = I915_FIFO_LINE_SIZE,
3031	serge	1013	};
4560	Serge	1014	static const struct intel_watermark_params i830_wm_info = {
5060	serge	1015	.fifo_size = I855GM_FIFO_SIZE,
		1016	.max_wm = I915_MAX_WM,
		1017	.default_wm = 1,
		1018	.guard_size = 2,
		1019	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	1020	};
4560	Serge	1021	static const struct intel_watermark_params i845_wm_info = {
5060	serge	1022	.fifo_size = I830_FIFO_SIZE,
		1023	.max_wm = I915_MAX_WM,
		1024	.default_wm = 1,
		1025	.guard_size = 2,
		1026	.cacheline_size = I830_FIFO_LINE_SIZE,
3031	serge	1027	};
		1028
		1029	/**
		1030	* intel_calculate_wm - calculate watermark level
		1031	* @clock_in_khz: pixel clock
		1032	* @wm: chip FIFO params
		1033	* @pixel_size: display pixel size
		1034	* @latency_ns: memory latency for the platform
		1035	*
		1036	* Calculate the watermark level (the level at which the display plane will
		1037	* start fetching from memory again). Each chip has a different display
		1038	* FIFO size and allocation, so the caller needs to figure that out and pass
		1039	* in the correct intel_watermark_params structure.
		1040	*
		1041	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		1042	* on the pixel size. When it reaches the watermark level, it'll start
		1043	* fetching FIFO line sized based chunks from memory until the FIFO fills
		1044	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		1045	* will occur, and a display engine hang could result.
		1046	*/
		1047	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		1048	const struct intel_watermark_params *wm,
		1049	int fifo_size,
		1050	int pixel_size,
		1051	unsigned long latency_ns)
		1052	{
		1053	long entries_required, wm_size;
		1054
		1055	/*
		1056	* Note: we need to make sure we don't overflow for various clock &
		1057	* latency values.
		1058	* clocks go from a few thousand to several hundred thousand.
		1059	* latency is usually a few thousand
		1060	*/
		1061	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		1062	1000;
		1063	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		1064
		1065	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		1066
		1067	wm_size = fifo_size - (entries_required + wm->guard_size);
		1068
		1069	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		1070
		1071	/* Don't promote wm_size to unsigned... */
		1072	if (wm_size > (long)wm->max_wm)
		1073	wm_size = wm->max_wm;
		1074	if (wm_size <= 0)
		1075	wm_size = wm->default_wm;
		1076	return wm_size;
		1077	}
		1078
		1079	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		1080	{
		1081	struct drm_crtc crtc, enabled = NULL;
		1082
5060	serge	1083	for_each_crtc(dev, crtc) {
3243	Serge	1084	if (intel_crtc_active(crtc)) {
3031	serge	1085	if (enabled)
		1086	return NULL;
		1087	enabled = crtc;
		1088	}
		1089	}
		1090
		1091	return enabled;
		1092	}
		1093
4560	Serge	1094	static void pineview_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1095	{
4560	Serge	1096	struct drm_device *dev = unused_crtc->dev;
3031	serge	1097	struct drm_i915_private *dev_priv = dev->dev_private;
		1098	struct drm_crtc *crtc;
		1099	const struct cxsr_latency *latency;
		1100	u32 reg;
		1101	unsigned long wm;
		1102
		1103	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		1104	dev_priv->fsb_freq, dev_priv->mem_freq);
		1105	if (!latency) {
		1106	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
5060	serge	1107	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1108	return;
		1109	}
		1110
		1111	crtc = single_enabled_crtc(dev);
		1112	if (crtc) {
4560	Serge	1113	const struct drm_display_mode *adjusted_mode;
5060	serge	1114	int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
4560	Serge	1115	int clock;
3031	serge	1116
4560	Serge	1117	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1118	clock = adjusted_mode->crtc_clock;
		1119
3031	serge	1120	/* Display SR */
		1121	wm = intel_calculate_wm(clock, &pineview_display_wm,
		1122	pineview_display_wm.fifo_size,
		1123	pixel_size, latency->display_sr);
		1124	reg = I915_READ(DSPFW1);
		1125	reg &= ~DSPFW_SR_MASK;
		1126	reg \|= wm << DSPFW_SR_SHIFT;
		1127	I915_WRITE(DSPFW1, reg);
		1128	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		1129
		1130	/* cursor SR */
		1131	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		1132	pineview_display_wm.fifo_size,
		1133	pixel_size, latency->cursor_sr);
		1134	reg = I915_READ(DSPFW3);
		1135	reg &= ~DSPFW_CURSOR_SR_MASK;
		1136	reg \|= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
		1137	I915_WRITE(DSPFW3, reg);
		1138
		1139	/* Display HPLL off SR */
		1140	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		1141	pineview_display_hplloff_wm.fifo_size,
		1142	pixel_size, latency->display_hpll_disable);
		1143	reg = I915_READ(DSPFW3);
		1144	reg &= ~DSPFW_HPLL_SR_MASK;
		1145	reg \|= wm & DSPFW_HPLL_SR_MASK;
		1146	I915_WRITE(DSPFW3, reg);
		1147
		1148	/* cursor HPLL off SR */
		1149	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		1150	pineview_display_hplloff_wm.fifo_size,
		1151	pixel_size, latency->cursor_hpll_disable);
		1152	reg = I915_READ(DSPFW3);
		1153	reg &= ~DSPFW_HPLL_CURSOR_MASK;
		1154	reg \|= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
		1155	I915_WRITE(DSPFW3, reg);
		1156	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		1157
5060	serge	1158	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1159	} else {
5060	serge	1160	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1161	}
		1162	}
		1163
		1164	static bool g4x_compute_wm0(struct drm_device *dev,
		1165	int plane,
		1166	const struct intel_watermark_params *display,
		1167	int display_latency_ns,
		1168	const struct intel_watermark_params *cursor,
		1169	int cursor_latency_ns,
		1170	int *plane_wm,
		1171	int *cursor_wm)
		1172	{
		1173	struct drm_crtc *crtc;
4560	Serge	1174	const struct drm_display_mode *adjusted_mode;
3031	serge	1175	int htotal, hdisplay, clock, pixel_size;
		1176	int line_time_us, line_count;
		1177	int entries, tlb_miss;
		1178
		1179	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1180	if (!intel_crtc_active(crtc)) {
3031	serge	1181	*cursor_wm = cursor->guard_size;
		1182	*plane_wm = display->guard_size;
		1183	return false;
		1184	}
		1185
4560	Serge	1186	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1187	clock = adjusted_mode->crtc_clock;
		1188	htotal = adjusted_mode->crtc_htotal;
		1189	hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
5060	serge	1190	pixel_size = crtc->primary->fb->bits_per_pixel / 8;
3031	serge	1191
		1192	/* Use the small buffer method to calculate plane watermark */
		1193	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1194	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		1195	if (tlb_miss > 0)
		1196	entries += tlb_miss;
		1197	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1198	*plane_wm = entries + display->guard_size;
		1199	if (*plane_wm > (int)display->max_wm)
		1200	*plane_wm = display->max_wm;
		1201
		1202	/* Use the large buffer method to calculate cursor watermark */
5060	serge	1203	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1204	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
5060	serge	1205	entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size;
3031	serge	1206	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		1207	if (tlb_miss > 0)
		1208	entries += tlb_miss;
		1209	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1210	*cursor_wm = entries + cursor->guard_size;
		1211	if (*cursor_wm > (int)cursor->max_wm)
		1212	*cursor_wm = (int)cursor->max_wm;
		1213
		1214	return true;
		1215	}
		1216
		1217	/*
		1218	* Check the wm result.
		1219	*
		1220	* If any calculated watermark values is larger than the maximum value that
		1221	* can be programmed into the associated watermark register, that watermark
		1222	* must be disabled.
		1223	*/
		1224	static bool g4x_check_srwm(struct drm_device *dev,
		1225	int display_wm, int cursor_wm,
		1226	const struct intel_watermark_params *display,
		1227	const struct intel_watermark_params *cursor)
		1228	{
		1229	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		1230	display_wm, cursor_wm);
		1231
		1232	if (display_wm > display->max_wm) {
		1233	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		1234	display_wm, display->max_wm);
		1235	return false;
		1236	}
		1237
		1238	if (cursor_wm > cursor->max_wm) {
		1239	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		1240	cursor_wm, cursor->max_wm);
		1241	return false;
		1242	}
		1243
		1244	if (!(display_wm \|\| cursor_wm)) {
		1245	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		1246	return false;
		1247	}
		1248
		1249	return true;
		1250	}
		1251
		1252	static bool g4x_compute_srwm(struct drm_device *dev,
		1253	int plane,
		1254	int latency_ns,
		1255	const struct intel_watermark_params *display,
		1256	const struct intel_watermark_params *cursor,
		1257	int display_wm, int cursor_wm)
		1258	{
		1259	struct drm_crtc *crtc;
4560	Serge	1260	const struct drm_display_mode *adjusted_mode;
3031	serge	1261	int hdisplay, htotal, pixel_size, clock;
		1262	unsigned long line_time_us;
		1263	int line_count, line_size;
		1264	int small, large;
		1265	int entries;
		1266
		1267	if (!latency_ns) {
		1268	display_wm = cursor_wm = 0;
		1269	return false;
		1270	}
		1271
		1272	crtc = intel_get_crtc_for_plane(dev, plane);
4560	Serge	1273	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1274	clock = adjusted_mode->crtc_clock;
		1275	htotal = adjusted_mode->crtc_htotal;
		1276	hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
5060	serge	1277	pixel_size = crtc->primary->fb->bits_per_pixel / 8;
3031	serge	1278
5060	serge	1279	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1280	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1281	line_size = hdisplay * pixel_size;
		1282
		1283	/* Use the minimum of the small and large buffer method for primary */
		1284	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1285	large = line_count * line_size;
		1286
		1287	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1288	*display_wm = entries + display->guard_size;
		1289
		1290	/* calculate the self-refresh watermark for display cursor */
5060	serge	1291	entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width;
3031	serge	1292	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1293	*cursor_wm = entries + cursor->guard_size;
		1294
		1295	return g4x_check_srwm(dev,
		1296	display_wm, cursor_wm,
		1297	display, cursor);
		1298	}
		1299
		1300	static bool vlv_compute_drain_latency(struct drm_device *dev,
		1301	int plane,
		1302	int *plane_prec_mult,
		1303	int *plane_dl,
		1304	int *cursor_prec_mult,
		1305	int *cursor_dl)
		1306	{
		1307	struct drm_crtc *crtc;
		1308	int clock, pixel_size;
		1309	int entries;
		1310
		1311	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1312	if (!intel_crtc_active(crtc))
3031	serge	1313	return false;
		1314
4560	Serge	1315	clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
5060	serge	1316	pixel_size = crtc->primary->fb->bits_per_pixel / 8; /* BPP */
3031	serge	1317
		1318	entries = (clock / 1000) * pixel_size;
5060	serge	1319	*plane_prec_mult = (entries > 128) ?
		1320	DRAIN_LATENCY_PRECISION_64 : DRAIN_LATENCY_PRECISION_32;
		1321	plane_dl = (64 (plane_prec_mult) 4) / entries;
3031	serge	1322
		1323	entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
5060	serge	1324	*cursor_prec_mult = (entries > 128) ?
		1325	DRAIN_LATENCY_PRECISION_64 : DRAIN_LATENCY_PRECISION_32;
		1326	cursor_dl = (64 (cursor_prec_mult) 4) / entries;
3031	serge	1327
		1328	return true;
		1329	}
		1330
		1331	/*
		1332	* Update drain latency registers of memory arbiter
		1333	*
		1334	* Valleyview SoC has a new memory arbiter and needs drain latency registers
		1335	* to be programmed. Each plane has a drain latency multiplier and a drain
		1336	* latency value.
		1337	*/
		1338
		1339	static void vlv_update_drain_latency(struct drm_device *dev)
		1340	{
		1341	struct drm_i915_private *dev_priv = dev->dev_private;
		1342	int planea_prec, planea_dl, planeb_prec, planeb_dl;
		1343	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
		1344	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
		1345	either 16 or 32 */
		1346
		1347	/* For plane A, Cursor A */
		1348	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
		1349	&cursor_prec_mult, &cursora_dl)) {
		1350	cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
5060	serge	1351	DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_64;
3031	serge	1352	planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
5060	serge	1353	DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_64;
3031	serge	1354
		1355	I915_WRITE(VLV_DDL1, cursora_prec \|
		1356	(cursora_dl << DDL_CURSORA_SHIFT) \|
		1357	planea_prec \| planea_dl);
		1358	}
		1359
		1360	/* For plane B, Cursor B */
		1361	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
		1362	&cursor_prec_mult, &cursorb_dl)) {
		1363	cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
5060	serge	1364	DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_64;
3031	serge	1365	planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
5060	serge	1366	DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_64;
3031	serge	1367
		1368	I915_WRITE(VLV_DDL2, cursorb_prec \|
		1369	(cursorb_dl << DDL_CURSORB_SHIFT) \|
		1370	planeb_prec \| planeb_dl);
		1371	}
		1372	}
		1373
		1374	#define single_plane_enabled(mask) is_power_of_2(mask)
		1375
4560	Serge	1376	static void valleyview_update_wm(struct drm_crtc *crtc)
3031	serge	1377	{
4560	Serge	1378	struct drm_device *dev = crtc->dev;
3031	serge	1379	static const int sr_latency_ns = 12000;
		1380	struct drm_i915_private *dev_priv = dev->dev_private;
		1381	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1382	int plane_sr, cursor_sr;
3243	Serge	1383	int ignore_plane_sr, ignore_cursor_sr;
3031	serge	1384	unsigned int enabled = 0;
5060	serge	1385	bool cxsr_enabled;
3031	serge	1386
		1387	vlv_update_drain_latency(dev);
		1388
3746	Serge	1389	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1390	&valleyview_wm_info, latency_ns,
		1391	&valleyview_cursor_wm_info, latency_ns,
		1392	&planea_wm, &cursora_wm))
3746	Serge	1393	enabled \|= 1 << PIPE_A;
3031	serge	1394
3746	Serge	1395	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1396	&valleyview_wm_info, latency_ns,
		1397	&valleyview_cursor_wm_info, latency_ns,
		1398	&planeb_wm, &cursorb_wm))
3746	Serge	1399	enabled \|= 1 << PIPE_B;
3031	serge	1400
		1401	if (single_plane_enabled(enabled) &&
		1402	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1403	sr_latency_ns,
		1404	&valleyview_wm_info,
		1405	&valleyview_cursor_wm_info,
3243	Serge	1406	&plane_sr, &ignore_cursor_sr) &&
		1407	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1408	2*sr_latency_ns,
		1409	&valleyview_wm_info,
		1410	&valleyview_cursor_wm_info,
		1411	&ignore_plane_sr, &cursor_sr)) {
5060	serge	1412	cxsr_enabled = true;
3243	Serge	1413	} else {
5060	serge	1414	cxsr_enabled = false;
		1415	intel_set_memory_cxsr(dev_priv, false);
3243	Serge	1416	plane_sr = cursor_sr = 0;
		1417	}
3031	serge	1418
		1419	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1420	planea_wm, cursora_wm,
		1421	planeb_wm, cursorb_wm,
		1422	plane_sr, cursor_sr);
		1423
		1424	I915_WRITE(DSPFW1,
		1425	(plane_sr << DSPFW_SR_SHIFT) \|
		1426	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1427	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1428	planea_wm);
		1429	I915_WRITE(DSPFW2,
3243	Serge	1430	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1431	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1432	I915_WRITE(DSPFW3,
3243	Serge	1433	(I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) \|
		1434	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
5060	serge	1435
		1436	if (cxsr_enabled)
		1437	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1438	}
		1439
4560	Serge	1440	static void g4x_update_wm(struct drm_crtc *crtc)
3031	serge	1441	{
4560	Serge	1442	struct drm_device *dev = crtc->dev;
3031	serge	1443	static const int sr_latency_ns = 12000;
		1444	struct drm_i915_private *dev_priv = dev->dev_private;
		1445	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1446	int plane_sr, cursor_sr;
		1447	unsigned int enabled = 0;
5060	serge	1448	bool cxsr_enabled;
3031	serge	1449
3746	Serge	1450	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1451	&g4x_wm_info, latency_ns,
		1452	&g4x_cursor_wm_info, latency_ns,
		1453	&planea_wm, &cursora_wm))
3746	Serge	1454	enabled \|= 1 << PIPE_A;
3031	serge	1455
3746	Serge	1456	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1457	&g4x_wm_info, latency_ns,
		1458	&g4x_cursor_wm_info, latency_ns,
		1459	&planeb_wm, &cursorb_wm))
3746	Serge	1460	enabled \|= 1 << PIPE_B;
3031	serge	1461
		1462	if (single_plane_enabled(enabled) &&
		1463	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1464	sr_latency_ns,
		1465	&g4x_wm_info,
		1466	&g4x_cursor_wm_info,
3243	Serge	1467	&plane_sr, &cursor_sr)) {
5060	serge	1468	cxsr_enabled = true;
3243	Serge	1469	} else {
5060	serge	1470	cxsr_enabled = false;
		1471	intel_set_memory_cxsr(dev_priv, false);
3243	Serge	1472	plane_sr = cursor_sr = 0;
		1473	}
3031	serge	1474
		1475	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1476	planea_wm, cursora_wm,
		1477	planeb_wm, cursorb_wm,
		1478	plane_sr, cursor_sr);
		1479
		1480	I915_WRITE(DSPFW1,
		1481	(plane_sr << DSPFW_SR_SHIFT) \|
		1482	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1483	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1484	planea_wm);
		1485	I915_WRITE(DSPFW2,
3243	Serge	1486	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1487	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1488	/* HPLL off in SR has some issues on G4x... disable it */
		1489	I915_WRITE(DSPFW3,
3243	Serge	1490	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
3031	serge	1491	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
5060	serge	1492
		1493	if (cxsr_enabled)
		1494	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1495	}
		1496
4560	Serge	1497	static void i965_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1498	{
4560	Serge	1499	struct drm_device *dev = unused_crtc->dev;
3031	serge	1500	struct drm_i915_private *dev_priv = dev->dev_private;
		1501	struct drm_crtc *crtc;
		1502	int srwm = 1;
		1503	int cursor_sr = 16;
5060	serge	1504	bool cxsr_enabled;
3031	serge	1505
		1506	/* Calc sr entries for one plane configs */
		1507	crtc = single_enabled_crtc(dev);
		1508	if (crtc) {
		1509	/* self-refresh has much higher latency */
		1510	static const int sr_latency_ns = 12000;
4560	Serge	1511	const struct drm_display_mode *adjusted_mode =
		1512	&to_intel_crtc(crtc)->config.adjusted_mode;
		1513	int clock = adjusted_mode->crtc_clock;
		1514	int htotal = adjusted_mode->crtc_htotal;
		1515	int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
5060	serge	1516	int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
3031	serge	1517	unsigned long line_time_us;
		1518	int entries;
		1519
5060	serge	1520	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1521
		1522	/* Use ns/us then divide to preserve precision */
		1523	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1524	pixel_size * hdisplay;
		1525	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1526	srwm = I965_FIFO_SIZE - entries;
		1527	if (srwm < 0)
		1528	srwm = 1;
		1529	srwm &= 0x1ff;
		1530	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1531	entries, srwm);
		1532
		1533	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
5060	serge	1534	pixel_size * to_intel_crtc(crtc)->cursor_width;
3031	serge	1535	entries = DIV_ROUND_UP(entries,
		1536	i965_cursor_wm_info.cacheline_size);
		1537	cursor_sr = i965_cursor_wm_info.fifo_size -
		1538	(entries + i965_cursor_wm_info.guard_size);
		1539
		1540	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1541	cursor_sr = i965_cursor_wm_info.max_wm;
		1542
		1543	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1544	"cursor %d\n", srwm, cursor_sr);
		1545
5060	serge	1546	cxsr_enabled = true;
3031	serge	1547	} else {
5060	serge	1548	cxsr_enabled = false;
3031	serge	1549	/* Turn off self refresh if both pipes are enabled */
5060	serge	1550	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1551	}
		1552
		1553	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1554	srwm);
		1555
		1556	/* 965 has limitations... */
		1557	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) \|
		1558	(8 << 16) \| (8 << 8) \| (8 << 0));
		1559	I915_WRITE(DSPFW2, (8 << 8) \| (8 << 0));
		1560	/* update cursor SR watermark */
		1561	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
5060	serge	1562
		1563	if (cxsr_enabled)
		1564	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1565	}
		1566
4560	Serge	1567	static void i9xx_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1568	{
4560	Serge	1569	struct drm_device *dev = unused_crtc->dev;
3031	serge	1570	struct drm_i915_private *dev_priv = dev->dev_private;
		1571	const struct intel_watermark_params *wm_info;
		1572	uint32_t fwater_lo;
		1573	uint32_t fwater_hi;
		1574	int cwm, srwm = 1;
		1575	int fifo_size;
		1576	int planea_wm, planeb_wm;
		1577	struct drm_crtc crtc, enabled = NULL;
		1578
		1579	if (IS_I945GM(dev))
		1580	wm_info = &i945_wm_info;
		1581	else if (!IS_GEN2(dev))
		1582	wm_info = &i915_wm_info;
		1583	else
4560	Serge	1584	wm_info = &i830_wm_info;
3031	serge	1585
		1586	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1587	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1588	if (intel_crtc_active(crtc)) {
4560	Serge	1589	const struct drm_display_mode *adjusted_mode;
5060	serge	1590	int cpp = crtc->primary->fb->bits_per_pixel / 8;
3243	Serge	1591	if (IS_GEN2(dev))
		1592	cpp = 4;
		1593
4560	Serge	1594	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1595	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1596	wm_info, fifo_size, cpp,
3031	serge	1597	latency_ns);
		1598	enabled = crtc;
		1599	} else
		1600	planea_wm = fifo_size - wm_info->guard_size;
		1601
		1602	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1603	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1604	if (intel_crtc_active(crtc)) {
4560	Serge	1605	const struct drm_display_mode *adjusted_mode;
5060	serge	1606	int cpp = crtc->primary->fb->bits_per_pixel / 8;
3243	Serge	1607	if (IS_GEN2(dev))
		1608	cpp = 4;
		1609
4560	Serge	1610	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1611	planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
3243	Serge	1612	wm_info, fifo_size, cpp,
3031	serge	1613	latency_ns);
		1614	if (enabled == NULL)
		1615	enabled = crtc;
		1616	else
		1617	enabled = NULL;
		1618	} else
		1619	planeb_wm = fifo_size - wm_info->guard_size;
		1620
		1621	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1622
5060	serge	1623	if (IS_I915GM(dev) && enabled) {
		1624	struct drm_i915_gem_object *obj;
		1625
		1626	obj = intel_fb_obj(enabled->primary->fb);
		1627
		1628	/* self-refresh seems busted with untiled */
		1629	if (obj->tiling_mode == I915_TILING_NONE)
		1630	enabled = NULL;
		1631	}
		1632
3031	serge	1633	/*
		1634	* Overlay gets an aggressive default since video jitter is bad.
		1635	*/
		1636	cwm = 2;
		1637
		1638	/* Play safe and disable self-refresh before adjusting watermarks. */
5060	serge	1639	intel_set_memory_cxsr(dev_priv, false);
3031	serge	1640
		1641	/* Calc sr entries for one plane configs */
		1642	if (HAS_FW_BLC(dev) && enabled) {
		1643	/* self-refresh has much higher latency */
		1644	static const int sr_latency_ns = 6000;
4560	Serge	1645	const struct drm_display_mode *adjusted_mode =
		1646	&to_intel_crtc(enabled)->config.adjusted_mode;
		1647	int clock = adjusted_mode->crtc_clock;
		1648	int htotal = adjusted_mode->crtc_htotal;
		1649	int hdisplay = to_intel_crtc(enabled)->config.pipe_src_w;
5060	serge	1650	int pixel_size = enabled->primary->fb->bits_per_pixel / 8;
3031	serge	1651	unsigned long line_time_us;
		1652	int entries;
		1653
5060	serge	1654	line_time_us = max(htotal * 1000 / clock, 1);
3031	serge	1655
		1656	/* Use ns/us then divide to preserve precision */
		1657	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1658	pixel_size * hdisplay;
		1659	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1660	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1661	srwm = wm_info->fifo_size - entries;
		1662	if (srwm < 0)
		1663	srwm = 1;
		1664
		1665	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1666	I915_WRITE(FW_BLC_SELF,
		1667	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1668	else if (IS_I915GM(dev))
		1669	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1670	}
		1671
		1672	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1673	planea_wm, planeb_wm, cwm, srwm);
		1674
		1675	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1676	fwater_hi = (cwm & 0x1f);
		1677
		1678	/* Set request length to 8 cachelines per fetch */
		1679	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1680	fwater_hi = fwater_hi \| (1 << 8);
		1681
		1682	I915_WRITE(FW_BLC, fwater_lo);
		1683	I915_WRITE(FW_BLC2, fwater_hi);
		1684
5060	serge	1685	if (enabled)
		1686	intel_set_memory_cxsr(dev_priv, true);
3031	serge	1687	}
		1688
4560	Serge	1689	static void i845_update_wm(struct drm_crtc *unused_crtc)
3031	serge	1690	{
4560	Serge	1691	struct drm_device *dev = unused_crtc->dev;
3031	serge	1692	struct drm_i915_private *dev_priv = dev->dev_private;
		1693	struct drm_crtc *crtc;
4560	Serge	1694	const struct drm_display_mode *adjusted_mode;
3031	serge	1695	uint32_t fwater_lo;
		1696	int planea_wm;
		1697
		1698	crtc = single_enabled_crtc(dev);
		1699	if (crtc == NULL)
		1700	return;
		1701
4560	Serge	1702	adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
		1703	planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
		1704	&i845_wm_info,
3031	serge	1705	dev_priv->display.get_fifo_size(dev, 0),
3243	Serge	1706	4, latency_ns);
3031	serge	1707	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1708	fwater_lo \|= (3<<8) \| planea_wm;
		1709
		1710	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1711
		1712	I915_WRITE(FW_BLC, fwater_lo);
		1713	}
		1714
4104	Serge	1715	static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
		1716	struct drm_crtc *crtc)
3031	serge	1717	{
4104	Serge	1718	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1719	uint32_t pixel_rate;
		1720
4560	Serge	1721	pixel_rate = intel_crtc->config.adjusted_mode.crtc_clock;
4104	Serge	1722
		1723	/* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
		1724	* adjust the pixel_rate here. */
		1725
		1726	if (intel_crtc->config.pch_pfit.enabled) {
		1727	uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
		1728	uint32_t pfit_size = intel_crtc->config.pch_pfit.size;
		1729
4560	Serge	1730	pipe_w = intel_crtc->config.pipe_src_w;
		1731	pipe_h = intel_crtc->config.pipe_src_h;
4104	Serge	1732	pfit_w = (pfit_size >> 16) & 0xFFFF;
		1733	pfit_h = pfit_size & 0xFFFF;
		1734	if (pipe_w < pfit_w)
		1735	pipe_w = pfit_w;
		1736	if (pipe_h < pfit_h)
		1737	pipe_h = pfit_h;
		1738
		1739	pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
		1740	pfit_w * pfit_h);
		1741	}
		1742
		1743	return pixel_rate;
		1744	}
		1745
		1746	/* latency must be in 0.1us units. */
		1747	static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
		1748	uint32_t latency)
		1749	{
		1750	uint64_t ret;
		1751
		1752	if (WARN(latency == 0, "Latency value missing\n"))
		1753	return UINT_MAX;
		1754
		1755	ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
		1756	ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
		1757
		1758	return ret;
		1759	}
		1760
		1761	/* latency must be in 0.1us units. */
		1762	static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
		1763	uint32_t horiz_pixels, uint8_t bytes_per_pixel,
		1764	uint32_t latency)
		1765	{
		1766	uint32_t ret;
		1767
		1768	if (WARN(latency == 0, "Latency value missing\n"))
		1769	return UINT_MAX;
		1770
		1771	ret = (latency * pixel_rate) / (pipe_htotal * 10000);
		1772	ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
		1773	ret = DIV_ROUND_UP(ret, 64) + 2;
		1774	return ret;
		1775	}
		1776
		1777	static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
		1778	uint8_t bytes_per_pixel)
		1779	{
		1780	return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
		1781	}
		1782
4560	Serge	1783	struct ilk_pipe_wm_parameters {
4104	Serge	1784	bool active;
		1785	uint32_t pipe_htotal;
		1786	uint32_t pixel_rate;
		1787	struct intel_plane_wm_parameters pri;
		1788	struct intel_plane_wm_parameters spr;
		1789	struct intel_plane_wm_parameters cur;
		1790	};
		1791
4560	Serge	1792	struct ilk_wm_maximums {
4104	Serge	1793	uint16_t pri;
		1794	uint16_t spr;
		1795	uint16_t cur;
		1796	uint16_t fbc;
		1797	};
		1798
		1799	/* used in computing the new watermarks state */
		1800	struct intel_wm_config {
		1801	unsigned int num_pipes_active;
		1802	bool sprites_enabled;
		1803	bool sprites_scaled;
		1804	};
		1805
		1806	/*
		1807	* For both WM_PIPE and WM_LP.
		1808	* mem_value must be in 0.1us units.
		1809	*/
4560	Serge	1810	static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	1811	uint32_t mem_value,
		1812	bool is_lp)
		1813	{
		1814	uint32_t method1, method2;
		1815
		1816	if (!params->active \|\| !params->pri.enabled)
		1817	return 0;
		1818
		1819	method1 = ilk_wm_method1(params->pixel_rate,
		1820	params->pri.bytes_per_pixel,
		1821	mem_value);
		1822
		1823	if (!is_lp)
		1824	return method1;
		1825
		1826	method2 = ilk_wm_method2(params->pixel_rate,
		1827	params->pipe_htotal,
		1828	params->pri.horiz_pixels,
		1829	params->pri.bytes_per_pixel,
		1830	mem_value);
		1831
		1832	return min(method1, method2);
		1833	}
		1834
		1835	/*
		1836	* For both WM_PIPE and WM_LP.
		1837	* mem_value must be in 0.1us units.
		1838	*/
4560	Serge	1839	static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	1840	uint32_t mem_value)
		1841	{
		1842	uint32_t method1, method2;
		1843
		1844	if (!params->active \|\| !params->spr.enabled)
		1845	return 0;
		1846
		1847	method1 = ilk_wm_method1(params->pixel_rate,
		1848	params->spr.bytes_per_pixel,
		1849	mem_value);
		1850	method2 = ilk_wm_method2(params->pixel_rate,
		1851	params->pipe_htotal,
		1852	params->spr.horiz_pixels,
		1853	params->spr.bytes_per_pixel,
		1854	mem_value);
		1855	return min(method1, method2);
		1856	}
		1857
		1858	/*
		1859	* For both WM_PIPE and WM_LP.
		1860	* mem_value must be in 0.1us units.
		1861	*/
4560	Serge	1862	static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	1863	uint32_t mem_value)
		1864	{
		1865	if (!params->active \|\| !params->cur.enabled)
		1866	return 0;
		1867
		1868	return ilk_wm_method2(params->pixel_rate,
		1869	params->pipe_htotal,
		1870	params->cur.horiz_pixels,
		1871	params->cur.bytes_per_pixel,
		1872	mem_value);
		1873	}
		1874
		1875	/* Only for WM_LP. */
4560	Serge	1876	static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
4104	Serge	1877	uint32_t pri_val)
		1878	{
		1879	if (!params->active \|\| !params->pri.enabled)
		1880	return 0;
		1881
		1882	return ilk_wm_fbc(pri_val,
		1883	params->pri.horiz_pixels,
		1884	params->pri.bytes_per_pixel);
		1885	}
		1886
		1887	static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
		1888	{
4560	Serge	1889	if (INTEL_INFO(dev)->gen >= 8)
		1890	return 3072;
		1891	else if (INTEL_INFO(dev)->gen >= 7)
4104	Serge	1892	return 768;
		1893	else
		1894	return 512;
		1895	}
		1896
5060	serge	1897	static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
		1898	int level, bool is_sprite)
		1899	{
		1900	if (INTEL_INFO(dev)->gen >= 8)
		1901	/* BDW primary/sprite plane watermarks */
		1902	return level == 0 ? 255 : 2047;
		1903	else if (INTEL_INFO(dev)->gen >= 7)
		1904	/* IVB/HSW primary/sprite plane watermarks */
		1905	return level == 0 ? 127 : 1023;
		1906	else if (!is_sprite)
		1907	/* ILK/SNB primary plane watermarks */
		1908	return level == 0 ? 127 : 511;
		1909	else
		1910	/* ILK/SNB sprite plane watermarks */
		1911	return level == 0 ? 63 : 255;
		1912	}
		1913
		1914	static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
		1915	int level)
		1916	{
		1917	if (INTEL_INFO(dev)->gen >= 7)
		1918	return level == 0 ? 63 : 255;
		1919	else
		1920	return level == 0 ? 31 : 63;
		1921	}
		1922
		1923	static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
		1924	{
		1925	if (INTEL_INFO(dev)->gen >= 8)
		1926	return 31;
		1927	else
		1928	return 15;
		1929	}
		1930
4104	Serge	1931	/* Calculate the maximum primary/sprite plane watermark */
		1932	static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
		1933	int level,
		1934	const struct intel_wm_config *config,
		1935	enum intel_ddb_partitioning ddb_partitioning,
		1936	bool is_sprite)
		1937	{
		1938	unsigned int fifo_size = ilk_display_fifo_size(dev);
		1939
		1940	/* if sprites aren't enabled, sprites get nothing */
		1941	if (is_sprite && !config->sprites_enabled)
		1942	return 0;
		1943
		1944	/* HSW allows LP1+ watermarks even with multiple pipes */
		1945	if (level == 0 \|\| config->num_pipes_active > 1) {
		1946	fifo_size /= INTEL_INFO(dev)->num_pipes;
		1947
		1948	/*
		1949	* For some reason the non self refresh
		1950	* FIFO size is only half of the self
		1951	* refresh FIFO size on ILK/SNB.
		1952	*/
		1953	if (INTEL_INFO(dev)->gen <= 6)
		1954	fifo_size /= 2;
		1955	}
		1956
		1957	if (config->sprites_enabled) {
		1958	/* level 0 is always calculated with 1:1 split */
		1959	if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
		1960	if (is_sprite)
		1961	fifo_size *= 5;
		1962	fifo_size /= 6;
		1963	} else {
		1964	fifo_size /= 2;
		1965	}
		1966	}
		1967
		1968	/* clamp to max that the registers can hold */
5060	serge	1969	return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
4104	Serge	1970	}
		1971
		1972	/* Calculate the maximum cursor plane watermark */
		1973	static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
		1974	int level,
		1975	const struct intel_wm_config *config)
		1976	{
		1977	/* HSW LP1+ watermarks w/ multiple pipes */
		1978	if (level > 0 && config->num_pipes_active > 1)
		1979	return 64;
		1980
		1981	/* otherwise just report max that registers can hold */
5060	serge	1982	return ilk_cursor_wm_reg_max(dev, level);
4539	Serge	1983	}
4104	Serge	1984
5060	serge	1985	static void ilk_compute_wm_maximums(const struct drm_device *dev,
4104	Serge	1986	int level,
		1987	const struct intel_wm_config *config,
		1988	enum intel_ddb_partitioning ddb_partitioning,
4560	Serge	1989	struct ilk_wm_maximums *max)
4104	Serge	1990	{
		1991	max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
		1992	max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
		1993	max->cur = ilk_cursor_wm_max(dev, level, config);
5060	serge	1994	max->fbc = ilk_fbc_wm_reg_max(dev);
4539	Serge	1995	}
4104	Serge	1996
5060	serge	1997	static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
		1998	int level,
		1999	struct ilk_wm_maximums *max)
		2000	{
		2001	max->pri = ilk_plane_wm_reg_max(dev, level, false);
		2002	max->spr = ilk_plane_wm_reg_max(dev, level, true);
		2003	max->cur = ilk_cursor_wm_reg_max(dev, level);
		2004	max->fbc = ilk_fbc_wm_reg_max(dev);
		2005	}
		2006
4560	Serge	2007	static bool ilk_validate_wm_level(int level,
		2008	const struct ilk_wm_maximums *max,
4104	Serge	2009	struct intel_wm_level *result)
		2010	{
		2011	bool ret;
		2012
		2013	/* already determined to be invalid? */
		2014	if (!result->enable)
		2015	return false;
		2016
		2017	result->enable = result->pri_val <= max->pri &&
		2018	result->spr_val <= max->spr &&
		2019	result->cur_val <= max->cur;
		2020
		2021	ret = result->enable;
		2022
		2023	/*
		2024	* HACK until we can pre-compute everything,
		2025	* and thus fail gracefully if LP0 watermarks
		2026	* are exceeded...
		2027	*/
		2028	if (level == 0 && !result->enable) {
		2029	if (result->pri_val > max->pri)
		2030	DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
		2031	level, result->pri_val, max->pri);
		2032	if (result->spr_val > max->spr)
		2033	DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
		2034	level, result->spr_val, max->spr);
		2035	if (result->cur_val > max->cur)
		2036	DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
		2037	level, result->cur_val, max->cur);
		2038
		2039	result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
		2040	result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
		2041	result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
		2042	result->enable = true;
		2043	}
		2044
		2045	return ret;
		2046	}
		2047
5060	serge	2048	static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
4104	Serge	2049	int level,
4560	Serge	2050	const struct ilk_pipe_wm_parameters *p,
4104	Serge	2051	struct intel_wm_level *result)
		2052	{
		2053	uint16_t pri_latency = dev_priv->wm.pri_latency[level];
		2054	uint16_t spr_latency = dev_priv->wm.spr_latency[level];
		2055	uint16_t cur_latency = dev_priv->wm.cur_latency[level];
		2056
		2057	/* WM1+ latency values stored in 0.5us units */
		2058	if (level > 0) {
		2059	pri_latency *= 5;
		2060	spr_latency *= 5;
		2061	cur_latency *= 5;
		2062	}
		2063
		2064	result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
		2065	result->spr_val = ilk_compute_spr_wm(p, spr_latency);
		2066	result->cur_val = ilk_compute_cur_wm(p, cur_latency);
		2067	result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
		2068	result->enable = true;
		2069	}
		2070
		2071	static uint32_t
		2072	hsw_compute_linetime_wm(struct drm_device dev, struct drm_crtc crtc)
		2073	{
3031	serge	2074	struct drm_i915_private *dev_priv = dev->dev_private;
4104	Serge	2075	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2076	struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
		2077	u32 linetime, ips_linetime;
3031	serge	2078
4104	Serge	2079	if (!intel_crtc_active(crtc))
		2080	return 0;
3031	serge	2081
		2082	/* The WM are computed with base on how long it takes to fill a single
		2083	* row at the given clock rate, multiplied by 8.
		2084	* */
4560	Serge	2085	linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
		2086	mode->crtc_clock);
		2087	ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
4104	Serge	2088	intel_ddi_get_cdclk_freq(dev_priv));
3031	serge	2089
4104	Serge	2090	return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) \|
		2091	PIPE_WM_LINETIME_TIME(linetime);
		2092	}
		2093
		2094	static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2095	{
		2096	struct drm_i915_private *dev_priv = dev->dev_private;
		2097
4560	Serge	2098	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2099	uint64_t sskpd = I915_READ64(MCH_SSKPD);
		2100
		2101	wm[0] = (sskpd >> 56) & 0xFF;
		2102	if (wm[0] == 0)
		2103	wm[0] = sskpd & 0xF;
		2104	wm[1] = (sskpd >> 4) & 0xFF;
		2105	wm[2] = (sskpd >> 12) & 0xFF;
		2106	wm[3] = (sskpd >> 20) & 0x1FF;
		2107	wm[4] = (sskpd >> 32) & 0x1FF;
		2108	} else if (INTEL_INFO(dev)->gen >= 6) {
		2109	uint32_t sskpd = I915_READ(MCH_SSKPD);
		2110
		2111	wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
		2112	wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
		2113	wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
		2114	wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
		2115	} else if (INTEL_INFO(dev)->gen >= 5) {
		2116	uint32_t mltr = I915_READ(MLTR_ILK);
		2117
		2118	/* ILK primary LP0 latency is 700 ns */
		2119	wm[0] = 7;
		2120	wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
		2121	wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
		2122	}
		2123	}
		2124
		2125	static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2126	{
		2127	/* ILK sprite LP0 latency is 1300 ns */
		2128	if (INTEL_INFO(dev)->gen == 5)
		2129	wm[0] = 13;
		2130	}
		2131
		2132	static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
		2133	{
		2134	/* ILK cursor LP0 latency is 1300 ns */
		2135	if (INTEL_INFO(dev)->gen == 5)
		2136	wm[0] = 13;
		2137
		2138	/* WaDoubleCursorLP3Latency:ivb */
		2139	if (IS_IVYBRIDGE(dev))
		2140	wm[3] *= 2;
		2141	}
		2142
5060	serge	2143	int ilk_wm_max_level(const struct drm_device *dev)
4560	Serge	2144	{
		2145	/* how many WM levels are we expecting */
		2146	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2147	return 4;
		2148	else if (INTEL_INFO(dev)->gen >= 6)
		2149	return 3;
		2150	else
		2151	return 2;
		2152	}
		2153
4104	Serge	2154	static void intel_print_wm_latency(struct drm_device *dev,
		2155	const char *name,
		2156	const uint16_t wm[5])
		2157	{
4560	Serge	2158	int level, max_level = ilk_wm_max_level(dev);
4104	Serge	2159
		2160	for (level = 0; level <= max_level; level++) {
		2161	unsigned int latency = wm[level];
		2162
		2163	if (latency == 0) {
		2164	DRM_ERROR("%s WM%d latency not provided\n",
		2165	name, level);
		2166	continue;
		2167	}
		2168
		2169	/* WM1+ latency values in 0.5us units */
		2170	if (level > 0)
		2171	latency *= 5;
		2172
		2173	DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
		2174	name, level, wm[level],
		2175	latency / 10, latency % 10);
		2176	}
		2177	}
		2178
5060	serge	2179	static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
		2180	uint16_t wm[5], uint16_t min)
4104	Serge	2181	{
5060	serge	2182	int level, max_level = ilk_wm_max_level(dev_priv->dev);
		2183
		2184	if (wm[0] >= min)
		2185	return false;
		2186
		2187	wm[0] = max(wm[0], min);
		2188	for (level = 1; level <= max_level; level++)
		2189	wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
		2190
		2191	return true;
		2192	}
		2193
		2194	static void snb_wm_latency_quirk(struct drm_device *dev)
		2195	{
4104	Serge	2196	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	2197	bool changed;
4104	Serge	2198
5060	serge	2199	/*
		2200	* The BIOS provided WM memory latency values are often
		2201	* inadequate for high resolution displays. Adjust them.
		2202	*/
		2203	changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) \|
		2204	ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) \|
		2205	ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
		2206
		2207	if (!changed)
		2208	return;
		2209
		2210	DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
		2211	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2212	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2213	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
		2214	}
		2215
		2216	static void ilk_setup_wm_latency(struct drm_device *dev)
		2217	{
		2218	struct drm_i915_private *dev_priv = dev->dev_private;
		2219
4104	Serge	2220	intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
		2221
		2222	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
		2223	sizeof(dev_priv->wm.pri_latency));
		2224	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
		2225	sizeof(dev_priv->wm.pri_latency));
		2226
		2227	intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
		2228	intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
		2229
		2230	intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
		2231	intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
		2232	intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
5060	serge	2233
		2234	if (IS_GEN6(dev))
		2235	snb_wm_latency_quirk(dev);
4104	Serge	2236	}
		2237
4560	Serge	2238	static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
5060	serge	2239	struct ilk_pipe_wm_parameters *p)
4104	Serge	2240	{
4560	Serge	2241	struct drm_device *dev = crtc->dev;
		2242	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2243	enum pipe pipe = intel_crtc->pipe;
4104	Serge	2244	struct drm_plane *plane;
		2245
5060	serge	2246	if (!intel_crtc_active(crtc))
		2247	return;
		2248
		2249	p->active = true;
4560	Serge	2250	p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal;
4104	Serge	2251	p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
5060	serge	2252	p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8;
4104	Serge	2253	p->cur.bytes_per_pixel = 4;
4560	Serge	2254	p->pri.horiz_pixels = intel_crtc->config.pipe_src_w;
5060	serge	2255	p->cur.horiz_pixels = intel_crtc->cursor_width;
4104	Serge	2256	/* TODO: for now, assume primary and cursor planes are always enabled. */
		2257	p->pri.enabled = true;
		2258	p->cur.enabled = true;
		2259
5060	serge	2260	drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
4104	Serge	2261	struct intel_plane *intel_plane = to_intel_plane(plane);
		2262
5060	serge	2263	if (intel_plane->pipe == pipe) {
4104	Serge	2264	p->spr = intel_plane->wm;
5060	serge	2265	break;
		2266	}
		2267	}
		2268	}
4104	Serge	2269
5060	serge	2270	static void ilk_compute_wm_config(struct drm_device *dev,
		2271	struct intel_wm_config *config)
		2272	{
		2273	struct intel_crtc *intel_crtc;
		2274
		2275	/* Compute the currently _active_ config */
		2276	for_each_intel_crtc(dev, intel_crtc) {
		2277	const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
		2278
		2279	if (!wm->pipe_enabled)
		2280	continue;
		2281
		2282	config->sprites_enabled \|= wm->sprites_enabled;
		2283	config->sprites_scaled \|= wm->sprites_scaled;
		2284	config->num_pipes_active++;
4104	Serge	2285	}
4560	Serge	2286	}
4104	Serge	2287
4560	Serge	2288	/* Compute new watermarks for the pipe */
		2289	static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
		2290	const struct ilk_pipe_wm_parameters *params,
		2291	struct intel_pipe_wm *pipe_wm)
		2292	{
		2293	struct drm_device *dev = crtc->dev;
5060	serge	2294	const struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2295	int level, max_level = ilk_wm_max_level(dev);
		2296	/* LP0 watermark maximums depend on this pipe alone */
		2297	struct intel_wm_config config = {
		2298	.num_pipes_active = 1,
		2299	.sprites_enabled = params->spr.enabled,
		2300	.sprites_scaled = params->spr.scaled,
		2301	};
		2302	struct ilk_wm_maximums max;
4104	Serge	2303
5060	serge	2304	pipe_wm->pipe_enabled = params->active;
		2305	pipe_wm->sprites_enabled = params->spr.enabled;
		2306	pipe_wm->sprites_scaled = params->spr.scaled;
4560	Serge	2307
		2308	/* ILK/SNB: LP2+ watermarks only w/o sprites */
		2309	if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
		2310	max_level = 1;
		2311
		2312	/* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
		2313	if (params->spr.scaled)
		2314	max_level = 0;
		2315
5060	serge	2316	ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]);
4560	Serge	2317
		2318	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2319	pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
		2320
5060	serge	2321	/* LP0 watermarks always use 1/2 DDB partitioning */
		2322	ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
		2323
4560	Serge	2324	/* At least LP0 must be valid */
5060	serge	2325	if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
		2326	return false;
		2327
		2328	ilk_compute_wm_reg_maximums(dev, 1, &max);
		2329
		2330	for (level = 1; level <= max_level; level++) {
		2331	struct intel_wm_level wm = {};
		2332
		2333	ilk_compute_wm_level(dev_priv, level, params, &wm);
		2334
		2335	/*
		2336	* Disable any watermark level that exceeds the
		2337	* register maximums since such watermarks are
		2338	* always invalid.
		2339	*/
		2340	if (!ilk_validate_wm_level(level, &max, &wm))
		2341	break;
		2342
		2343	pipe_wm->wm[level] = wm;
		2344	}
		2345
		2346	return true;
4104	Serge	2347	}
		2348
4560	Serge	2349	/*
		2350	* Merge the watermarks from all active pipes for a specific level.
		2351	*/
		2352	static void ilk_merge_wm_level(struct drm_device *dev,
		2353	int level,
		2354	struct intel_wm_level *ret_wm)
4104	Serge	2355	{
4560	Serge	2356	const struct intel_crtc *intel_crtc;
4104	Serge	2357
5060	serge	2358	ret_wm->enable = true;
4104	Serge	2359
5060	serge	2360	for_each_intel_crtc(dev, intel_crtc) {
		2361	const struct intel_pipe_wm *active = &intel_crtc->wm.active;
		2362	const struct intel_wm_level *wm = &active->wm[level];
		2363
		2364	if (!active->pipe_enabled)
		2365	continue;
		2366
		2367	/*
		2368	* The watermark values may have been used in the past,
		2369	* so we must maintain them in the registers for some
		2370	* time even if the level is now disabled.
		2371	*/
4560	Serge	2372	if (!wm->enable)
5060	serge	2373	ret_wm->enable = false;
4104	Serge	2374
4560	Serge	2375	ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
		2376	ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
		2377	ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
		2378	ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
		2379	}
		2380	}
		2381
		2382	/*
		2383	* Merge all low power watermarks for all active pipes.
		2384	*/
		2385	static void ilk_wm_merge(struct drm_device *dev,
		2386	const struct intel_wm_config *config,
		2387	const struct ilk_wm_maximums *max,
		2388	struct intel_pipe_wm *merged)
		2389	{
		2390	int level, max_level = ilk_wm_max_level(dev);
5060	serge	2391	int last_enabled_level = max_level;
4560	Serge	2392
		2393	/* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
		2394	if ((INTEL_INFO(dev)->gen <= 6 \|\| IS_IVYBRIDGE(dev)) &&
		2395	config->num_pipes_active > 1)
		2396	return;
		2397
		2398	/* ILK: FBC WM must be disabled always */
		2399	merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
		2400
		2401	/* merge each WM1+ level */
4104	Serge	2402	for (level = 1; level <= max_level; level++) {
4560	Serge	2403	struct intel_wm_level *wm = &merged->wm[level];
		2404
		2405	ilk_merge_wm_level(dev, level, wm);
		2406
5060	serge	2407	if (level > last_enabled_level)
		2408	wm->enable = false;
		2409	else if (!ilk_validate_wm_level(level, max, wm))
		2410	/* make sure all following levels get disabled */
		2411	last_enabled_level = level - 1;
4560	Serge	2412
		2413	/*
		2414	* The spec says it is preferred to disable
		2415	* FBC WMs instead of disabling a WM level.
		2416	*/
		2417	if (wm->fbc_val > max->fbc) {
5060	serge	2418	if (wm->enable)
4560	Serge	2419	merged->fbc_wm_enabled = false;
		2420	wm->fbc_val = 0;
4104	Serge	2421	}
		2422	}
		2423
4560	Serge	2424	/* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
		2425	/*
		2426	* FIXME this is racy. FBC might get enabled later.
		2427	* What we should check here is whether FBC can be
		2428	* enabled sometime later.
		2429	*/
		2430	if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
		2431	for (level = 2; level <= max_level; level++) {
		2432	struct intel_wm_level *wm = &merged->wm[level];
		2433
		2434	wm->enable = false;
		2435	}
		2436	}
		2437	}
		2438
		2439	static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
		2440	{
		2441	/* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
		2442	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
		2443	}
		2444
		2445	/* The value we need to program into the WM_LPx latency field */
		2446	static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
		2447	{
		2448	struct drm_i915_private *dev_priv = dev->dev_private;
		2449
		2450	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2451	return 2 * level;
		2452	else
		2453	return dev_priv->wm.pri_latency[level];
		2454	}
		2455
		2456	static void ilk_compute_wm_results(struct drm_device *dev,
		2457	const struct intel_pipe_wm *merged,
		2458	enum intel_ddb_partitioning partitioning,
		2459	struct ilk_wm_values *results)
		2460	{
		2461	struct intel_crtc *intel_crtc;
		2462	int level, wm_lp;
		2463
		2464	results->enable_fbc_wm = merged->fbc_wm_enabled;
		2465	results->partitioning = partitioning;
		2466
		2467	/* LP1+ register values */
4104	Serge	2468	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2469	const struct intel_wm_level *r;
		2470
4560	Serge	2471	level = ilk_wm_lp_to_level(wm_lp, merged);
		2472
		2473	r = &merged->wm[level];
4104	Serge	2474
5060	serge	2475	/*
		2476	* Maintain the watermark values even if the level is
		2477	* disabled. Doing otherwise could cause underruns.
		2478	*/
		2479	results->wm_lp[wm_lp - 1] =
4560	Serge	2480	(ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) \|
		2481	(r->pri_val << WM1_LP_SR_SHIFT) \|
		2482	r->cur_val;
		2483
5060	serge	2484	if (r->enable)
		2485	results->wm_lp[wm_lp - 1] \|= WM1_LP_SR_EN;
		2486
4560	Serge	2487	if (INTEL_INFO(dev)->gen >= 8)
		2488	results->wm_lp[wm_lp - 1] \|=
		2489	r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
		2490	else
		2491	results->wm_lp[wm_lp - 1] \|=
		2492	r->fbc_val << WM1_LP_FBC_SHIFT;
		2493
5060	serge	2494	/*
		2495	* Always set WM1S_LP_EN when spr_val != 0, even if the
		2496	* level is disabled. Doing otherwise could cause underruns.
		2497	*/
4560	Serge	2498	if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
		2499	WARN_ON(wm_lp != 1);
		2500	results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN \| r->spr_val;
		2501	} else
4104	Serge	2502	results->wm_lp_spr[wm_lp - 1] = r->spr_val;
		2503	}
		2504
4560	Serge	2505	/* LP0 register values */
5060	serge	2506	for_each_intel_crtc(dev, intel_crtc) {
4560	Serge	2507	enum pipe pipe = intel_crtc->pipe;
		2508	const struct intel_wm_level *r =
		2509	&intel_crtc->wm.active.wm[0];
4104	Serge	2510
4560	Serge	2511	if (WARN_ON(!r->enable))
		2512	continue;
		2513
		2514	results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
		2515
		2516	results->wm_pipe[pipe] =
		2517	(r->pri_val << WM0_PIPE_PLANE_SHIFT) \|
		2518	(r->spr_val << WM0_PIPE_SPRITE_SHIFT) \|
		2519	r->cur_val;
4104	Serge	2520	}
		2521	}
		2522
		2523	/* Find the result with the highest level enabled. Check for enable_fbc_wm in
		2524	* case both are at the same level. Prefer r1 in case they're the same. */
4560	Serge	2525	static struct intel_pipe_wm ilk_find_best_result(struct drm_device dev,
		2526	struct intel_pipe_wm *r1,
		2527	struct intel_pipe_wm *r2)
4104	Serge	2528	{
4560	Serge	2529	int level, max_level = ilk_wm_max_level(dev);
		2530	int level1 = 0, level2 = 0;
4104	Serge	2531
4560	Serge	2532	for (level = 1; level <= max_level; level++) {
		2533	if (r1->wm[level].enable)
		2534	level1 = level;
		2535	if (r2->wm[level].enable)
		2536	level2 = level;
4104	Serge	2537	}
		2538
4560	Serge	2539	if (level1 == level2) {
		2540	if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
4104	Serge	2541	return r2;
		2542	else
		2543	return r1;
4560	Serge	2544	} else if (level1 > level2) {
4104	Serge	2545	return r1;
		2546	} else {
		2547	return r2;
		2548	}
		2549	}
		2550
4560	Serge	2551	/* dirty bits used to track which watermarks need changes */
		2552	#define WM_DIRTY_PIPE(pipe) (1 << (pipe))
		2553	#define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
		2554	#define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
		2555	#define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) \| WM_DIRTY_LP(2) \| WM_DIRTY_LP(3))
		2556	#define WM_DIRTY_FBC (1 << 24)
		2557	#define WM_DIRTY_DDB (1 << 25)
		2558
		2559	static unsigned int ilk_compute_wm_dirty(struct drm_device *dev,
		2560	const struct ilk_wm_values *old,
		2561	const struct ilk_wm_values *new)
		2562	{
		2563	unsigned int dirty = 0;
		2564	enum pipe pipe;
		2565	int wm_lp;
		2566
		2567	for_each_pipe(pipe) {
		2568	if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
		2569	dirty \|= WM_DIRTY_LINETIME(pipe);
		2570	/* Must disable LP1+ watermarks too */
		2571	dirty \|= WM_DIRTY_LP_ALL;
		2572	}
		2573
		2574	if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
		2575	dirty \|= WM_DIRTY_PIPE(pipe);
		2576	/* Must disable LP1+ watermarks too */
		2577	dirty \|= WM_DIRTY_LP_ALL;
		2578	}
		2579	}
		2580
		2581	if (old->enable_fbc_wm != new->enable_fbc_wm) {
		2582	dirty \|= WM_DIRTY_FBC;
		2583	/* Must disable LP1+ watermarks too */
		2584	dirty \|= WM_DIRTY_LP_ALL;
		2585	}
		2586
		2587	if (old->partitioning != new->partitioning) {
		2588	dirty \|= WM_DIRTY_DDB;
		2589	/* Must disable LP1+ watermarks too */
		2590	dirty \|= WM_DIRTY_LP_ALL;
		2591	}
		2592
		2593	/* LP1+ watermarks already deemed dirty, no need to continue */
		2594	if (dirty & WM_DIRTY_LP_ALL)
		2595	return dirty;
		2596
		2597	/* Find the lowest numbered LP1+ watermark in need of an update... */
		2598	for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
		2599	if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] \|\|
		2600	old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
		2601	break;
		2602	}
		2603
		2604	/* ...and mark it and all higher numbered LP1+ watermarks as dirty */
		2605	for (; wm_lp <= 3; wm_lp++)
		2606	dirty \|= WM_DIRTY_LP(wm_lp);
		2607
		2608	return dirty;
		2609	}
		2610
		2611	static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
		2612	unsigned int dirty)
		2613	{
		2614	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2615	bool changed = false;
		2616
		2617	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
		2618	previous->wm_lp[2] &= ~WM1_LP_SR_EN;
		2619	I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
		2620	changed = true;
		2621	}
		2622	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
		2623	previous->wm_lp[1] &= ~WM1_LP_SR_EN;
		2624	I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
		2625	changed = true;
		2626	}
		2627	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
		2628	previous->wm_lp[0] &= ~WM1_LP_SR_EN;
		2629	I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
		2630	changed = true;
		2631	}
		2632
		2633	/*
		2634	* Don't touch WM1S_LP_EN here.
		2635	* Doing so could cause underruns.
		2636	*/
		2637
		2638	return changed;
		2639	}
		2640
4104	Serge	2641	/*
		2642	* The spec says we shouldn't write when we don't need, because every write
		2643	* causes WMs to be re-evaluated, expending some power.
3031	serge	2644	*/
4560	Serge	2645	static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
		2646	struct ilk_wm_values *results)
4104	Serge	2647	{
4560	Serge	2648	struct drm_device *dev = dev_priv->dev;
		2649	struct ilk_wm_values *previous = &dev_priv->wm.hw;
		2650	unsigned int dirty;
4104	Serge	2651	uint32_t val;
3031	serge	2652
4560	Serge	2653	dirty = ilk_compute_wm_dirty(dev, previous, results);
		2654	if (!dirty)
4104	Serge	2655	return;
		2656
4560	Serge	2657	_ilk_disable_lp_wm(dev_priv, dirty);
4104	Serge	2658
4560	Serge	2659	if (dirty & WM_DIRTY_PIPE(PIPE_A))
4104	Serge	2660	I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
4560	Serge	2661	if (dirty & WM_DIRTY_PIPE(PIPE_B))
4104	Serge	2662	I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
4560	Serge	2663	if (dirty & WM_DIRTY_PIPE(PIPE_C))
4104	Serge	2664	I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
		2665
4560	Serge	2666	if (dirty & WM_DIRTY_LINETIME(PIPE_A))
4104	Serge	2667	I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
4560	Serge	2668	if (dirty & WM_DIRTY_LINETIME(PIPE_B))
4104	Serge	2669	I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
4560	Serge	2670	if (dirty & WM_DIRTY_LINETIME(PIPE_C))
4104	Serge	2671	I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
		2672
4560	Serge	2673	if (dirty & WM_DIRTY_DDB) {
		2674	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev)) {
4104	Serge	2675	val = I915_READ(WM_MISC);
4560	Serge	2676	if (results->partitioning == INTEL_DDB_PART_1_2)
4104	Serge	2677	val &= ~WM_MISC_DATA_PARTITION_5_6;
		2678	else
		2679	val \|= WM_MISC_DATA_PARTITION_5_6;
		2680	I915_WRITE(WM_MISC, val);
4560	Serge	2681	} else {
		2682	val = I915_READ(DISP_ARB_CTL2);
		2683	if (results->partitioning == INTEL_DDB_PART_1_2)
		2684	val &= ~DISP_DATA_PARTITION_5_6;
		2685	else
		2686	val \|= DISP_DATA_PARTITION_5_6;
		2687	I915_WRITE(DISP_ARB_CTL2, val);
		2688	}
4104	Serge	2689	}
		2690
4560	Serge	2691	if (dirty & WM_DIRTY_FBC) {
4104	Serge	2692	val = I915_READ(DISP_ARB_CTL);
		2693	if (results->enable_fbc_wm)
		2694	val &= ~DISP_FBC_WM_DIS;
		2695	else
		2696	val \|= DISP_FBC_WM_DIS;
		2697	I915_WRITE(DISP_ARB_CTL, val);
		2698	}
		2699
4560	Serge	2700	if (dirty & WM_DIRTY_LP(1) &&
		2701	previous->wm_lp_spr[0] != results->wm_lp_spr[0])
4104	Serge	2702	I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
4560	Serge	2703
		2704	if (INTEL_INFO(dev)->gen >= 7) {
		2705	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
4104	Serge	2706	I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
4560	Serge	2707	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
4104	Serge	2708	I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
4560	Serge	2709	}
4104	Serge	2710
4560	Serge	2711	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
4104	Serge	2712	I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
4560	Serge	2713	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
4104	Serge	2714	I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
4560	Serge	2715	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
4104	Serge	2716	I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
4560	Serge	2717
		2718	dev_priv->wm.hw = *results;
3031	serge	2719	}
		2720
4560	Serge	2721	static bool ilk_disable_lp_wm(struct drm_device *dev)
4104	Serge	2722	{
		2723	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2724
		2725	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
		2726	}
		2727
		2728	static void ilk_update_wm(struct drm_crtc *crtc)
		2729	{
		2730	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2731	struct drm_device *dev = crtc->dev;
		2732	struct drm_i915_private *dev_priv = dev->dev_private;
		2733	struct ilk_wm_maximums max;
		2734	struct ilk_pipe_wm_parameters params = {};
		2735	struct ilk_wm_values results = {};
4104	Serge	2736	enum intel_ddb_partitioning partitioning;
4560	Serge	2737	struct intel_pipe_wm pipe_wm = {};
		2738	struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
		2739	struct intel_wm_config config = {};
4104	Serge	2740
5060	serge	2741	ilk_compute_wm_parameters(crtc, ¶ms);
4104	Serge	2742
4560	Serge	2743	intel_compute_pipe_wm(crtc, ¶ms, &pipe_wm);
		2744
		2745	if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
		2746	return;
		2747
		2748	intel_crtc->wm.active = pipe_wm;
		2749
5060	serge	2750	ilk_compute_wm_config(dev, &config);
		2751
4560	Serge	2752	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
		2753	ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
		2754
		2755	/* 5/6 split only in single pipe config on IVB+ */
		2756	if (INTEL_INFO(dev)->gen >= 7 &&
		2757	config.num_pipes_active == 1 && config.sprites_enabled) {
		2758	ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
		2759	ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
		2760
		2761	best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
4104	Serge	2762	} else {
4560	Serge	2763	best_lp_wm = &lp_wm_1_2;
4104	Serge	2764	}
		2765
4560	Serge	2766	partitioning = (best_lp_wm == &lp_wm_1_2) ?
4104	Serge	2767	INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
		2768
4560	Serge	2769	ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
		2770
		2771	ilk_write_wm_values(dev_priv, &results);
4104	Serge	2772	}
		2773
5060	serge	2774	static void
		2775	ilk_update_sprite_wm(struct drm_plane *plane,
4104	Serge	2776	struct drm_crtc *crtc,
5060	serge	2777	uint32_t sprite_width, uint32_t sprite_height,
		2778	int pixel_size, bool enabled, bool scaled)
4104	Serge	2779	{
4560	Serge	2780	struct drm_device *dev = plane->dev;
4104	Serge	2781	struct intel_plane *intel_plane = to_intel_plane(plane);
		2782
		2783	intel_plane->wm.enabled = enabled;
		2784	intel_plane->wm.scaled = scaled;
		2785	intel_plane->wm.horiz_pixels = sprite_width;
5060	serge	2786	intel_plane->wm.vert_pixels = sprite_width;
4104	Serge	2787	intel_plane->wm.bytes_per_pixel = pixel_size;
		2788
4560	Serge	2789	/*
		2790	* IVB workaround: must disable low power watermarks for at least
		2791	* one frame before enabling scaling. LP watermarks can be re-enabled
		2792	* when scaling is disabled.
		2793	*
		2794	* WaCxSRDisabledForSpriteScaling:ivb
		2795	*/
		2796	if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
		2797	intel_wait_for_vblank(dev, intel_plane->pipe);
		2798
		2799	ilk_update_wm(crtc);
4104	Serge	2800	}
		2801
4560	Serge	2802	static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3031	serge	2803	{
4560	Serge	2804	struct drm_device *dev = crtc->dev;
		2805	struct drm_i915_private *dev_priv = dev->dev_private;
		2806	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		2807	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		2808	struct intel_pipe_wm *active = &intel_crtc->wm.active;
		2809	enum pipe pipe = intel_crtc->pipe;
		2810	static const unsigned int wm0_pipe_reg[] = {
		2811	[PIPE_A] = WM0_PIPEA_ILK,
		2812	[PIPE_B] = WM0_PIPEB_ILK,
		2813	[PIPE_C] = WM0_PIPEC_IVB,
		2814	};
3031	serge	2815
4560	Serge	2816	hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
		2817	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2818	hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3031	serge	2819
5060	serge	2820	active->pipe_enabled = intel_crtc_active(crtc);
		2821
		2822	if (active->pipe_enabled) {
4560	Serge	2823	u32 tmp = hw->wm_pipe[pipe];
3031	serge	2824
4560	Serge	2825	/*
		2826	* For active pipes LP0 watermark is marked as
		2827	* enabled, and LP1+ watermaks as disabled since
		2828	* we can't really reverse compute them in case
		2829	* multiple pipes are active.
		2830	*/
		2831	active->wm[0].enable = true;
		2832	active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
		2833	active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
		2834	active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
		2835	active->linetime = hw->wm_linetime[pipe];
		2836	} else {
		2837	int level, max_level = ilk_wm_max_level(dev);
3031	serge	2838
4560	Serge	2839	/*
		2840	* For inactive pipes, all watermark levels
		2841	* should be marked as enabled but zeroed,
		2842	* which is what we'd compute them to.
		2843	*/
		2844	for (level = 0; level <= max_level; level++)
		2845	active->wm[level].enable = true;
3031	serge	2846	}
		2847	}
		2848
4560	Serge	2849	void ilk_wm_get_hw_state(struct drm_device *dev)
3031	serge	2850	{
		2851	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	2852	struct ilk_wm_values *hw = &dev_priv->wm.hw;
		2853	struct drm_crtc *crtc;
3031	serge	2854
5060	serge	2855	for_each_crtc(dev, crtc)
4560	Serge	2856	ilk_pipe_wm_get_hw_state(crtc);
4104	Serge	2857
4560	Serge	2858	hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
		2859	hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
		2860	hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3031	serge	2861
4560	Serge	2862	hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
5060	serge	2863	if (INTEL_INFO(dev)->gen >= 7) {
4560	Serge	2864	hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
		2865	hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
5060	serge	2866	}
3031	serge	2867
4560	Serge	2868	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
		2869	hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
		2870	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
		2871	else if (IS_IVYBRIDGE(dev))
		2872	hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
		2873	INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3031	serge	2874
4560	Serge	2875	hw->enable_fbc_wm =
		2876	!(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3031	serge	2877	}
		2878
		2879	/**
		2880	* intel_update_watermarks - update FIFO watermark values based on current modes
		2881	*
		2882	* Calculate watermark values for the various WM regs based on current mode
		2883	* and plane configuration.
		2884	*
		2885	* There are several cases to deal with here:
		2886	* - normal (i.e. non-self-refresh)
		2887	* - self-refresh (SR) mode
		2888	* - lines are large relative to FIFO size (buffer can hold up to 2)
		2889	* - lines are small relative to FIFO size (buffer can hold more than 2
		2890	* lines), so need to account for TLB latency
		2891	*
		2892	* The normal calculation is:
		2893	* watermark = dotclock * bytes per pixel * latency
		2894	* where latency is platform & configuration dependent (we assume pessimal
		2895	* values here).
		2896	*
		2897	* The SR calculation is:
		2898	* watermark = (trunc(latency/line time)+1) * surface width *
		2899	* bytes per pixel
		2900	* where
		2901	* line time = htotal / dotclock
		2902	* surface width = hdisplay for normal plane and 64 for cursor
		2903	* and latency is assumed to be high, as above.
		2904	*
		2905	* The final value programmed to the register should always be rounded up,
		2906	* and include an extra 2 entries to account for clock crossings.
		2907	*
		2908	* We don't use the sprite, so we can ignore that. And on Crestline we have
		2909	* to set the non-SR watermarks to 8.
		2910	*/
4560	Serge	2911	void intel_update_watermarks(struct drm_crtc *crtc)
3031	serge	2912	{
4560	Serge	2913	struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3031	serge	2914
		2915	if (dev_priv->display.update_wm)
4560	Serge	2916	dev_priv->display.update_wm(crtc);
3031	serge	2917	}
		2918
4104	Serge	2919	void intel_update_sprite_watermarks(struct drm_plane *plane,
		2920	struct drm_crtc *crtc,
5060	serge	2921	uint32_t sprite_width,
		2922	uint32_t sprite_height,
		2923	int pixel_size,
4104	Serge	2924	bool enabled, bool scaled)
3031	serge	2925	{
4104	Serge	2926	struct drm_i915_private *dev_priv = plane->dev->dev_private;
3031	serge	2927
		2928	if (dev_priv->display.update_sprite_wm)
5060	serge	2929	dev_priv->display.update_sprite_wm(plane, crtc,
		2930	sprite_width, sprite_height,
4104	Serge	2931	pixel_size, enabled, scaled);
3031	serge	2932	}
		2933
		2934	static struct drm_i915_gem_object *
		2935	intel_alloc_context_page(struct drm_device *dev)
		2936	{
		2937	struct drm_i915_gem_object *ctx;
		2938	int ret;
		2939
		2940	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2941
		2942	ctx = i915_gem_alloc_object(dev, 4096);
		2943	if (!ctx) {
		2944	DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
		2945	return NULL;
		2946	}
		2947
5060	serge	2948	ret = i915_gem_obj_ggtt_pin(ctx, 4096, 0);
3031	serge	2949	if (ret) {
		2950	DRM_ERROR("failed to pin power context: %d\n", ret);
		2951	goto err_unref;
		2952	}
		2953
		2954	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
		2955	if (ret) {
		2956	DRM_ERROR("failed to set-domain on power context: %d\n", ret);
		2957	goto err_unpin;
		2958	}
		2959
		2960	return ctx;
		2961
		2962	err_unpin:
5060	serge	2963	i915_gem_object_ggtt_unpin(ctx);
3031	serge	2964	err_unref:
		2965	drm_gem_object_unreference(&ctx->base);
		2966	return NULL;
		2967	}
		2968
		2969	/**
		2970	* Lock protecting IPS related data structures
		2971	*/
		2972	DEFINE_SPINLOCK(mchdev_lock);
		2973
		2974	/* Global for IPS driver to get at the current i915 device. Protected by
		2975	* mchdev_lock. */
		2976	static struct drm_i915_private *i915_mch_dev;
		2977
		2978	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		2979	{
		2980	struct drm_i915_private *dev_priv = dev->dev_private;
		2981	u16 rgvswctl;
		2982
		2983	assert_spin_locked(&mchdev_lock);
		2984
		2985	rgvswctl = I915_READ16(MEMSWCTL);
		2986	if (rgvswctl & MEMCTL_CMD_STS) {
		2987	DRM_DEBUG("gpu busy, RCS change rejected\n");
		2988	return false; /* still busy with another command */
		2989	}
		2990
		2991	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		2992	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		2993	I915_WRITE16(MEMSWCTL, rgvswctl);
		2994	POSTING_READ16(MEMSWCTL);
		2995
		2996	rgvswctl \|= MEMCTL_CMD_STS;
		2997	I915_WRITE16(MEMSWCTL, rgvswctl);
		2998
		2999	return true;
		3000	}
		3001
		3002	static void ironlake_enable_drps(struct drm_device *dev)
		3003	{
		3004	struct drm_i915_private *dev_priv = dev->dev_private;
		3005	u32 rgvmodectl = I915_READ(MEMMODECTL);
		3006	u8 fmax, fmin, fstart, vstart;
		3007
		3008	spin_lock_irq(&mchdev_lock);
		3009
		3010	/* Enable temp reporting */
		3011	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		3012	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		3013
		3014	/* 100ms RC evaluation intervals */
		3015	I915_WRITE(RCUPEI, 100000);
		3016	I915_WRITE(RCDNEI, 100000);
		3017
		3018	/* Set max/min thresholds to 90ms and 80ms respectively */
		3019	I915_WRITE(RCBMAXAVG, 90000);
		3020	I915_WRITE(RCBMINAVG, 80000);
		3021
		3022	I915_WRITE(MEMIHYST, 1);
		3023
		3024	/* Set up min, max, and cur for interrupt handling */
		3025	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		3026	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		3027	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		3028	MEMMODE_FSTART_SHIFT;
		3029
		3030	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
		3031	PXVFREQ_PX_SHIFT;
		3032
		3033	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		3034	dev_priv->ips.fstart = fstart;
		3035
		3036	dev_priv->ips.max_delay = fstart;
		3037	dev_priv->ips.min_delay = fmin;
		3038	dev_priv->ips.cur_delay = fstart;
		3039
		3040	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		3041	fmax, fmin, fstart);
		3042
		3043	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		3044
		3045	/*
		3046	* Interrupts will be enabled in ironlake_irq_postinstall
		3047	*/
		3048
		3049	I915_WRITE(VIDSTART, vstart);
		3050	POSTING_READ(VIDSTART);
		3051
		3052	rgvmodectl \|= MEMMODE_SWMODE_EN;
		3053	I915_WRITE(MEMMODECTL, rgvmodectl);
		3054
		3055	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		3056	DRM_ERROR("stuck trying to change perf mode\n");
		3057	mdelay(1);
		3058
		3059	ironlake_set_drps(dev, fstart);
		3060
		3061	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
		3062	I915_READ(0x112e0);
5060	serge	3063	dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
3031	serge	3064	dev_priv->ips.last_count2 = I915_READ(0x112f4);
5060	serge	3065	dev_priv->ips.last_time2 = ktime_get_raw_ns();
3031	serge	3066
		3067	spin_unlock_irq(&mchdev_lock);
		3068	}
		3069
		3070	static void ironlake_disable_drps(struct drm_device *dev)
		3071	{
		3072	struct drm_i915_private *dev_priv = dev->dev_private;
		3073	u16 rgvswctl;
		3074
		3075	spin_lock_irq(&mchdev_lock);
		3076
		3077	rgvswctl = I915_READ16(MEMSWCTL);
		3078
		3079	/* Ack interrupts, disable EFC interrupt */
		3080	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		3081	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		3082	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		3083	I915_WRITE(DEIIR, DE_PCU_EVENT);
		3084	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		3085
		3086	/* Go back to the starting frequency */
		3087	ironlake_set_drps(dev, dev_priv->ips.fstart);
		3088	mdelay(1);
		3089	rgvswctl \|= MEMCTL_CMD_STS;
		3090	I915_WRITE(MEMSWCTL, rgvswctl);
		3091	mdelay(1);
		3092
		3093	spin_unlock_irq(&mchdev_lock);
		3094	}
		3095
		3096	/* There's a funny hw issue where the hw returns all 0 when reading from
		3097	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		3098	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		3099	* all limits and the gpu stuck at whatever frequency it is at atm).
		3100	*/
4560	Serge	3101	static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3031	serge	3102	{
		3103	u32 limits;
		3104
		3105	/* Only set the down limit when we've reached the lowest level to avoid
		3106	* getting more interrupts, otherwise leave this clear. This prevents a
		3107	* race in the hw when coming out of rc6: There's a tiny window where
		3108	* the hw runs at the minimal clock before selecting the desired
		3109	* frequency, if the down threshold expires in that window we will not
		3110	* receive a down interrupt. */
5060	serge	3111	limits = dev_priv->rps.max_freq_softlimit << 24;
		3112	if (val <= dev_priv->rps.min_freq_softlimit)
		3113	limits \|= dev_priv->rps.min_freq_softlimit << 16;
3031	serge	3114
		3115	return limits;
		3116	}
		3117
4560	Serge	3118	static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
		3119	{
		3120	int new_power;
		3121
		3122	new_power = dev_priv->rps.power;
		3123	switch (dev_priv->rps.power) {
		3124	case LOW_POWER:
5060	serge	3125	if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
4560	Serge	3126	new_power = BETWEEN;
		3127	break;
		3128
		3129	case BETWEEN:
5060	serge	3130	if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
4560	Serge	3131	new_power = LOW_POWER;
5060	serge	3132	else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
4560	Serge	3133	new_power = HIGH_POWER;
		3134	break;
		3135
		3136	case HIGH_POWER:
5060	serge	3137	if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
4560	Serge	3138	new_power = BETWEEN;
		3139	break;
		3140	}
		3141	/* Max/min bins are special */
5060	serge	3142	if (val == dev_priv->rps.min_freq_softlimit)
4560	Serge	3143	new_power = LOW_POWER;
5060	serge	3144	if (val == dev_priv->rps.max_freq_softlimit)
4560	Serge	3145	new_power = HIGH_POWER;
		3146	if (new_power == dev_priv->rps.power)
		3147	return;
		3148
		3149	/* Note the units here are not exactly 1us, but 1280ns. */
		3150	switch (new_power) {
		3151	case LOW_POWER:
		3152	/* Upclock if more than 95% busy over 16ms */
		3153	I915_WRITE(GEN6_RP_UP_EI, 12500);
		3154	I915_WRITE(GEN6_RP_UP_THRESHOLD, 11800);
		3155
		3156	/* Downclock if less than 85% busy over 32ms */
		3157	I915_WRITE(GEN6_RP_DOWN_EI, 25000);
		3158	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 21250);
		3159
		3160	I915_WRITE(GEN6_RP_CONTROL,
		3161	GEN6_RP_MEDIA_TURBO \|
		3162	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3163	GEN6_RP_MEDIA_IS_GFX \|
		3164	GEN6_RP_ENABLE \|
		3165	GEN6_RP_UP_BUSY_AVG \|
		3166	GEN6_RP_DOWN_IDLE_AVG);
		3167	break;
		3168
		3169	case BETWEEN:
		3170	/* Upclock if more than 90% busy over 13ms */
		3171	I915_WRITE(GEN6_RP_UP_EI, 10250);
		3172	I915_WRITE(GEN6_RP_UP_THRESHOLD, 9225);
		3173
		3174	/* Downclock if less than 75% busy over 32ms */
		3175	I915_WRITE(GEN6_RP_DOWN_EI, 25000);
		3176	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 18750);
		3177
		3178	I915_WRITE(GEN6_RP_CONTROL,
		3179	GEN6_RP_MEDIA_TURBO \|
		3180	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3181	GEN6_RP_MEDIA_IS_GFX \|
		3182	GEN6_RP_ENABLE \|
		3183	GEN6_RP_UP_BUSY_AVG \|
		3184	GEN6_RP_DOWN_IDLE_AVG);
		3185	break;
		3186
		3187	case HIGH_POWER:
		3188	/* Upclock if more than 85% busy over 10ms */
		3189	I915_WRITE(GEN6_RP_UP_EI, 8000);
		3190	I915_WRITE(GEN6_RP_UP_THRESHOLD, 6800);
		3191
		3192	/* Downclock if less than 60% busy over 32ms */
		3193	I915_WRITE(GEN6_RP_DOWN_EI, 25000);
		3194	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 15000);
		3195
		3196	I915_WRITE(GEN6_RP_CONTROL,
		3197	GEN6_RP_MEDIA_TURBO \|
		3198	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3199	GEN6_RP_MEDIA_IS_GFX \|
		3200	GEN6_RP_ENABLE \|
		3201	GEN6_RP_UP_BUSY_AVG \|
		3202	GEN6_RP_DOWN_IDLE_AVG);
		3203	break;
		3204	}
		3205
		3206	dev_priv->rps.power = new_power;
		3207	dev_priv->rps.last_adj = 0;
		3208	}
		3209
5060	serge	3210	static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
		3211	{
		3212	u32 mask = 0;
		3213
		3214	if (val > dev_priv->rps.min_freq_softlimit)
		3215	mask \|= GEN6_PM_RP_DOWN_THRESHOLD \| GEN6_PM_RP_DOWN_TIMEOUT;
		3216	if (val < dev_priv->rps.max_freq_softlimit)
		3217	mask \|= GEN6_PM_RP_UP_THRESHOLD;
		3218
		3219	mask \|= dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED \| GEN6_PM_RP_UP_EI_EXPIRED);
		3220	mask &= dev_priv->pm_rps_events;
		3221
		3222	/* IVB and SNB hard hangs on looping batchbuffer
		3223	* if GEN6_PM_UP_EI_EXPIRED is masked.
		3224	*/
		3225	if (INTEL_INFO(dev_priv->dev)->gen <= 7 && !IS_HASWELL(dev_priv->dev))
		3226	mask \|= GEN6_PM_RP_UP_EI_EXPIRED;
		3227
		3228	if (IS_GEN8(dev_priv->dev))
		3229	mask \|= GEN8_PMINTR_REDIRECT_TO_NON_DISP;
		3230
		3231	return ~mask;
		3232	}
		3233
		3234	/* gen6_set_rps is called to update the frequency request, but should also be
		3235	* called when the range (min_delay and max_delay) is modified so that we can
		3236	* update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
3031	serge	3237	void gen6_set_rps(struct drm_device *dev, u8 val)
		3238	{
		3239	struct drm_i915_private *dev_priv = dev->dev_private;
		3240
3243	Serge	3241	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5060	serge	3242	WARN_ON(val > dev_priv->rps.max_freq_softlimit);
		3243	WARN_ON(val < dev_priv->rps.min_freq_softlimit);
3031	serge	3244
5060	serge	3245	/* min/max delay may still have been modified so be sure to
		3246	* write the limits value.
		3247	*/
		3248	if (val != dev_priv->rps.cur_freq) {
4560	Serge	3249	gen6_set_rps_thresholds(dev_priv, val);
		3250
5060	serge	3251	if (IS_HASWELL(dev) \|\| IS_BROADWELL(dev))
3746	Serge	3252	I915_WRITE(GEN6_RPNSWREQ,
		3253	HSW_FREQUENCY(val));
		3254	else
3031	serge	3255	I915_WRITE(GEN6_RPNSWREQ,
		3256	GEN6_FREQUENCY(val) \|
		3257	GEN6_OFFSET(0) \|
		3258	GEN6_AGGRESSIVE_TURBO);
5060	serge	3259	}
3031	serge	3260
		3261	/* Make sure we continue to get interrupts
		3262	* until we hit the minimum or maximum frequencies.
		3263	*/
5060	serge	3264	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, gen6_rps_limits(dev_priv, val));
		3265	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3031	serge	3266
		3267	POSTING_READ(GEN6_RPNSWREQ);
		3268
5060	serge	3269	dev_priv->rps.cur_freq = val;
3031	serge	3270	trace_intel_gpu_freq_change(val * 50);
		3271	}
		3272
5060	serge	3273	/* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down
		3274	*
		3275	* * If Gfx is Idle, then
		3276	* 1. Mask Turbo interrupts
		3277	* 2. Bring up Gfx clock
		3278	* 3. Change the freq to Rpn and wait till P-Unit updates freq
		3279	* 4. Clear the Force GFX CLK ON bit so that Gfx can down
		3280	* 5. Unmask Turbo interrupts
		3281	*/
		3282	static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
		3283	{
		3284	struct drm_device *dev = dev_priv->dev;
		3285
		3286	/* Latest VLV doesn't need to force the gfx clock */
		3287	if (dev->pdev->revision >= 0xd) {
		3288	valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
		3289	return;
		3290	}
		3291
		3292	/*
		3293	* When we are idle. Drop to min voltage state.
		3294	*/
		3295
		3296	if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit)
		3297	return;
		3298
		3299	/* Mask turbo interrupt so that they will not come in between */
		3300	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
		3301
		3302	vlv_force_gfx_clock(dev_priv, true);
		3303
		3304	dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit;
		3305
		3306	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ,
		3307	dev_priv->rps.min_freq_softlimit);
		3308
		3309	if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS))
		3310	& GENFREQSTATUS) == 0, 5))
		3311	DRM_ERROR("timed out waiting for Punit\n");
		3312
		3313	vlv_force_gfx_clock(dev_priv, false);
		3314
		3315	I915_WRITE(GEN6_PMINTRMSK,
		3316	gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
		3317	}
		3318
4560	Serge	3319	void gen6_rps_idle(struct drm_i915_private *dev_priv)
3031	serge	3320	{
4560	Serge	3321	struct drm_device *dev = dev_priv->dev;
4104	Serge	3322
4560	Serge	3323	mutex_lock(&dev_priv->rps.hw_lock);
		3324	if (dev_priv->rps.enabled) {
5060	serge	3325	if (IS_CHERRYVIEW(dev))
		3326	valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
		3327	else if (IS_VALLEYVIEW(dev))
		3328	vlv_set_rps_idle(dev_priv);
4560	Serge	3329	else
5060	serge	3330	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4560	Serge	3331	dev_priv->rps.last_adj = 0;
		3332	}
		3333	mutex_unlock(&dev_priv->rps.hw_lock);
		3334	}
4104	Serge	3335
4560	Serge	3336	void gen6_rps_boost(struct drm_i915_private *dev_priv)
		3337	{
		3338	struct drm_device *dev = dev_priv->dev;
4104	Serge	3339
4560	Serge	3340	mutex_lock(&dev_priv->rps.hw_lock);
		3341	if (dev_priv->rps.enabled) {
		3342	if (IS_VALLEYVIEW(dev))
5060	serge	3343	valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
4560	Serge	3344	else
5060	serge	3345	gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
4560	Serge	3346	dev_priv->rps.last_adj = 0;
		3347	}
		3348	mutex_unlock(&dev_priv->rps.hw_lock);
4104	Serge	3349	}
		3350
		3351	void valleyview_set_rps(struct drm_device *dev, u8 val)
		3352	{
3031	serge	3353	struct drm_i915_private *dev_priv = dev->dev_private;
		3354
4104	Serge	3355	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5060	serge	3356	WARN_ON(val > dev_priv->rps.max_freq_softlimit);
		3357	WARN_ON(val < dev_priv->rps.min_freq_softlimit);
4104	Serge	3358
		3359	DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
5060	serge	3360	vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
		3361	dev_priv->rps.cur_freq,
4560	Serge	3362	vlv_gpu_freq(dev_priv, val), val);
4104	Serge	3363
5060	serge	3364	if (val != dev_priv->rps.cur_freq)
4104	Serge	3365	vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
		3366
5060	serge	3367	I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
4104	Serge	3368
5060	serge	3369	dev_priv->rps.cur_freq = val;
4560	Serge	3370	trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val));
4104	Serge	3371	}
		3372
5060	serge	3373	static void gen8_disable_rps_interrupts(struct drm_device *dev)
		3374	{
		3375	struct drm_i915_private *dev_priv = dev->dev_private;
		3376
		3377	I915_WRITE(GEN6_PMINTRMSK, ~GEN8_PMINTR_REDIRECT_TO_NON_DISP);
		3378	I915_WRITE(GEN8_GT_IER(2), I915_READ(GEN8_GT_IER(2)) &
		3379	~dev_priv->pm_rps_events);
		3380	/* Complete PM interrupt masking here doesn't race with the rps work
		3381	* item again unmasking PM interrupts because that is using a different
		3382	* register (GEN8_GT_IMR(2)) to mask PM interrupts. The only risk is in
		3383	* leaving stale bits in GEN8_GT_IIR(2) and GEN8_GT_IMR(2) which
		3384	* gen8_enable_rps will clean up. */
		3385
		3386	spin_lock_irq(&dev_priv->irq_lock);
		3387	dev_priv->rps.pm_iir = 0;
		3388	spin_unlock_irq(&dev_priv->irq_lock);
		3389
		3390	I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events);
		3391	}
		3392
4104	Serge	3393	static void gen6_disable_rps_interrupts(struct drm_device *dev)
		3394	{
		3395	struct drm_i915_private *dev_priv = dev->dev_private;
		3396
3031	serge	3397	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
5060	serge	3398	I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) &
		3399	~dev_priv->pm_rps_events);
3031	serge	3400	/* Complete PM interrupt masking here doesn't race with the rps work
		3401	* item again unmasking PM interrupts because that is using a different
		3402	* register (PMIMR) to mask PM interrupts. The only risk is in leaving
		3403	* stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
		3404
4104	Serge	3405	spin_lock_irq(&dev_priv->irq_lock);
3031	serge	3406	dev_priv->rps.pm_iir = 0;
4104	Serge	3407	spin_unlock_irq(&dev_priv->irq_lock);
3031	serge	3408
5060	serge	3409	I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events);
3031	serge	3410	}
		3411
4104	Serge	3412	static void gen6_disable_rps(struct drm_device *dev)
		3413	{
		3414	struct drm_i915_private *dev_priv = dev->dev_private;
		3415
		3416	I915_WRITE(GEN6_RC_CONTROL, 0);
		3417	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		3418
5060	serge	3419	if (IS_BROADWELL(dev))
		3420	gen8_disable_rps_interrupts(dev);
		3421	else
4104	Serge	3422	gen6_disable_rps_interrupts(dev);
		3423	}
		3424
5060	serge	3425	static void cherryview_disable_rps(struct drm_device *dev)
		3426	{
		3427	struct drm_i915_private *dev_priv = dev->dev_private;
		3428
		3429	I915_WRITE(GEN6_RC_CONTROL, 0);
		3430
		3431	gen8_disable_rps_interrupts(dev);
		3432	}
		3433
4104	Serge	3434	static void valleyview_disable_rps(struct drm_device *dev)
		3435	{
		3436	struct drm_i915_private *dev_priv = dev->dev_private;
		3437
		3438	I915_WRITE(GEN6_RC_CONTROL, 0);
		3439
		3440	gen6_disable_rps_interrupts(dev);
		3441	}
		3442
4560	Serge	3443	static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
		3444	{
5060	serge	3445	if (IS_VALLEYVIEW(dev)) {
		3446	if (mode & (GEN7_RC_CTL_TO_MODE \| GEN6_RC_CTL_EI_MODE(1)))
		3447	mode = GEN6_RC_CTL_RC6_ENABLE;
		3448	else
		3449	mode = 0;
		3450	}
		3451	DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
4560	Serge	3452	(mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		3453	(mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		3454	(mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
		3455	}
		3456
5060	serge	3457	static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
3031	serge	3458	{
4104	Serge	3459	/* No RC6 before Ironlake */
		3460	if (INTEL_INFO(dev)->gen < 5)
		3461	return 0;
		3462
5060	serge	3463	/* RC6 is only on Ironlake mobile not on desktop */
		3464	if (INTEL_INFO(dev)->gen == 5 && !IS_IRONLAKE_M(dev))
		3465	return 0;
		3466
3031	serge	3467	/* Respect the kernel parameter if it is set */
5060	serge	3468	if (enable_rc6 >= 0) {
		3469	int mask;
3031	serge	3470
5060	serge	3471	if (INTEL_INFO(dev)->gen == 6 \|\| IS_IVYBRIDGE(dev))
		3472	mask = INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE \|
		3473	INTEL_RC6pp_ENABLE;
		3474	else
		3475	mask = INTEL_RC6_ENABLE;
		3476
		3477	if ((enable_rc6 & mask) != enable_rc6)
		3478	DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
		3479	enable_rc6 & mask, enable_rc6, mask);
		3480
		3481	return enable_rc6 & mask;
		3482	}
		3483
3120	serge	3484	/* Disable RC6 on Ironlake */
		3485	if (INTEL_INFO(dev)->gen == 5)
		3486	return 0;
3031	serge	3487
5060	serge	3488	if (IS_IVYBRIDGE(dev))
		3489	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
3031	serge	3490
		3491	return INTEL_RC6_ENABLE;
5060	serge	3492	}
3031	serge	3493
5060	serge	3494	int intel_enable_rc6(const struct drm_device *dev)
		3495	{
		3496	return i915.enable_rc6;
3031	serge	3497	}
		3498
5060	serge	3499	static void gen8_enable_rps_interrupts(struct drm_device *dev)
		3500	{
		3501	struct drm_i915_private *dev_priv = dev->dev_private;
		3502
		3503	spin_lock_irq(&dev_priv->irq_lock);
		3504	WARN_ON(dev_priv->rps.pm_iir);
		3505	gen8_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		3506	I915_WRITE(GEN8_GT_IIR(2), dev_priv->pm_rps_events);
		3507	spin_unlock_irq(&dev_priv->irq_lock);
		3508	}
		3509
4104	Serge	3510	static void gen6_enable_rps_interrupts(struct drm_device *dev)
		3511	{
		3512	struct drm_i915_private *dev_priv = dev->dev_private;
		3513
		3514	spin_lock_irq(&dev_priv->irq_lock);
		3515	WARN_ON(dev_priv->rps.pm_iir);
5060	serge	3516	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
		3517	I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events);
4104	Serge	3518	spin_unlock_irq(&dev_priv->irq_lock);
5060	serge	3519	}
4104	Serge	3520
5060	serge	3521	static void parse_rp_state_cap(struct drm_i915_private *dev_priv, u32 rp_state_cap)
		3522	{
		3523	/* All of these values are in units of 50MHz */
		3524	dev_priv->rps.cur_freq = 0;
		3525	/* static values from HW: RP0 < RPe < RP1 < RPn (min_freq) */
		3526	dev_priv->rps.rp1_freq = (rp_state_cap >> 8) & 0xff;
		3527	dev_priv->rps.rp0_freq = (rp_state_cap >> 0) & 0xff;
		3528	dev_priv->rps.min_freq = (rp_state_cap >> 16) & 0xff;
		3529	/* XXX: only BYT has a special efficient freq */
		3530	dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
		3531	/* hw_max = RP0 until we check for overclocking */
		3532	dev_priv->rps.max_freq = dev_priv->rps.rp0_freq;
4104	Serge	3533
5060	serge	3534	/* Preserve min/max settings in case of re-init */
		3535	if (dev_priv->rps.max_freq_softlimit == 0)
		3536	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4104	Serge	3537
5060	serge	3538	if (dev_priv->rps.min_freq_softlimit == 0)
		3539	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
4104	Serge	3540	}
		3541
4560	Serge	3542	static void gen8_enable_rps(struct drm_device *dev)
		3543	{
		3544	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	3545	struct intel_engine_cs *ring;
4560	Serge	3546	uint32_t rc6_mask = 0, rp_state_cap;
		3547	int unused;
		3548
		3549	/* 1a: Software RC state - RC0 */
		3550	I915_WRITE(GEN6_RC_STATE, 0);
		3551
		3552	/* 1c & 1d: Get forcewake during program sequence. Although the driver
		3553	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
		3554	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
		3555
		3556	/* 2a: Disable RC states. */
		3557	I915_WRITE(GEN6_RC_CONTROL, 0);
		3558
		3559	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
5060	serge	3560	parse_rp_state_cap(dev_priv, rp_state_cap);
4560	Serge	3561
		3562	/* 2b: Program RC6 thresholds.*/
		3563	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		3564	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		3565	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		3566	for_each_ring(ring, dev_priv, unused)
		3567	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		3568	I915_WRITE(GEN6_RC_SLEEP, 0);
5060	serge	3569	if (IS_BROADWELL(dev))
		3570	I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
		3571	else
4560	Serge	3572	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
		3573
		3574	/* 3: Enable RC6 */
		3575	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		3576	rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
5060	serge	3577	intel_print_rc6_info(dev, rc6_mask);
		3578	if (IS_BROADWELL(dev))
		3579	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		3580	GEN7_RC_CTL_TO_MODE \|
		3581	rc6_mask);
		3582	else
4560	Serge	3583	I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE \|
		3584	GEN6_RC_CTL_EI_MODE(1) \|
		3585	rc6_mask);
		3586
		3587	/* 4 Program defaults and thresholds for RPS*/
5060	serge	3588	I915_WRITE(GEN6_RPNSWREQ,
		3589	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
		3590	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		3591	HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4560	Serge	3592	/* NB: Docs say 1s, and 1000000 - which aren't equivalent */
		3593	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
		3594
		3595	/* Docs recommend 900MHz, and 300 MHz respectively */
		3596	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
5060	serge	3597	dev_priv->rps.max_freq_softlimit << 24 \|
		3598	dev_priv->rps.min_freq_softlimit << 16);
4560	Serge	3599
		3600	I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
		3601	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
		3602	I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
		3603	I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
		3604
		3605	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		3606
		3607	/* 5: Enable RPS */
		3608	I915_WRITE(GEN6_RP_CONTROL,
		3609	GEN6_RP_MEDIA_TURBO \|
		3610	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		3611	GEN6_RP_MEDIA_IS_GFX \|
		3612	GEN6_RP_ENABLE \|
		3613	GEN6_RP_UP_BUSY_AVG \|
		3614	GEN6_RP_DOWN_IDLE_AVG);
		3615
		3616	/* 6: Ring frequency + overclocking (our driver does this later */
		3617
		3618	gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8);
		3619
5060	serge	3620	gen8_enable_rps_interrupts(dev);
4560	Serge	3621
		3622	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
		3623	}
		3624
3031	serge	3625	static void gen6_enable_rps(struct drm_device *dev)
		3626	{
		3627	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	3628	struct intel_engine_cs *ring;
3031	serge	3629	u32 rp_state_cap;
		3630	u32 gt_perf_status;
5060	serge	3631	u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
3031	serge	3632	u32 gtfifodbg;
		3633	int rc6_mode;
3243	Serge	3634	int i, ret;
3031	serge	3635
3243	Serge	3636	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3637
		3638	/* Here begins a magic sequence of register writes to enable
		3639	* auto-downclocking.
		3640	*
		3641	* Perhaps there might be some value in exposing these to
		3642	* userspace...
		3643	*/
		3644	I915_WRITE(GEN6_RC_STATE, 0);
		3645
		3646	/* Clear the DBG now so we don't confuse earlier errors */
		3647	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		3648	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		3649	I915_WRITE(GTFIFODBG, gtfifodbg);
		3650	}
		3651
4560	Serge	3652	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
3031	serge	3653
		3654	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		3655	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
		3656
5060	serge	3657	parse_rp_state_cap(dev_priv, rp_state_cap);
3031	serge	3658
		3659	/* disable the counters and set deterministic thresholds */
		3660	I915_WRITE(GEN6_RC_CONTROL, 0);
		3661
		3662	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		3663	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		3664	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		3665	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		3666	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		3667
		3668	for_each_ring(ring, dev_priv, i)
		3669	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		3670
		3671	I915_WRITE(GEN6_RC_SLEEP, 0);
		3672	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4560	Serge	3673	if (IS_IVYBRIDGE(dev))
4104	Serge	3674	I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
		3675	else
3031	serge	3676	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	3677	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	3678	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		3679
		3680	/* Check if we are enabling RC6 */
		3681	rc6_mode = intel_enable_rc6(dev_priv->dev);
		3682	if (rc6_mode & INTEL_RC6_ENABLE)
		3683	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		3684
		3685	/* We don't use those on Haswell */
		3686	if (!IS_HASWELL(dev)) {
		3687	if (rc6_mode & INTEL_RC6p_ENABLE)
		3688	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		3689
		3690	if (rc6_mode & INTEL_RC6pp_ENABLE)
		3691	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		3692	}
		3693
4560	Serge	3694	intel_print_rc6_info(dev, rc6_mask);
3031	serge	3695
		3696	I915_WRITE(GEN6_RC_CONTROL,
		3697	rc6_mask \|
		3698	GEN6_RC_CTL_EI_MODE(1) \|
		3699	GEN6_RC_CTL_HW_ENABLE);
		3700
4560	Serge	3701	/* Power down if completely idle for over 50ms */
		3702	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
3031	serge	3703	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		3704
3243	Serge	3705	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
5060	serge	3706	if (ret)
		3707	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		3708
3243	Serge	3709	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
3746	Serge	3710	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		3711	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
5060	serge	3712	(dev_priv->rps.max_freq_softlimit & 0xff) * 50,
3746	Serge	3713	(pcu_mbox & 0xff) * 50);
5060	serge	3714	dev_priv->rps.max_freq = pcu_mbox & 0xff;
3031	serge	3715	}
		3716
4560	Serge	3717	dev_priv->rps.power = HIGH_POWER; /* force a reset */
5060	serge	3718	gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
3031	serge	3719
4104	Serge	3720	gen6_enable_rps_interrupts(dev);
3031	serge	3721
3243	Serge	3722	rc6vids = 0;
		3723	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		3724	if (IS_GEN6(dev) && ret) {
		3725	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		3726	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		3727	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		3728	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		3729	rc6vids &= 0xffff00;
		3730	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		3731	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		3732	if (ret)
		3733	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		3734	}
		3735
4560	Serge	3736	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
3031	serge	3737	}
		3738
5060	serge	3739	static void __gen6_update_ring_freq(struct drm_device *dev)
3031	serge	3740	{
		3741	struct drm_i915_private *dev_priv = dev->dev_private;
		3742	int min_freq = 15;
3746	Serge	3743	unsigned int gpu_freq;
		3744	unsigned int max_ia_freq, min_ring_freq;
3031	serge	3745	int scaling_factor = 180;
4560	Serge	3746	struct cpufreq_policy *policy;
3031	serge	3747
3243	Serge	3748	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	3749
		3750	max_ia_freq = cpufreq_quick_get_max(0);
		3751	/*
5060	serge	3752	* Default to measured freq if none found, PCU will ensure we
		3753	* don't go over
3031	serge	3754	*/
		3755	max_ia_freq = tsc_khz;
		3756
		3757	/* Convert from kHz to MHz */
		3758	max_ia_freq /= 1000;
		3759
4560	Serge	3760	min_ring_freq = I915_READ(DCLK) & 0xf;
		3761	/* convert DDR frequency from units of 266.6MHz to bandwidth */
		3762	min_ring_freq = mult_frac(min_ring_freq, 8, 3);
3746	Serge	3763
3031	serge	3764	/*
		3765	* For each potential GPU frequency, load a ring frequency we'd like
		3766	* to use for memory access. We do this by specifying the IA frequency
		3767	* the PCU should use as a reference to determine the ring frequency.
		3768	*/
5060	serge	3769	for (gpu_freq = dev_priv->rps.max_freq_softlimit; gpu_freq >= dev_priv->rps.min_freq_softlimit;
3031	serge	3770	gpu_freq--) {
5060	serge	3771	int diff = dev_priv->rps.max_freq_softlimit - gpu_freq;
3746	Serge	3772	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	3773
4560	Serge	3774	if (INTEL_INFO(dev)->gen >= 8) {
		3775	/* max(2 * GT, DDR). NB: GT is 50MHz units */
		3776	ring_freq = max(min_ring_freq, gpu_freq);
		3777	} else if (IS_HASWELL(dev)) {
		3778	ring_freq = mult_frac(gpu_freq, 5, 4);
3746	Serge	3779	ring_freq = max(min_ring_freq, ring_freq);
		3780	/* leave ia_freq as the default, chosen by cpufreq */
		3781	} else {
		3782	/* On older processors, there is no separate ring
		3783	* clock domain, so in order to boost the bandwidth
		3784	* of the ring, we need to upclock the CPU (ia_freq).
		3785	*
		3786	* For GPU frequencies less than 750MHz,
		3787	* just use the lowest ring freq.
3031	serge	3788	*/
		3789	if (gpu_freq < min_freq)
		3790	ia_freq = 800;
		3791	else
		3792	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		3793	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	3794	}
3031	serge	3795
3243	Serge	3796	sandybridge_pcode_write(dev_priv,
		3797	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	3798	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		3799	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		3800	gpu_freq);
3031	serge	3801	}
		3802	}
		3803
5060	serge	3804	void gen6_update_ring_freq(struct drm_device *dev)
4104	Serge	3805	{
5060	serge	3806	struct drm_i915_private *dev_priv = dev->dev_private;
		3807
		3808	if (INTEL_INFO(dev)->gen < 6 \|\| IS_VALLEYVIEW(dev))
		3809	return;
		3810
		3811	mutex_lock(&dev_priv->rps.hw_lock);
		3812	__gen6_update_ring_freq(dev);
		3813	mutex_unlock(&dev_priv->rps.hw_lock);
		3814	}
		3815
		3816	static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
		3817	{
4104	Serge	3818	u32 val, rp0;
		3819
5060	serge	3820	val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
		3821	rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & PUNIT_GPU_STATUS_MAX_FREQ_MASK;
		3822
		3823	return rp0;
		3824	}
		3825
		3826	static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		3827	{
		3828	u32 val, rpe;
		3829
		3830	val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
		3831	rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
		3832
		3833	return rpe;
		3834	}
		3835
		3836	static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
		3837	{
		3838	u32 val, rp1;
		3839
		3840	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		3841	rp1 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) & PUNIT_GPU_STATUS_MAX_FREQ_MASK;
		3842
		3843	return rp1;
		3844	}
		3845
		3846	static int cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
		3847	{
		3848	u32 val, rpn;
		3849
		3850	val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
		3851	rpn = (val >> PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT) & PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK;
		3852	return rpn;
		3853	}
		3854
		3855	static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
		3856	{
		3857	u32 val, rp1;
		3858
4104	Serge	3859	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		3860
5060	serge	3861	rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
		3862
		3863	return rp1;
		3864	}
		3865
		3866	static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
		3867	{
		3868	u32 val, rp0;
		3869
		3870	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
		3871
4104	Serge	3872	rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
		3873	/* Clamp to max */
		3874	rp0 = min_t(u32, rp0, 0xea);
		3875
		3876	return rp0;
		3877	}
		3878
		3879	static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
		3880	{
		3881	u32 val, rpe;
		3882
		3883	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
		3884	rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
		3885	val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
		3886	rpe \|= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
		3887
		3888	return rpe;
		3889	}
		3890
5060	serge	3891	static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
4104	Serge	3892	{
		3893	return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
		3894	}
		3895
5060	serge	3896	/* Check that the pctx buffer wasn't move under us. */
		3897	static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
		3898	{
		3899	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		3900
		3901	WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
		3902	dev_priv->vlv_pctx->stolen->start);
		3903	}
		3904
		3905
		3906	/* Check that the pcbr address is not empty. */
		3907	static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
		3908	{
		3909	unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
		3910
		3911	WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
		3912	}
		3913
		3914	static void cherryview_setup_pctx(struct drm_device *dev)
		3915	{
		3916	struct drm_i915_private *dev_priv = dev->dev_private;
		3917	unsigned long pctx_paddr, paddr;
		3918	struct i915_gtt *gtt = &dev_priv->gtt;
		3919	u32 pcbr;
		3920	int pctx_size = 32*1024;
		3921
		3922	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		3923
		3924	pcbr = I915_READ(VLV_PCBR);
		3925	if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
		3926	paddr = (dev_priv->mm.stolen_base +
		3927	(gtt->stolen_size - pctx_size));
		3928
		3929	pctx_paddr = (paddr & (~4095));
		3930	I915_WRITE(VLV_PCBR, pctx_paddr);
		3931	}
		3932	}
		3933
4104	Serge	3934	static void valleyview_setup_pctx(struct drm_device *dev)
		3935	{
		3936	struct drm_i915_private *dev_priv = dev->dev_private;
		3937	struct drm_i915_gem_object *pctx;
		3938	unsigned long pctx_paddr;
		3939	u32 pcbr;
		3940	int pctx_size = 24*1024;
		3941
5060	serge	3942	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		3943
4104	Serge	3944	pcbr = I915_READ(VLV_PCBR);
		3945	if (pcbr) {
		3946	/* BIOS set it up already, grab the pre-alloc'd space */
		3947	int pcbr_offset;
		3948
		3949	pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
		3950	pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
		3951	pcbr_offset,
		3952	I915_GTT_OFFSET_NONE,
		3953	pctx_size);
		3954	goto out;
		3955	}
		3956
		3957	/*
		3958	* From the Gunit register HAS:
		3959	* The Gfx driver is expected to program this register and ensure
		3960	* proper allocation within Gfx stolen memory. For example, this
		3961	* register should be programmed such than the PCBR range does not
		3962	* overlap with other ranges, such as the frame buffer, protected
		3963	* memory, or any other relevant ranges.
		3964	*/
		3965	pctx = i915_gem_object_create_stolen(dev, pctx_size);
		3966	if (!pctx) {
		3967	DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
		3968	return;
		3969	}
		3970
		3971	pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
		3972	I915_WRITE(VLV_PCBR, pctx_paddr);
		3973
		3974	out:
		3975	dev_priv->vlv_pctx = pctx;
		3976	}
		3977
5060	serge	3978	static void valleyview_cleanup_pctx(struct drm_device *dev)
		3979	{
		3980	struct drm_i915_private *dev_priv = dev->dev_private;
		3981
		3982	if (WARN_ON(!dev_priv->vlv_pctx))
		3983	return;
		3984
		3985	drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
		3986	dev_priv->vlv_pctx = NULL;
		3987	}
		3988
		3989	static void valleyview_init_gt_powersave(struct drm_device *dev)
		3990	{
		3991	struct drm_i915_private *dev_priv = dev->dev_private;
		3992
		3993	valleyview_setup_pctx(dev);
		3994
		3995	mutex_lock(&dev_priv->rps.hw_lock);
		3996
		3997	dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
		3998	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		3999	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
		4000	vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq),
		4001	dev_priv->rps.max_freq);
		4002
		4003	dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
		4004	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
		4005	vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
		4006	dev_priv->rps.efficient_freq);
		4007
		4008	dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
		4009	DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
		4010	vlv_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
		4011	dev_priv->rps.rp1_freq);
		4012
		4013	dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
		4014	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
		4015	vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq),
		4016	dev_priv->rps.min_freq);
		4017
		4018	/* Preserve min/max settings in case of re-init */
		4019	if (dev_priv->rps.max_freq_softlimit == 0)
		4020	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		4021
		4022	if (dev_priv->rps.min_freq_softlimit == 0)
		4023	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		4024
		4025	mutex_unlock(&dev_priv->rps.hw_lock);
		4026	}
		4027
		4028	static void cherryview_init_gt_powersave(struct drm_device *dev)
		4029	{
		4030	struct drm_i915_private *dev_priv = dev->dev_private;
		4031
		4032	cherryview_setup_pctx(dev);
		4033
		4034	mutex_lock(&dev_priv->rps.hw_lock);
		4035
		4036	dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
		4037	dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
		4038	DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
		4039	vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq),
		4040	dev_priv->rps.max_freq);
		4041
		4042	dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
		4043	DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
		4044	vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
		4045	dev_priv->rps.efficient_freq);
		4046
		4047	dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
		4048	DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
		4049	vlv_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
		4050	dev_priv->rps.rp1_freq);
		4051
		4052	dev_priv->rps.min_freq = cherryview_rps_min_freq(dev_priv);
		4053	DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
		4054	vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq),
		4055	dev_priv->rps.min_freq);
		4056
		4057	/* Preserve min/max settings in case of re-init */
		4058	if (dev_priv->rps.max_freq_softlimit == 0)
		4059	dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
		4060
		4061	if (dev_priv->rps.min_freq_softlimit == 0)
		4062	dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
		4063
		4064	mutex_unlock(&dev_priv->rps.hw_lock);
		4065	}
		4066
		4067	static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
		4068	{
		4069	valleyview_cleanup_pctx(dev);
		4070	}
		4071
		4072	static void cherryview_enable_rps(struct drm_device *dev)
		4073	{
		4074	struct drm_i915_private *dev_priv = dev->dev_private;
		4075	struct intel_engine_cs *ring;
		4076	u32 gtfifodbg, val, rc6_mode = 0, pcbr;
		4077	int i;
		4078
		4079	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4080
		4081	gtfifodbg = I915_READ(GTFIFODBG);
		4082	if (gtfifodbg) {
		4083	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		4084	gtfifodbg);
		4085	I915_WRITE(GTFIFODBG, gtfifodbg);
		4086	}
		4087
		4088	cherryview_check_pctx(dev_priv);
		4089
		4090	/* 1a & 1b: Get forcewake during program sequence. Although the driver
		4091	* hasn't enabled a state yet where we need forcewake, BIOS may have.*/
		4092	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
		4093
		4094	/* 2a: Program RC6 thresholds.*/
		4095	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
		4096	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
		4097	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
		4098
		4099	for_each_ring(ring, dev_priv, i)
		4100	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4101	I915_WRITE(GEN6_RC_SLEEP, 0);
		4102
		4103	I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
		4104
		4105	/* allows RC6 residency counter to work */
		4106	I915_WRITE(VLV_COUNTER_CONTROL,
		4107	_MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH \|
		4108	VLV_MEDIA_RC6_COUNT_EN \|
		4109	VLV_RENDER_RC6_COUNT_EN));
		4110
		4111	/* For now we assume BIOS is allocating and populating the PCBR */
		4112	pcbr = I915_READ(VLV_PCBR);
		4113
		4114	DRM_DEBUG_DRIVER("PCBR offset : 0x%x\n", pcbr);
		4115
		4116	/* 3: Enable RC6 */
		4117	if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
		4118	(pcbr >> VLV_PCBR_ADDR_SHIFT))
		4119	rc6_mode = GEN6_RC_CTL_EI_MODE(1);
		4120
		4121	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		4122
		4123	/* 4 Program defaults and thresholds for RPS*/
		4124	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		4125	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		4126	I915_WRITE(GEN6_RP_UP_EI, 66000);
		4127	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		4128
		4129	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		4130
		4131	/* WaDisablePwrmtrEvent:chv (pre-production hw) */
		4132	I915_WRITE(0xA80C, I915_READ(0xA80C) & 0x00ffffff);
		4133	I915_WRITE(0xA810, I915_READ(0xA810) & 0xffffff00);
		4134
		4135	/* 5: Enable RPS */
		4136	I915_WRITE(GEN6_RP_CONTROL,
		4137	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4138	GEN6_RP_MEDIA_IS_GFX \| /* WaSetMaskForGfxBusyness:chv (pre-production hw ?) */
		4139	GEN6_RP_ENABLE \|
		4140	GEN6_RP_UP_BUSY_AVG \|
		4141	GEN6_RP_DOWN_IDLE_AVG);
		4142
		4143	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		4144
		4145	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
		4146	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		4147
		4148	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
		4149	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
		4150	vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
		4151	dev_priv->rps.cur_freq);
		4152
		4153	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
		4154	vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
		4155	dev_priv->rps.efficient_freq);
		4156
		4157	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
		4158
		4159	gen8_enable_rps_interrupts(dev);
		4160
		4161	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
		4162	}
		4163
4104	Serge	4164	static void valleyview_enable_rps(struct drm_device *dev)
		4165	{
		4166	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4167	struct intel_engine_cs *ring;
4560	Serge	4168	u32 gtfifodbg, val, rc6_mode = 0;
4104	Serge	4169	int i;
		4170
		4171	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4172
5060	serge	4173	valleyview_check_pctx(dev_priv);
		4174
4104	Serge	4175	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4560	Serge	4176	DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
		4177	gtfifodbg);
4104	Serge	4178	I915_WRITE(GTFIFODBG, gtfifodbg);
		4179	}
		4180
4560	Serge	4181	/* If VLV, Forcewake all wells, else re-direct to regular path */
		4182	gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
4104	Serge	4183
		4184	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		4185	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		4186	I915_WRITE(GEN6_RP_UP_EI, 66000);
		4187	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		4188
		4189	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5060	serge	4190	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 0xf4240);
4104	Serge	4191
		4192	I915_WRITE(GEN6_RP_CONTROL,
		4193	GEN6_RP_MEDIA_TURBO \|
		4194	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		4195	GEN6_RP_MEDIA_IS_GFX \|
		4196	GEN6_RP_ENABLE \|
		4197	GEN6_RP_UP_BUSY_AVG \|
		4198	GEN6_RP_DOWN_IDLE_CONT);
		4199
		4200	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
		4201	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		4202	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		4203
		4204	for_each_ring(ring, dev_priv, i)
		4205	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		4206
4560	Serge	4207	I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4104	Serge	4208
		4209	/* allows RC6 residency counter to work */
4560	Serge	4210	I915_WRITE(VLV_COUNTER_CONTROL,
5060	serge	4211	_MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN \|
		4212	VLV_RENDER_RC0_COUNT_EN \|
4560	Serge	4213	VLV_MEDIA_RC6_COUNT_EN \|
		4214	VLV_RENDER_RC6_COUNT_EN));
5060	serge	4215
4560	Serge	4216	if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
		4217	rc6_mode = GEN7_RC_CTL_TO_MODE \| VLV_RC_CTL_CTX_RST_PARALLEL;
4104	Serge	4218
4560	Serge	4219	intel_print_rc6_info(dev, rc6_mode);
		4220
		4221	I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
		4222
4104	Serge	4223	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		4224
		4225	DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
		4226	DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
		4227
5060	serge	4228	dev_priv->rps.cur_freq = (val >> 8) & 0xff;
4104	Serge	4229	DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
5060	serge	4230	vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
		4231	dev_priv->rps.cur_freq);
4104	Serge	4232
		4233	DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
5060	serge	4234	vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
		4235	dev_priv->rps.efficient_freq);
4104	Serge	4236
5060	serge	4237	valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
4104	Serge	4238
		4239	gen6_enable_rps_interrupts(dev);
		4240
4560	Serge	4241	gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
4104	Serge	4242	}
		4243
3031	serge	4244	void ironlake_teardown_rc6(struct drm_device *dev)
		4245	{
		4246	struct drm_i915_private *dev_priv = dev->dev_private;
		4247
3243	Serge	4248	if (dev_priv->ips.renderctx) {
5060	serge	4249	i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx);
3243	Serge	4250	drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
		4251	dev_priv->ips.renderctx = NULL;
3031	serge	4252	}
		4253
3243	Serge	4254	if (dev_priv->ips.pwrctx) {
5060	serge	4255	i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx);
3243	Serge	4256	drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
		4257	dev_priv->ips.pwrctx = NULL;
3031	serge	4258	}
		4259	}
		4260
		4261	static void ironlake_disable_rc6(struct drm_device *dev)
		4262	{
		4263	struct drm_i915_private *dev_priv = dev->dev_private;
		4264
		4265	if (I915_READ(PWRCTXA)) {
		4266	/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
		4267	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) \| RCX_SW_EXIT);
		4268	wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
		4269	50);
		4270
		4271	I915_WRITE(PWRCTXA, 0);
		4272	POSTING_READ(PWRCTXA);
		4273
		4274	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		4275	POSTING_READ(RSTDBYCTL);
		4276	}
		4277	}
		4278
		4279	static int ironlake_setup_rc6(struct drm_device *dev)
		4280	{
		4281	struct drm_i915_private *dev_priv = dev->dev_private;
		4282
3243	Serge	4283	if (dev_priv->ips.renderctx == NULL)
		4284	dev_priv->ips.renderctx = intel_alloc_context_page(dev);
		4285	if (!dev_priv->ips.renderctx)
3031	serge	4286	return -ENOMEM;
		4287
3243	Serge	4288	if (dev_priv->ips.pwrctx == NULL)
		4289	dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
		4290	if (!dev_priv->ips.pwrctx) {
3031	serge	4291	ironlake_teardown_rc6(dev);
		4292	return -ENOMEM;
		4293	}
		4294
		4295	return 0;
		4296	}
		4297
		4298	static void ironlake_enable_rc6(struct drm_device *dev)
		4299	{
		4300	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	4301	struct intel_engine_cs *ring = &dev_priv->ring[RCS];
3243	Serge	4302	bool was_interruptible;
3031	serge	4303	int ret;
		4304
		4305	/* rc6 disabled by default due to repeated reports of hanging during
		4306	* boot and resume.
		4307	*/
		4308	if (!intel_enable_rc6(dev))
		4309	return;
		4310
		4311	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		4312
		4313	ret = ironlake_setup_rc6(dev);
		4314	if (ret)
		4315	return;
		4316
3243	Serge	4317	was_interruptible = dev_priv->mm.interruptible;
		4318	dev_priv->mm.interruptible = false;
		4319
3031	serge	4320	/*
		4321	* GPU can automatically power down the render unit if given a page
		4322	* to save state.
		4323	*/
		4324	ret = intel_ring_begin(ring, 6);
		4325	if (ret) {
		4326	ironlake_teardown_rc6(dev);
3243	Serge	4327	dev_priv->mm.interruptible = was_interruptible;
3031	serge	4328	return;
		4329	}
		4330
		4331	intel_ring_emit(ring, MI_SUSPEND_FLUSH \| MI_SUSPEND_FLUSH_EN);
		4332	intel_ring_emit(ring, MI_SET_CONTEXT);
4104	Serge	4333	intel_ring_emit(ring, i915_gem_obj_ggtt_offset(dev_priv->ips.renderctx) \|
3031	serge	4334	MI_MM_SPACE_GTT \|
		4335	MI_SAVE_EXT_STATE_EN \|
		4336	MI_RESTORE_EXT_STATE_EN \|
		4337	MI_RESTORE_INHIBIT);
		4338	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
		4339	intel_ring_emit(ring, MI_NOOP);
		4340	intel_ring_emit(ring, MI_FLUSH);
		4341	intel_ring_advance(ring);
		4342
		4343	/*
		4344	* Wait for the command parser to advance past MI_SET_CONTEXT. The HW
		4345	* does an implicit flush, combined with MI_FLUSH above, it should be
		4346	* safe to assume that renderctx is valid
		4347	*/
3243	Serge	4348	ret = intel_ring_idle(ring);
		4349	dev_priv->mm.interruptible = was_interruptible;
3031	serge	4350	if (ret) {
3746	Serge	4351	DRM_ERROR("failed to enable ironlake power savings\n");
3031	serge	4352	ironlake_teardown_rc6(dev);
		4353	return;
		4354	}
		4355
4104	Serge	4356	I915_WRITE(PWRCTXA, i915_gem_obj_ggtt_offset(dev_priv->ips.pwrctx) \| PWRCTX_EN);
3031	serge	4357	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
4560	Serge	4358
5060	serge	4359	intel_print_rc6_info(dev, GEN6_RC_CTL_RC6_ENABLE);
3031	serge	4360	}
		4361
		4362	static unsigned long intel_pxfreq(u32 vidfreq)
		4363	{
		4364	unsigned long freq;
		4365	int div = (vidfreq & 0x3f0000) >> 16;
		4366	int post = (vidfreq & 0x3000) >> 12;
		4367	int pre = (vidfreq & 0x7);
		4368
		4369	if (!pre)
		4370	return 0;
		4371
		4372	freq = ((div * 133333) / ((1<
		4373
		4374	return freq;
		4375	}
		4376
		4377	static const struct cparams {
		4378	u16 i;
		4379	u16 t;
		4380	u16 m;
		4381	u16 c;
		4382	} cparams[] = {
		4383	{ 1, 1333, 301, 28664 },
		4384	{ 1, 1066, 294, 24460 },
		4385	{ 1, 800, 294, 25192 },
		4386	{ 0, 1333, 276, 27605 },
		4387	{ 0, 1066, 276, 27605 },
		4388	{ 0, 800, 231, 23784 },
		4389	};
		4390
		4391	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		4392	{
		4393	u64 total_count, diff, ret;
		4394	u32 count1, count2, count3, m = 0, c = 0;
5060	serge	4395	unsigned long now = jiffies_to_msecs(jiffies), diff1;
3031	serge	4396	int i;
		4397
		4398	assert_spin_locked(&mchdev_lock);
		4399
		4400	diff1 = now - dev_priv->ips.last_time1;
		4401
		4402	/* Prevent division-by-zero if we are asking too fast.
		4403	* Also, we don't get interesting results if we are polling
		4404	* faster than once in 10ms, so just return the saved value
		4405	* in such cases.
		4406	*/
		4407	if (diff1 <= 10)
		4408	return dev_priv->ips.chipset_power;
		4409
		4410	count1 = I915_READ(DMIEC);
		4411	count2 = I915_READ(DDREC);
		4412	count3 = I915_READ(CSIEC);
		4413
		4414	total_count = count1 + count2 + count3;
		4415
		4416	/* FIXME: handle per-counter overflow */
		4417	if (total_count < dev_priv->ips.last_count1) {
		4418	diff = ~0UL - dev_priv->ips.last_count1;
		4419	diff += total_count;
		4420	} else {
		4421	diff = total_count - dev_priv->ips.last_count1;
		4422	}
		4423
		4424	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		4425	if (cparams[i].i == dev_priv->ips.c_m &&
		4426	cparams[i].t == dev_priv->ips.r_t) {
		4427	m = cparams[i].m;
		4428	c = cparams[i].c;
		4429	break;
		4430	}
		4431	}
		4432
		4433	diff = div_u64(diff, diff1);
		4434	ret = ((m * diff) + c);
		4435	ret = div_u64(ret, 10);
		4436
		4437	dev_priv->ips.last_count1 = total_count;
		4438	dev_priv->ips.last_time1 = now;
		4439
		4440	dev_priv->ips.chipset_power = ret;
		4441
		4442	return ret;
		4443	}
		4444
		4445	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		4446	{
5060	serge	4447	struct drm_device *dev = dev_priv->dev;
3031	serge	4448	unsigned long val;
		4449
5060	serge	4450	if (INTEL_INFO(dev)->gen != 5)
3031	serge	4451	return 0;
		4452
		4453	spin_lock_irq(&mchdev_lock);
		4454
		4455	val = __i915_chipset_val(dev_priv);
		4456
		4457	spin_unlock_irq(&mchdev_lock);
		4458
		4459	return val;
		4460	}
		4461
		4462	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		4463	{
		4464	unsigned long m, x, b;
		4465	u32 tsfs;
		4466
		4467	tsfs = I915_READ(TSFS);
		4468
		4469	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		4470	x = I915_READ8(TR1);
		4471
		4472	b = tsfs & TSFS_INTR_MASK;
		4473
		4474	return ((m * x) / 127) - b;
		4475	}
		4476
		4477	static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		4478	{
5060	serge	4479	struct drm_device *dev = dev_priv->dev;
3031	serge	4480	static const struct v_table {
		4481	u16 vd; /* in .1 mil */
		4482	u16 vm; /* in .1 mil */
		4483	} v_table[] = {
		4484	{ 0, 0, },
		4485	{ 375, 0, },
		4486	{ 500, 0, },
		4487	{ 625, 0, },
		4488	{ 750, 0, },
		4489	{ 875, 0, },
		4490	{ 1000, 0, },
		4491	{ 1125, 0, },
		4492	{ 4125, 3000, },
		4493	{ 4125, 3000, },
		4494	{ 4125, 3000, },
		4495	{ 4125, 3000, },
		4496	{ 4125, 3000, },
		4497	{ 4125, 3000, },
		4498	{ 4125, 3000, },
		4499	{ 4125, 3000, },
		4500	{ 4125, 3000, },
		4501	{ 4125, 3000, },
		4502	{ 4125, 3000, },
		4503	{ 4125, 3000, },
		4504	{ 4125, 3000, },
		4505	{ 4125, 3000, },
		4506	{ 4125, 3000, },
		4507	{ 4125, 3000, },
		4508	{ 4125, 3000, },
		4509	{ 4125, 3000, },
		4510	{ 4125, 3000, },
		4511	{ 4125, 3000, },
		4512	{ 4125, 3000, },
		4513	{ 4125, 3000, },
		4514	{ 4125, 3000, },
		4515	{ 4125, 3000, },
		4516	{ 4250, 3125, },
		4517	{ 4375, 3250, },
		4518	{ 4500, 3375, },
		4519	{ 4625, 3500, },
		4520	{ 4750, 3625, },
		4521	{ 4875, 3750, },
		4522	{ 5000, 3875, },
		4523	{ 5125, 4000, },
		4524	{ 5250, 4125, },
		4525	{ 5375, 4250, },
		4526	{ 5500, 4375, },
		4527	{ 5625, 4500, },
		4528	{ 5750, 4625, },
		4529	{ 5875, 4750, },
		4530	{ 6000, 4875, },
		4531	{ 6125, 5000, },
		4532	{ 6250, 5125, },
		4533	{ 6375, 5250, },
		4534	{ 6500, 5375, },
		4535	{ 6625, 5500, },
		4536	{ 6750, 5625, },
		4537	{ 6875, 5750, },
		4538	{ 7000, 5875, },
		4539	{ 7125, 6000, },
		4540	{ 7250, 6125, },
		4541	{ 7375, 6250, },
		4542	{ 7500, 6375, },
		4543	{ 7625, 6500, },
		4544	{ 7750, 6625, },
		4545	{ 7875, 6750, },
		4546	{ 8000, 6875, },
		4547	{ 8125, 7000, },
		4548	{ 8250, 7125, },
		4549	{ 8375, 7250, },
		4550	{ 8500, 7375, },
		4551	{ 8625, 7500, },
		4552	{ 8750, 7625, },
		4553	{ 8875, 7750, },
		4554	{ 9000, 7875, },
		4555	{ 9125, 8000, },
		4556	{ 9250, 8125, },
		4557	{ 9375, 8250, },
		4558	{ 9500, 8375, },
		4559	{ 9625, 8500, },
		4560	{ 9750, 8625, },
		4561	{ 9875, 8750, },
		4562	{ 10000, 8875, },
		4563	{ 10125, 9000, },
		4564	{ 10250, 9125, },
		4565	{ 10375, 9250, },
		4566	{ 10500, 9375, },
		4567	{ 10625, 9500, },
		4568	{ 10750, 9625, },
		4569	{ 10875, 9750, },
		4570	{ 11000, 9875, },
		4571	{ 11125, 10000, },
		4572	{ 11250, 10125, },
		4573	{ 11375, 10250, },
		4574	{ 11500, 10375, },
		4575	{ 11625, 10500, },
		4576	{ 11750, 10625, },
		4577	{ 11875, 10750, },
		4578	{ 12000, 10875, },
		4579	{ 12125, 11000, },
		4580	{ 12250, 11125, },
		4581	{ 12375, 11250, },
		4582	{ 12500, 11375, },
		4583	{ 12625, 11500, },
		4584	{ 12750, 11625, },
		4585	{ 12875, 11750, },
		4586	{ 13000, 11875, },
		4587	{ 13125, 12000, },
		4588	{ 13250, 12125, },
		4589	{ 13375, 12250, },
		4590	{ 13500, 12375, },
		4591	{ 13625, 12500, },
		4592	{ 13750, 12625, },
		4593	{ 13875, 12750, },
		4594	{ 14000, 12875, },
		4595	{ 14125, 13000, },
		4596	{ 14250, 13125, },
		4597	{ 14375, 13250, },
		4598	{ 14500, 13375, },
		4599	{ 14625, 13500, },
		4600	{ 14750, 13625, },
		4601	{ 14875, 13750, },
		4602	{ 15000, 13875, },
		4603	{ 15125, 14000, },
		4604	{ 15250, 14125, },
		4605	{ 15375, 14250, },
		4606	{ 15500, 14375, },
		4607	{ 15625, 14500, },
		4608	{ 15750, 14625, },
		4609	{ 15875, 14750, },
		4610	{ 16000, 14875, },
		4611	{ 16125, 15000, },
		4612	};
5060	serge	4613	if (INTEL_INFO(dev)->is_mobile)
3031	serge	4614	return v_table[pxvid].vm;
		4615	else
		4616	return v_table[pxvid].vd;
		4617	}
		4618
		4619	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		4620	{
5060	serge	4621	u64 now, diff, diffms;
3031	serge	4622	u32 count;
		4623
		4624	assert_spin_locked(&mchdev_lock);
		4625
5060	serge	4626	now = ktime_get_raw_ns();
		4627	diffms = now - dev_priv->ips.last_time2;
		4628	do_div(diffms, NSEC_PER_MSEC);
3031	serge	4629
		4630	/* Don't divide by 0 */
		4631	if (!diffms)
		4632	return;
		4633
		4634	count = I915_READ(GFXEC);
		4635
		4636	if (count < dev_priv->ips.last_count2) {
		4637	diff = ~0UL - dev_priv->ips.last_count2;
		4638	diff += count;
		4639	} else {
		4640	diff = count - dev_priv->ips.last_count2;
		4641	}
		4642
		4643	dev_priv->ips.last_count2 = count;
		4644	dev_priv->ips.last_time2 = now;
		4645
		4646	/* More magic constants... */
		4647	diff = diff * 1181;
		4648	diff = div_u64(diff, diffms * 10);
		4649	dev_priv->ips.gfx_power = diff;
		4650	}
		4651
		4652	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		4653	{
5060	serge	4654	struct drm_device *dev = dev_priv->dev;
		4655
		4656	if (INTEL_INFO(dev)->gen != 5)
3031	serge	4657	return;
		4658
		4659	spin_lock_irq(&mchdev_lock);
		4660
		4661	__i915_update_gfx_val(dev_priv);
		4662
		4663	spin_unlock_irq(&mchdev_lock);
		4664	}
		4665
		4666	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		4667	{
		4668	unsigned long t, corr, state1, corr2, state2;
		4669	u32 pxvid, ext_v;
		4670
		4671	assert_spin_locked(&mchdev_lock);
		4672
5060	serge	4673	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4));
3031	serge	4674	pxvid = (pxvid >> 24) & 0x7f;
		4675	ext_v = pvid_to_extvid(dev_priv, pxvid);
		4676
		4677	state1 = ext_v;
		4678
		4679	t = i915_mch_val(dev_priv);
		4680
		4681	/* Revel in the empirically derived constants */
		4682
		4683	/* Correction factor in 1/100000 units */
		4684	if (t > 80)
		4685	corr = ((t * 2349) + 135940);
		4686	else if (t >= 50)
		4687	corr = ((t * 964) + 29317);
		4688	else /* < 50 */
		4689	corr = ((t * 301) + 1004);
		4690
		4691	corr = corr * ((150142 * state1) / 10000 - 78642);
		4692	corr /= 100000;
		4693	corr2 = (corr * dev_priv->ips.corr);
		4694
		4695	state2 = (corr2 * state1) / 10000;
		4696	state2 /= 100; /* convert to mW */
		4697
		4698	__i915_update_gfx_val(dev_priv);
		4699
		4700	return dev_priv->ips.gfx_power + state2;
		4701	}
		4702
		4703	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		4704	{
5060	serge	4705	struct drm_device *dev = dev_priv->dev;
3031	serge	4706	unsigned long val;
		4707
5060	serge	4708	if (INTEL_INFO(dev)->gen != 5)
3031	serge	4709	return 0;
		4710
		4711	spin_lock_irq(&mchdev_lock);
		4712
		4713	val = __i915_gfx_val(dev_priv);
		4714
		4715	spin_unlock_irq(&mchdev_lock);
		4716
		4717	return val;
		4718	}
		4719
		4720	/**
		4721	* i915_read_mch_val - return value for IPS use
		4722	*
		4723	* Calculate and return a value for the IPS driver to use when deciding whether
		4724	* we have thermal and power headroom to increase CPU or GPU power budget.
		4725	*/
		4726	unsigned long i915_read_mch_val(void)
		4727	{
		4728	struct drm_i915_private *dev_priv;
		4729	unsigned long chipset_val, graphics_val, ret = 0;
		4730
		4731	spin_lock_irq(&mchdev_lock);
		4732	if (!i915_mch_dev)
		4733	goto out_unlock;
		4734	dev_priv = i915_mch_dev;
		4735
		4736	chipset_val = __i915_chipset_val(dev_priv);
		4737	graphics_val = __i915_gfx_val(dev_priv);
		4738
		4739	ret = chipset_val + graphics_val;
		4740
		4741	out_unlock:
		4742	spin_unlock_irq(&mchdev_lock);
		4743
		4744	return ret;
		4745	}
		4746	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		4747
		4748	/**
		4749	* i915_gpu_raise - raise GPU frequency limit
		4750	*
		4751	* Raise the limit; IPS indicates we have thermal headroom.
		4752	*/
		4753	bool i915_gpu_raise(void)
		4754	{
		4755	struct drm_i915_private *dev_priv;
		4756	bool ret = true;
		4757
		4758	spin_lock_irq(&mchdev_lock);
		4759	if (!i915_mch_dev) {
		4760	ret = false;
		4761	goto out_unlock;
		4762	}
		4763	dev_priv = i915_mch_dev;
		4764
		4765	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		4766	dev_priv->ips.max_delay--;
		4767
		4768	out_unlock:
		4769	spin_unlock_irq(&mchdev_lock);
		4770
		4771	return ret;
		4772	}
		4773	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		4774
		4775	/**
		4776	* i915_gpu_lower - lower GPU frequency limit
		4777	*
		4778	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		4779	* frequency maximum.
		4780	*/
		4781	bool i915_gpu_lower(void)
		4782	{
		4783	struct drm_i915_private *dev_priv;
		4784	bool ret = true;
		4785
		4786	spin_lock_irq(&mchdev_lock);
		4787	if (!i915_mch_dev) {
		4788	ret = false;
		4789	goto out_unlock;
		4790	}
		4791	dev_priv = i915_mch_dev;
		4792
		4793	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		4794	dev_priv->ips.max_delay++;
		4795
		4796	out_unlock:
		4797	spin_unlock_irq(&mchdev_lock);
		4798
		4799	return ret;
		4800	}
		4801	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		4802
		4803	/**
		4804	* i915_gpu_busy - indicate GPU business to IPS
		4805	*
		4806	* Tell the IPS driver whether or not the GPU is busy.
		4807	*/
		4808	bool i915_gpu_busy(void)
		4809	{
		4810	struct drm_i915_private *dev_priv;
5060	serge	4811	struct intel_engine_cs *ring;
3031	serge	4812	bool ret = false;
		4813	int i;
		4814
		4815	spin_lock_irq(&mchdev_lock);
		4816	if (!i915_mch_dev)
		4817	goto out_unlock;
		4818	dev_priv = i915_mch_dev;
		4819
		4820	for_each_ring(ring, dev_priv, i)
		4821	ret \|= !list_empty(&ring->request_list);
		4822
		4823	out_unlock:
		4824	spin_unlock_irq(&mchdev_lock);
		4825
		4826	return ret;
		4827	}
		4828	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		4829
		4830	/**
		4831	* i915_gpu_turbo_disable - disable graphics turbo
		4832	*
		4833	* Disable graphics turbo by resetting the max frequency and setting the
		4834	* current frequency to the default.
		4835	*/
		4836	bool i915_gpu_turbo_disable(void)
		4837	{
		4838	struct drm_i915_private *dev_priv;
		4839	bool ret = true;
		4840
		4841	spin_lock_irq(&mchdev_lock);
		4842	if (!i915_mch_dev) {
		4843	ret = false;
		4844	goto out_unlock;
		4845	}
		4846	dev_priv = i915_mch_dev;
		4847
		4848	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		4849
		4850	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		4851	ret = false;
		4852
		4853	out_unlock:
		4854	spin_unlock_irq(&mchdev_lock);
		4855
		4856	return ret;
		4857	}
		4858	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		4859
		4860	/**
		4861	* Tells the intel_ips driver that the i915 driver is now loaded, if
		4862	* IPS got loaded first.
		4863	*
		4864	* This awkward dance is so that neither module has to depend on the
		4865	* other in order for IPS to do the appropriate communication of
		4866	* GPU turbo limits to i915.
		4867	*/
		4868	static void
		4869	ips_ping_for_i915_load(void)
		4870	{
		4871	void (*link)(void);
		4872
		4873	// link = symbol_get(ips_link_to_i915_driver);
		4874	// if (link) {
		4875	// link();
		4876	// symbol_put(ips_link_to_i915_driver);
		4877	// }
		4878	}
		4879
		4880	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		4881	{
		4882	/* We only register the i915 ips part with intel-ips once everything is
		4883	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		4884	spin_lock_irq(&mchdev_lock);
		4885	i915_mch_dev = dev_priv;
		4886	spin_unlock_irq(&mchdev_lock);
		4887
		4888	ips_ping_for_i915_load();
		4889	}
		4890
		4891	void intel_gpu_ips_teardown(void)
		4892	{
		4893	spin_lock_irq(&mchdev_lock);
		4894	i915_mch_dev = NULL;
		4895	spin_unlock_irq(&mchdev_lock);
		4896	}
5060	serge	4897
3031	serge	4898	static void intel_init_emon(struct drm_device *dev)
		4899	{
		4900	struct drm_i915_private *dev_priv = dev->dev_private;
		4901	u32 lcfuse;
		4902	u8 pxw[16];
		4903	int i;
		4904
		4905	/* Disable to program */
		4906	I915_WRITE(ECR, 0);
		4907	POSTING_READ(ECR);
		4908
		4909	/* Program energy weights for various events */
		4910	I915_WRITE(SDEW, 0x15040d00);
		4911	I915_WRITE(CSIEW0, 0x007f0000);
		4912	I915_WRITE(CSIEW1, 0x1e220004);
		4913	I915_WRITE(CSIEW2, 0x04000004);
		4914
		4915	for (i = 0; i < 5; i++)
		4916	I915_WRITE(PEW + (i * 4), 0);
		4917	for (i = 0; i < 3; i++)
		4918	I915_WRITE(DEW + (i * 4), 0);
		4919
		4920	/* Program P-state weights to account for frequency power adjustment */
		4921	for (i = 0; i < 16; i++) {
		4922	u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
		4923	unsigned long freq = intel_pxfreq(pxvidfreq);
		4924	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		4925	PXVFREQ_PX_SHIFT;
		4926	unsigned long val;
		4927
		4928	val = vid * vid;
		4929	val *= (freq / 1000);
		4930	val *= 255;
		4931	val /= (127127900);
		4932	if (val > 0xff)
		4933	DRM_ERROR("bad pxval: %ld\n", val);
		4934	pxw[i] = val;
		4935	}
		4936	/* Render standby states get 0 weight */
		4937	pxw[14] = 0;
		4938	pxw[15] = 0;
		4939
		4940	for (i = 0; i < 4; i++) {
		4941	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		4942	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
		4943	I915_WRITE(PXW + (i * 4), val);
		4944	}
		4945
		4946	/* Adjust magic regs to magic values (more experimental results) */
		4947	I915_WRITE(OGW0, 0);
		4948	I915_WRITE(OGW1, 0);
		4949	I915_WRITE(EG0, 0x00007f00);
		4950	I915_WRITE(EG1, 0x0000000e);
		4951	I915_WRITE(EG2, 0x000e0000);
		4952	I915_WRITE(EG3, 0x68000300);
		4953	I915_WRITE(EG4, 0x42000000);
		4954	I915_WRITE(EG5, 0x00140031);
		4955	I915_WRITE(EG6, 0);
		4956	I915_WRITE(EG7, 0);
		4957
		4958	for (i = 0; i < 8; i++)
		4959	I915_WRITE(PXWL + (i * 4), 0);
		4960
		4961	/* Enable PMON + select events */
		4962	I915_WRITE(ECR, 0x80000019);
		4963
		4964	lcfuse = I915_READ(LCFUSE02);
		4965
		4966	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		4967	}
		4968
5060	serge	4969	void intel_init_gt_powersave(struct drm_device *dev)
		4970	{
		4971	i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
		4972
		4973	if (IS_CHERRYVIEW(dev))
		4974	cherryview_init_gt_powersave(dev);
		4975	else if (IS_VALLEYVIEW(dev))
		4976	valleyview_init_gt_powersave(dev);
		4977	}
		4978
		4979	void intel_cleanup_gt_powersave(struct drm_device *dev)
		4980	{
		4981	if (IS_CHERRYVIEW(dev))
		4982	return;
		4983	else if (IS_VALLEYVIEW(dev))
		4984	valleyview_cleanup_gt_powersave(dev);
		4985	}
		4986
		4987	/**
		4988	* intel_suspend_gt_powersave - suspend PM work and helper threads
		4989	* @dev: drm device
		4990	*
		4991	* We don't want to disable RC6 or other features here, we just want
		4992	* to make sure any work we've queued has finished and won't bother
		4993	* us while we're suspended.
		4994	*/
		4995	void intel_suspend_gt_powersave(struct drm_device *dev)
		4996	{
		4997	struct drm_i915_private *dev_priv = dev->dev_private;
		4998
		4999	/* Interrupts should be disabled already to avoid re-arming. */
		5000	WARN_ON(intel_irqs_enabled(dev_priv));
		5001
		5002	// flush_delayed_work(&dev_priv->rps.delayed_resume_work);
		5003
		5004	cancel_work_sync(&dev_priv->rps.work);
		5005
		5006	/* Force GPU to min freq during suspend */
		5007	gen6_rps_idle(dev_priv);
		5008	}
		5009
3031	serge	5010	void intel_disable_gt_powersave(struct drm_device *dev)
		5011	{
3243	Serge	5012	struct drm_i915_private *dev_priv = dev->dev_private;
		5013
4104	Serge	5014	/* Interrupts should be disabled already to avoid re-arming. */
5060	serge	5015	WARN_ON(intel_irqs_enabled(dev_priv));
4104	Serge	5016
3031	serge	5017	if (IS_IRONLAKE_M(dev)) {
		5018	ironlake_disable_drps(dev);
		5019	ironlake_disable_rc6(dev);
4293	Serge	5020	} else if (INTEL_INFO(dev)->gen >= 6) {
5060	serge	5021	intel_suspend_gt_powersave(dev);
		5022
3482	Serge	5023	mutex_lock(&dev_priv->rps.hw_lock);
5060	serge	5024	if (IS_CHERRYVIEW(dev))
		5025	cherryview_disable_rps(dev);
		5026	else if (IS_VALLEYVIEW(dev))
4104	Serge	5027	valleyview_disable_rps(dev);
		5028	else
3031	serge	5029	gen6_disable_rps(dev);
4560	Serge	5030	dev_priv->rps.enabled = false;
3480	Serge	5031	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	5032	}
		5033	}
		5034
3482	Serge	5035	static void intel_gen6_powersave_work(struct work_struct *work)
		5036	{
		5037	struct drm_i915_private *dev_priv =
		5038	container_of(work, struct drm_i915_private,
		5039	rps.delayed_resume_work.work);
		5040	struct drm_device *dev = dev_priv->dev;
		5041
		5042	mutex_lock(&dev_priv->rps.hw_lock);
4104	Serge	5043
5060	serge	5044	if (IS_CHERRYVIEW(dev)) {
		5045	cherryview_enable_rps(dev);
		5046	} else if (IS_VALLEYVIEW(dev)) {
4104	Serge	5047	valleyview_enable_rps(dev);
4560	Serge	5048	} else if (IS_BROADWELL(dev)) {
		5049	gen8_enable_rps(dev);
5060	serge	5050	__gen6_update_ring_freq(dev);
4104	Serge	5051	} else {
3482	Serge	5052	gen6_enable_rps(dev);
5060	serge	5053	__gen6_update_ring_freq(dev);
4104	Serge	5054	}
4560	Serge	5055	dev_priv->rps.enabled = true;
3482	Serge	5056	mutex_unlock(&dev_priv->rps.hw_lock);
5060	serge	5057
		5058	intel_runtime_pm_put(dev_priv);
3482	Serge	5059	}
		5060
3031	serge	5061	void intel_enable_gt_powersave(struct drm_device *dev)
		5062	{
3243	Serge	5063	struct drm_i915_private *dev_priv = dev->dev_private;
		5064
3031	serge	5065	if (IS_IRONLAKE_M(dev)) {
5060	serge	5066	mutex_lock(&dev->struct_mutex);
3031	serge	5067	ironlake_enable_drps(dev);
		5068	ironlake_enable_rc6(dev);
		5069	intel_init_emon(dev);
5060	serge	5070	mutex_unlock(&dev->struct_mutex);
		5071	} else if (INTEL_INFO(dev)->gen >= 6) {
3243	Serge	5072	/*
		5073	* PCU communication is slow and this doesn't need to be
		5074	* done at any specific time, so do this out of our fast path
		5075	* to make resume and init faster.
5060	serge	5076	*
		5077	* We depend on the HW RC6 power context save/restore
		5078	* mechanism when entering D3 through runtime PM suspend. So
		5079	* disable RPM until RPS/RC6 is properly setup. We can only
		5080	* get here via the driver load/system resume/runtime resume
		5081	* paths, so the _noresume version is enough (and in case of
		5082	* runtime resume it's necessary).
3243	Serge	5083	*/
5060	serge	5084	if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		5085	round_jiffies_up_relative(HZ)))
		5086	intel_runtime_pm_get_noresume(dev_priv);
3031	serge	5087	}
		5088	}
		5089
5060	serge	5090	void intel_reset_gt_powersave(struct drm_device *dev)
		5091	{
		5092	struct drm_i915_private *dev_priv = dev->dev_private;
		5093
		5094	dev_priv->rps.enabled = false;
		5095	intel_enable_gt_powersave(dev);
		5096	}
		5097
3243	Serge	5098	static void ibx_init_clock_gating(struct drm_device *dev)
		5099	{
		5100	struct drm_i915_private *dev_priv = dev->dev_private;
		5101
		5102	/*
		5103	* On Ibex Peak and Cougar Point, we need to disable clock
		5104	* gating for the panel power sequencer or it will fail to
		5105	* start up when no ports are active.
		5106	*/
		5107	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		5108	}
		5109
4104	Serge	5110	static void g4x_disable_trickle_feed(struct drm_device *dev)
		5111	{
		5112	struct drm_i915_private *dev_priv = dev->dev_private;
		5113	int pipe;
		5114
		5115	for_each_pipe(pipe) {
		5116	I915_WRITE(DSPCNTR(pipe),
		5117	I915_READ(DSPCNTR(pipe)) \|
		5118	DISPPLANE_TRICKLE_FEED_DISABLE);
4560	Serge	5119	intel_flush_primary_plane(dev_priv, pipe);
4104	Serge	5120	}
		5121	}
		5122
4560	Serge	5123	static void ilk_init_lp_watermarks(struct drm_device *dev)
		5124	{
		5125	struct drm_i915_private *dev_priv = dev->dev_private;
		5126
		5127	I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
		5128	I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
		5129	I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
		5130
		5131	/*
		5132	* Don't touch WM1S_LP_EN here.
		5133	* Doing so could cause underruns.
		5134	*/
		5135	}
		5136
3031	serge	5137	static void ironlake_init_clock_gating(struct drm_device *dev)
		5138	{
		5139	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	5140	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	5141
4104	Serge	5142	/*
		5143	* Required for FBC
		5144	* WaFbcDisableDpfcClockGating:ilk
		5145	*/
3243	Serge	5146	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		5147	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		5148	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	5149
		5150	I915_WRITE(PCH_3DCGDIS0,
		5151	MARIUNIT_CLOCK_GATE_DISABLE \|
		5152	SVSMUNIT_CLOCK_GATE_DISABLE);
		5153	I915_WRITE(PCH_3DCGDIS1,
		5154	VFMUNIT_CLOCK_GATE_DISABLE);
		5155
		5156	/*
		5157	* According to the spec the following bits should be set in
		5158	* order to enable memory self-refresh
		5159	* The bit 22/21 of 0x42004
		5160	* The bit 5 of 0x42020
		5161	* The bit 15 of 0x45000
		5162	*/
		5163	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		5164	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		5165	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	5166	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	5167	I915_WRITE(DISP_ARB_CTL,
		5168	(I915_READ(DISP_ARB_CTL) \|
		5169	DISP_FBC_WM_DIS));
		5170
4560	Serge	5171	ilk_init_lp_watermarks(dev);
		5172
3031	serge	5173	/*
		5174	* Based on the document from hardware guys the following bits
		5175	* should be set unconditionally in order to enable FBC.
		5176	* The bit 22 of 0x42000
		5177	* The bit 22 of 0x42004
		5178	* The bit 7,8,9 of 0x42020.
		5179	*/
		5180	if (IS_IRONLAKE_M(dev)) {
4104	Serge	5181	/* WaFbcAsynchFlipDisableFbcQueue:ilk */
3031	serge	5182	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		5183	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		5184	ILK_FBCQ_DIS);
		5185	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		5186	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		5187	ILK_DPARB_GATE);
		5188	}
		5189
3243	Serge	5190	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		5191
3031	serge	5192	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		5193	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		5194	ILK_ELPIN_409_SELECT);
		5195	I915_WRITE(_3D_CHICKEN2,
		5196	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		5197	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	5198
4104	Serge	5199	/* WaDisableRenderCachePipelinedFlush:ilk */
3243	Serge	5200	I915_WRITE(CACHE_MODE_0,
		5201	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		5202
5060	serge	5203	/* WaDisable_RenderCache_OperationalFlush:ilk */
		5204	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		5205
4104	Serge	5206	g4x_disable_trickle_feed(dev);
		5207
3243	Serge	5208	ibx_init_clock_gating(dev);
3031	serge	5209	}
		5210
3243	Serge	5211	static void cpt_init_clock_gating(struct drm_device *dev)
		5212	{
		5213	struct drm_i915_private *dev_priv = dev->dev_private;
		5214	int pipe;
3746	Serge	5215	uint32_t val;
3243	Serge	5216
		5217	/*
		5218	* On Ibex Peak and Cougar Point, we need to disable clock
		5219	* gating for the panel power sequencer or it will fail to
		5220	* start up when no ports are active.
		5221	*/
4280	Serge	5222	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE \|
		5223	PCH_DPLUNIT_CLOCK_GATE_DISABLE \|
		5224	PCH_CPUNIT_CLOCK_GATE_DISABLE);
3243	Serge	5225	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		5226	DPLS_EDP_PPS_FIX_DIS);
		5227	/* The below fixes the weird display corruption, a few pixels shifted
		5228	* downward, on (only) LVDS of some HP laptops with IVY.
		5229	*/
3746	Serge	5230	for_each_pipe(pipe) {
		5231	val = I915_READ(TRANS_CHICKEN2(pipe));
		5232	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		5233	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4104	Serge	5234	if (dev_priv->vbt.fdi_rx_polarity_inverted)
3746	Serge	5235	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		5236	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		5237	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		5238	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		5239	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		5240	}
3243	Serge	5241	/* WADP0ClockGatingDisable */
		5242	for_each_pipe(pipe) {
		5243	I915_WRITE(TRANS_CHICKEN1(pipe),
		5244	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		5245	}
		5246	}
		5247
3480	Serge	5248	static void gen6_check_mch_setup(struct drm_device *dev)
		5249	{
		5250	struct drm_i915_private *dev_priv = dev->dev_private;
		5251	uint32_t tmp;
		5252
		5253	tmp = I915_READ(MCH_SSKPD);
5060	serge	5254	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
		5255	DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
		5256	tmp);
3480	Serge	5257	}
		5258
3031	serge	5259	static void gen6_init_clock_gating(struct drm_device *dev)
		5260	{
		5261	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	5262	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	5263
3243	Serge	5264	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	5265
		5266	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		5267	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		5268	ILK_ELPIN_409_SELECT);
		5269
4104	Serge	5270	/* WaDisableHiZPlanesWhenMSAAEnabled:snb */
3243	Serge	5271	I915_WRITE(_3D_CHICKEN,
		5272	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		5273
4104	Serge	5274	/* WaSetupGtModeTdRowDispatch:snb */
3243	Serge	5275	if (IS_SNB_GT1(dev))
		5276	I915_WRITE(GEN6_GT_MODE,
		5277	_MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
		5278
5060	serge	5279	/* WaDisable_RenderCache_OperationalFlush:snb */
		5280	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		5281
		5282	/*
		5283	* BSpec recoomends 8x4 when MSAA is used,
		5284	* however in practice 16x4 seems fastest.
		5285	*
		5286	* Note that PS/WM thread counts depend on the WIZ hashing
		5287	* disable bit, which we don't touch here, but it's good
		5288	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		5289	*/
		5290	I915_WRITE(GEN6_GT_MODE,
		5291	GEN6_WIZ_HASHING_MASK \| GEN6_WIZ_HASHING_16x4);
		5292
4560	Serge	5293	ilk_init_lp_watermarks(dev);
3031	serge	5294
		5295	I915_WRITE(CACHE_MODE_0,
		5296	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		5297
		5298	I915_WRITE(GEN6_UCGCTL1,
		5299	I915_READ(GEN6_UCGCTL1) \|
		5300	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		5301	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		5302
		5303	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		5304	* gating disable must be set. Failure to set it results in
		5305	* flickering pixels due to Z write ordering failures after
		5306	* some amount of runtime in the Mesa "fire" demo, and Unigine
		5307	* Sanctuary and Tropics, and apparently anything else with
		5308	* alpha test or pixel discard.
		5309	*
		5310	* According to the spec, bit 11 (RCCUNIT) must also be set,
		5311	* but we didn't debug actual testcases to find it out.
		5312	*
5060	serge	5313	* WaDisableRCCUnitClockGating:snb
		5314	* WaDisableRCPBUnitClockGating:snb
3031	serge	5315	*/
		5316	I915_WRITE(GEN6_UCGCTL2,
		5317	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		5318	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		5319
5060	serge	5320	/* WaStripsFansDisableFastClipPerformanceFix:snb */
		5321	I915_WRITE(_3D_CHICKEN3,
		5322	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
3031	serge	5323
		5324	/*
5060	serge	5325	* Bspec says:
		5326	* "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
		5327	* 3DSTATE_SF number of SF output attributes is more than 16."
		5328	*/
		5329	I915_WRITE(_3D_CHICKEN3,
		5330	_MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
		5331
		5332	/*
3031	serge	5333	* According to the spec the following bits should be
		5334	* set in order to enable memory self-refresh and fbc:
		5335	* The bit21 and bit22 of 0x42000
		5336	* The bit21 and bit22 of 0x42004
		5337	* The bit5 and bit7 of 0x42020
		5338	* The bit14 of 0x70180
		5339	* The bit14 of 0x71180
4104	Serge	5340	*
		5341	* WaFbcAsynchFlipDisableFbcQueue:snb
3031	serge	5342	*/
		5343	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		5344	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		5345	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		5346	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		5347	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		5348	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	5349	I915_WRITE(ILK_DSPCLK_GATE_D,
		5350	I915_READ(ILK_DSPCLK_GATE_D) \|
		5351	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		5352	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	5353
4104	Serge	5354	g4x_disable_trickle_feed(dev);
3031	serge	5355
3243	Serge	5356	cpt_init_clock_gating(dev);
3480	Serge	5357
		5358	gen6_check_mch_setup(dev);
3031	serge	5359	}
		5360
		5361	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		5362	{
		5363	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		5364
5060	serge	5365	/*
		5366	* WaVSThreadDispatchOverride:ivb,vlv
		5367	*
		5368	* This actually overrides the dispatch
		5369	* mode for all thread types.
		5370	*/
3031	serge	5371	reg &= ~GEN7_FF_SCHED_MASK;
		5372	reg \|= GEN7_FF_TS_SCHED_HW;
		5373	reg \|= GEN7_FF_VS_SCHED_HW;
		5374	reg \|= GEN7_FF_DS_SCHED_HW;
		5375
		5376	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		5377	}
		5378
3243	Serge	5379	static void lpt_init_clock_gating(struct drm_device *dev)
		5380	{
		5381	struct drm_i915_private *dev_priv = dev->dev_private;
		5382
		5383	/*
		5384	* TODO: this bit should only be enabled when really needed, then
		5385	* disabled when not needed anymore in order to save power.
		5386	*/
		5387	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
		5388	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		5389	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		5390	PCH_LP_PARTITION_LEVEL_DISABLE);
4104	Serge	5391
		5392	/* WADPOClockGatingDisable:hsw */
		5393	I915_WRITE(_TRANSA_CHICKEN1,
		5394	I915_READ(_TRANSA_CHICKEN1) \|
		5395	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
3243	Serge	5396	}
		5397
4104	Serge	5398	static void lpt_suspend_hw(struct drm_device *dev)
		5399	{
		5400	struct drm_i915_private *dev_priv = dev->dev_private;
		5401
		5402	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
		5403	uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
		5404
		5405	val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
		5406	I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
		5407	}
		5408	}
		5409
4560	Serge	5410	static void gen8_init_clock_gating(struct drm_device *dev)
3031	serge	5411	{
		5412	struct drm_i915_private *dev_priv = dev->dev_private;
5060	serge	5413	enum pipe pipe;
3031	serge	5414
		5415	I915_WRITE(WM3_LP_ILK, 0);
		5416	I915_WRITE(WM2_LP_ILK, 0);
		5417	I915_WRITE(WM1_LP_ILK, 0);
		5418
4560	Serge	5419	/* FIXME(BDW): Check all the w/a, some might only apply to
		5420	* pre-production hw. */
		5421
5060	serge	5422	/* WaDisablePartialInstShootdown:bdw */
		5423	I915_WRITE(GEN8_ROW_CHICKEN,
		5424	_MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
		5425
		5426	/* WaDisableThreadStallDopClockGating:bdw */
		5427	/* FIXME: Unclear whether we really need this on production bdw. */
		5428	I915_WRITE(GEN8_ROW_CHICKEN,
		5429	_MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
		5430
		5431	/*
		5432	* This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for
		5433	* pre-production hardware
		5434	*/
4560	Serge	5435	I915_WRITE(HALF_SLICE_CHICKEN3,
		5436	_MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS));
		5437	I915_WRITE(HALF_SLICE_CHICKEN3,
		5438	_MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
		5439	I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE));
		5440
		5441	I915_WRITE(_3D_CHICKEN3,
5060	serge	5442	_MASKED_BIT_ENABLE(_3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2)));
4560	Serge	5443
		5444	I915_WRITE(COMMON_SLICE_CHICKEN2,
		5445	_MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE));
		5446
		5447	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		5448	_MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE));
		5449
5060	serge	5450	/* WaDisableDopClockGating:bdw May not be needed for production */
		5451	I915_WRITE(GEN7_ROW_CHICKEN2,
		5452	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		5453
4560	Serge	5454	/* WaSwitchSolVfFArbitrationPriority:bdw */
		5455	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		5456
		5457	/* WaPsrDPAMaskVBlankInSRD:bdw */
		5458	I915_WRITE(CHICKEN_PAR1_1,
		5459	I915_READ(CHICKEN_PAR1_1) \| DPA_MASK_VBLANK_SRD);
		5460
		5461	/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
5060	serge	5462	for_each_pipe(pipe) {
		5463	I915_WRITE(CHICKEN_PIPESL_1(pipe),
		5464	I915_READ(CHICKEN_PIPESL_1(pipe)) \|
		5465	BDW_DPRS_MASK_VBLANK_SRD);
4560	Serge	5466	}
		5467
		5468	/* Use Force Non-Coherent whenever executing a 3D context. This is a
		5469	* workaround for for a possible hang in the unlikely event a TLB
		5470	* invalidation occurs during a PSD flush.
		5471	*/
		5472	I915_WRITE(HDC_CHICKEN0,
		5473	I915_READ(HDC_CHICKEN0) \|
		5474	_MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT));
		5475
		5476	/* WaVSRefCountFullforceMissDisable:bdw */
		5477	/* WaDSRefCountFullforceMissDisable:bdw */
		5478	I915_WRITE(GEN7_FF_THREAD_MODE,
		5479	I915_READ(GEN7_FF_THREAD_MODE) &
		5480	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
5060	serge	5481
		5482	/*
		5483	* BSpec recommends 8x4 when MSAA is used,
		5484	* however in practice 16x4 seems fastest.
		5485	*
		5486	* Note that PS/WM thread counts depend on the WIZ hashing
		5487	* disable bit, which we don't touch here, but it's good
		5488	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		5489	*/
		5490	I915_WRITE(GEN7_GT_MODE,
		5491	GEN6_WIZ_HASHING_MASK \| GEN6_WIZ_HASHING_16x4);
		5492
		5493	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		5494	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		5495
		5496	/* WaDisableSDEUnitClockGating:bdw */
		5497	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		5498	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		5499
		5500	/* Wa4x4STCOptimizationDisable:bdw */
		5501	I915_WRITE(CACHE_MODE_1,
		5502	_MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE));
4560	Serge	5503	}
		5504
		5505	static void haswell_init_clock_gating(struct drm_device *dev)
		5506	{
		5507	struct drm_i915_private *dev_priv = dev->dev_private;
		5508
		5509	ilk_init_lp_watermarks(dev);
		5510
4104	Serge	5511	/* L3 caching of data atomics doesn't work -- disable it. */
		5512	I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
		5513	I915_WRITE(HSW_ROW_CHICKEN3,
		5514	_MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
		5515
		5516	/* This is required by WaCatErrorRejectionIssue:hsw */
3031	serge	5517	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		5518	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		5519	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		5520
4104	Serge	5521	/* WaVSRefCountFullforceMissDisable:hsw */
5060	serge	5522	I915_WRITE(GEN7_FF_THREAD_MODE,
		5523	I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
3031	serge	5524
5060	serge	5525	/* WaDisable_RenderCache_OperationalFlush:hsw */
		5526	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		5527
		5528	/* enable HiZ Raw Stall Optimization */
		5529	I915_WRITE(CACHE_MODE_0_GEN7,
		5530	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		5531
4104	Serge	5532	/* WaDisable4x2SubspanOptimization:hsw */
3031	serge	5533	I915_WRITE(CACHE_MODE_1,
		5534	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		5535
5060	serge	5536	/*
		5537	* BSpec recommends 8x4 when MSAA is used,
		5538	* however in practice 16x4 seems fastest.
		5539	*
		5540	* Note that PS/WM thread counts depend on the WIZ hashing
		5541	* disable bit, which we don't touch here, but it's good
		5542	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		5543	*/
		5544	I915_WRITE(GEN7_GT_MODE,
		5545	GEN6_WIZ_HASHING_MASK \| GEN6_WIZ_HASHING_16x4);
		5546
4104	Serge	5547	/* WaSwitchSolVfFArbitrationPriority:hsw */
3746	Serge	5548	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		5549
4104	Serge	5550	/* WaRsPkgCStateDisplayPMReq:hsw */
		5551	I915_WRITE(CHICKEN_PAR1_1,
		5552	I915_READ(CHICKEN_PAR1_1) \| FORCE_ARB_IDLE_PLANES);
3031	serge	5553
3243	Serge	5554	lpt_init_clock_gating(dev);
3031	serge	5555	}
		5556
		5557	static void ivybridge_init_clock_gating(struct drm_device *dev)
		5558	{
		5559	struct drm_i915_private *dev_priv = dev->dev_private;
		5560	uint32_t snpcr;
		5561
4560	Serge	5562	ilk_init_lp_watermarks(dev);
3031	serge	5563
3243	Serge	5564	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	5565
4104	Serge	5566	/* WaDisableEarlyCull:ivb */
3243	Serge	5567	I915_WRITE(_3D_CHICKEN3,
		5568	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		5569
4104	Serge	5570	/* WaDisableBackToBackFlipFix:ivb */
3031	serge	5571	I915_WRITE(IVB_CHICKEN3,
		5572	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		5573	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		5574
4104	Serge	5575	/* WaDisablePSDDualDispatchEnable:ivb */
3243	Serge	5576	if (IS_IVB_GT1(dev))
		5577	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		5578	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		5579
5060	serge	5580	/* WaDisable_RenderCache_OperationalFlush:ivb */
		5581	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		5582
4104	Serge	5583	/* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
3031	serge	5584	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		5585	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		5586
4104	Serge	5587	/* WaApplyL3ControlAndL3ChickenMode:ivb */
3031	serge	5588	I915_WRITE(GEN7_L3CNTLREG1,
		5589	GEN7_WA_FOR_GEN7_L3_CONTROL);
		5590	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		5591	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	5592	if (IS_IVB_GT1(dev))
		5593	I915_WRITE(GEN7_ROW_CHICKEN2,
		5594	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	5595	else {
		5596	/* must write both registers */
		5597	I915_WRITE(GEN7_ROW_CHICKEN2,
		5598	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3243	Serge	5599	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		5600	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
5060	serge	5601	}
3031	serge	5602
4104	Serge	5603	/* WaForceL3Serialization:ivb */
3243	Serge	5604	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		5605	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		5606
5060	serge	5607	/*
3031	serge	5608	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	5609	* This implements the WaDisableRCZUnitClockGating:ivb workaround.
3031	serge	5610	*/
		5611	I915_WRITE(GEN6_UCGCTL2,
5060	serge	5612	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	5613
4104	Serge	5614	/* This is required by WaCatErrorRejectionIssue:ivb */
3031	serge	5615	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		5616	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		5617	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		5618
4104	Serge	5619	g4x_disable_trickle_feed(dev);
3031	serge	5620
		5621	gen7_setup_fixed_func_scheduler(dev_priv);
		5622
5060	serge	5623	if (0) { /* causes HiZ corruption on ivb:gt1 */
		5624	/* enable HiZ Raw Stall Optimization */
		5625	I915_WRITE(CACHE_MODE_0_GEN7,
		5626	_MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
		5627	}
		5628
4104	Serge	5629	/* WaDisable4x2SubspanOptimization:ivb */
3031	serge	5630	I915_WRITE(CACHE_MODE_1,
		5631	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		5632
5060	serge	5633	/*
		5634	* BSpec recommends 8x4 when MSAA is used,
		5635	* however in practice 16x4 seems fastest.
		5636	*
		5637	* Note that PS/WM thread counts depend on the WIZ hashing
		5638	* disable bit, which we don't touch here, but it's good
		5639	* to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
		5640	*/
		5641	I915_WRITE(GEN7_GT_MODE,
		5642	GEN6_WIZ_HASHING_MASK \| GEN6_WIZ_HASHING_16x4);
		5643
3031	serge	5644	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		5645	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		5646	snpcr \|= GEN6_MBC_SNPCR_MED;
		5647	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	5648
3746	Serge	5649	if (!HAS_PCH_NOP(dev))
3243	Serge	5650	cpt_init_clock_gating(dev);
3480	Serge	5651
		5652	gen6_check_mch_setup(dev);
3031	serge	5653	}
		5654
		5655	static void valleyview_init_clock_gating(struct drm_device *dev)
		5656	{
		5657	struct drm_i915_private *dev_priv = dev->dev_private;
4560	Serge	5658	u32 val;
3031	serge	5659
4560	Serge	5660	mutex_lock(&dev_priv->rps.hw_lock);
		5661	val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
		5662	mutex_unlock(&dev_priv->rps.hw_lock);
		5663	switch ((val >> 6) & 3) {
		5664	case 0:
5060	serge	5665	case 1:
4560	Serge	5666	dev_priv->mem_freq = 800;
		5667	break;
5060	serge	5668	case 2:
4560	Serge	5669	dev_priv->mem_freq = 1066;
		5670	break;
		5671	case 3:
		5672	dev_priv->mem_freq = 1333;
		5673	break;
		5674	}
		5675	DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
		5676
4104	Serge	5677	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	5678
4104	Serge	5679	/* WaDisableEarlyCull:vlv */
3243	Serge	5680	I915_WRITE(_3D_CHICKEN3,
		5681	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		5682
4104	Serge	5683	/* WaDisableBackToBackFlipFix:vlv */
3031	serge	5684	I915_WRITE(IVB_CHICKEN3,
		5685	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		5686	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		5687
5060	serge	5688	/* WaPsdDispatchEnable:vlv */
4104	Serge	5689	/* WaDisablePSDDualDispatchEnable:vlv */
3243	Serge	5690	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	5691	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		5692	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	5693
5060	serge	5694	/* WaDisable_RenderCache_OperationalFlush:vlv */
		5695	I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	5696
4104	Serge	5697	/* WaForceL3Serialization:vlv */
3243	Serge	5698	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		5699	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		5700
4104	Serge	5701	/* WaDisableDopClockGating:vlv */
3243	Serge	5702	I915_WRITE(GEN7_ROW_CHICKEN2,
		5703	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		5704
4104	Serge	5705	/* This is required by WaCatErrorRejectionIssue:vlv */
3031	serge	5706	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		5707	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		5708	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		5709
5060	serge	5710	gen7_setup_fixed_func_scheduler(dev_priv);
		5711
		5712	/*
3031	serge	5713	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4104	Serge	5714	* This implements the WaDisableRCZUnitClockGating:vlv workaround.
3031	serge	5715	*/
		5716	I915_WRITE(GEN6_UCGCTL2,
5060	serge	5717	GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
3031	serge	5718
5060	serge	5719	/* WaDisableL3Bank2xClockGate:vlv
		5720	* Disabling L3 clock gating- MMIO 940c[25] = 1
		5721	* Set bit 25, to disable L3_BANK_2x_CLK_GATING */
		5722	I915_WRITE(GEN7_UCGCTL4,
		5723	I915_READ(GEN7_UCGCTL4) \| GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
3031	serge	5724
4104	Serge	5725	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
3031	serge	5726
5060	serge	5727	/*
		5728	* BSpec says this must be set, even though
		5729	* WaDisable4x2SubspanOptimization isn't listed for VLV.
		5730	*/
3031	serge	5731	I915_WRITE(CACHE_MODE_1,
		5732	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		5733
		5734	/*
5060	serge	5735	* WaIncreaseL3CreditsForVLVB0:vlv
		5736	* This is the hardware default actually.
		5737	*/
		5738	I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
		5739
		5740	/*
4104	Serge	5741	* WaDisableVLVClockGating_VBIIssue:vlv
3243	Serge	5742	* Disable clock gating on th GCFG unit to prevent a delay
		5743	* in the reporting of vblank events.
		5744	*/
5060	serge	5745	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
		5746	}
3746	Serge	5747
5060	serge	5748	static void cherryview_init_clock_gating(struct drm_device *dev)
		5749	{
		5750	struct drm_i915_private *dev_priv = dev->dev_private;
		5751	u32 val;
		5752
		5753	mutex_lock(&dev_priv->rps.hw_lock);
		5754	val = vlv_punit_read(dev_priv, CCK_FUSE_REG);
		5755	mutex_unlock(&dev_priv->rps.hw_lock);
		5756	switch ((val >> 2) & 0x7) {
		5757	case 0:
		5758	case 1:
		5759	dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_200;
		5760	dev_priv->mem_freq = 1600;
		5761	break;
		5762	case 2:
		5763	dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_267;
		5764	dev_priv->mem_freq = 1600;
		5765	break;
		5766	case 3:
		5767	dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_333;
		5768	dev_priv->mem_freq = 2000;
		5769	break;
		5770	case 4:
		5771	dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_320;
		5772	dev_priv->mem_freq = 1600;
		5773	break;
		5774	case 5:
		5775	dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_400;
		5776	dev_priv->mem_freq = 1600;
		5777	break;
		5778	}
		5779	DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
		5780
		5781	I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
		5782
		5783	I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
		5784
		5785	/* WaDisablePartialInstShootdown:chv */
		5786	I915_WRITE(GEN8_ROW_CHICKEN,
		5787	_MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
		5788
		5789	/* WaDisableThreadStallDopClockGating:chv */
		5790	I915_WRITE(GEN8_ROW_CHICKEN,
		5791	_MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
		5792
		5793	/* WaVSRefCountFullforceMissDisable:chv */
		5794	/* WaDSRefCountFullforceMissDisable:chv */
		5795	I915_WRITE(GEN7_FF_THREAD_MODE,
		5796	I915_READ(GEN7_FF_THREAD_MODE) &
		5797	~(GEN8_FF_DS_REF_CNT_FFME \| GEN7_FF_VS_REF_CNT_FFME));
		5798
		5799	/* WaDisableSemaphoreAndSyncFlipWait:chv */
		5800	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		5801	_MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
		5802
		5803	/* WaDisableCSUnitClockGating:chv */
		5804	I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) \|
		5805	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		5806
		5807	/* WaDisableSDEUnitClockGating:chv */
		5808	I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) \|
		5809	GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
		5810
		5811	/* WaDisableSamplerPowerBypass:chv (pre-production hw) */
		5812	I915_WRITE(HALF_SLICE_CHICKEN3,
		5813	_MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
		5814
		5815	/* WaDisableGunitClockGating:chv (pre-production hw) */
		5816	I915_WRITE(VLV_GUNIT_CLOCK_GATE, I915_READ(VLV_GUNIT_CLOCK_GATE) \|
		5817	GINT_DIS);
		5818
		5819	/* WaDisableFfDopClockGating:chv (pre-production hw) */
		5820	I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
		5821	_MASKED_BIT_ENABLE(GEN8_FF_DOP_CLOCK_GATE_DISABLE));
		5822
		5823	/* WaDisableDopClockGating:chv (pre-production hw) */
		5824	I915_WRITE(GEN7_ROW_CHICKEN2,
		5825	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		5826	I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) \|
		5827	GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
3031	serge	5828	}
		5829
		5830	static void g4x_init_clock_gating(struct drm_device *dev)
		5831	{
		5832	struct drm_i915_private *dev_priv = dev->dev_private;
		5833	uint32_t dspclk_gate;
		5834
		5835	I915_WRITE(RENCLK_GATE_D1, 0);
		5836	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		5837	GS_UNIT_CLOCK_GATE_DISABLE \|
		5838	CL_UNIT_CLOCK_GATE_DISABLE);
		5839	I915_WRITE(RAMCLK_GATE_D, 0);
		5840	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		5841	OVRUNIT_CLOCK_GATE_DISABLE \|
		5842	OVCUNIT_CLOCK_GATE_DISABLE;
		5843	if (IS_GM45(dev))
		5844	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		5845	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	5846
		5847	/* WaDisableRenderCachePipelinedFlush */
		5848	I915_WRITE(CACHE_MODE_0,
		5849	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
4104	Serge	5850
5060	serge	5851	/* WaDisable_RenderCache_OperationalFlush:g4x */
		5852	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
		5853
4104	Serge	5854	g4x_disable_trickle_feed(dev);
3031	serge	5855	}
		5856
		5857	static void crestline_init_clock_gating(struct drm_device *dev)
		5858	{
		5859	struct drm_i915_private *dev_priv = dev->dev_private;
		5860
		5861	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		5862	I915_WRITE(RENCLK_GATE_D2, 0);
		5863	I915_WRITE(DSPCLK_GATE_D, 0);
		5864	I915_WRITE(RAMCLK_GATE_D, 0);
		5865	I915_WRITE16(DEUC, 0);
4104	Serge	5866	I915_WRITE(MI_ARB_STATE,
		5867	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	5868
		5869	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		5870	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	5871	}
		5872
		5873	static void broadwater_init_clock_gating(struct drm_device *dev)
		5874	{
		5875	struct drm_i915_private *dev_priv = dev->dev_private;
		5876
		5877	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		5878	I965_RCC_CLOCK_GATE_DISABLE \|
		5879	I965_RCPB_CLOCK_GATE_DISABLE \|
		5880	I965_ISC_CLOCK_GATE_DISABLE \|
		5881	I965_FBC_CLOCK_GATE_DISABLE);
		5882	I915_WRITE(RENCLK_GATE_D2, 0);
4104	Serge	5883	I915_WRITE(MI_ARB_STATE,
		5884	_MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
5060	serge	5885
		5886	/* WaDisable_RenderCache_OperationalFlush:gen4 */
		5887	I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
3031	serge	5888	}
		5889
		5890	static void gen3_init_clock_gating(struct drm_device *dev)
		5891	{
		5892	struct drm_i915_private *dev_priv = dev->dev_private;
		5893	u32 dstate = I915_READ(D_STATE);
		5894
		5895	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		5896	DSTATE_DOT_CLOCK_GATING;
		5897	I915_WRITE(D_STATE, dstate);
		5898
		5899	if (IS_PINEVIEW(dev))
		5900	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		5901
		5902	/* IIR "flip pending" means done if this bit is set */
		5903	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
5060	serge	5904
		5905	/* interrupts should cause a wake up from C3 */
		5906	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
		5907
		5908	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
		5909	I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
3031	serge	5910	}
		5911
		5912	static void i85x_init_clock_gating(struct drm_device *dev)
		5913	{
		5914	struct drm_i915_private *dev_priv = dev->dev_private;
		5915
		5916	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
5060	serge	5917
		5918	/* interrupts should cause a wake up from C3 */
		5919	I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) \|
		5920	_MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
3031	serge	5921	}
		5922
		5923	static void i830_init_clock_gating(struct drm_device *dev)
		5924	{
		5925	struct drm_i915_private *dev_priv = dev->dev_private;
		5926
		5927	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
		5928	}
		5929
		5930	void intel_init_clock_gating(struct drm_device *dev)
		5931	{
		5932	struct drm_i915_private *dev_priv = dev->dev_private;
		5933
		5934	dev_priv->display.init_clock_gating(dev);
		5935	}
		5936
4104	Serge	5937	void intel_suspend_hw(struct drm_device *dev)
		5938	{
		5939	if (HAS_PCH_LPT(dev))
		5940	lpt_suspend_hw(dev);
		5941	}
		5942
4560	Serge	5943	#define for_each_power_well(i, power_well, domain_mask, power_domains) \
		5944	for (i = 0; \
		5945	i < (power_domains)->power_well_count && \
		5946	((power_well) = &(power_domains)->power_wells[i]); \
		5947	i++) \
		5948	if ((power_well)->domains & (domain_mask))
		5949
		5950	#define for_each_power_well_rev(i, power_well, domain_mask, power_domains) \
		5951	for (i = (power_domains)->power_well_count - 1; \
		5952	i >= 0 && ((power_well) = &(power_domains)->power_wells[i]);\
		5953	i--) \
		5954	if ((power_well)->domains & (domain_mask))
		5955
3746	Serge	5956	/**
		5957	* We should only use the power well if we explicitly asked the hardware to
		5958	* enable it, so check if it's enabled and also check if we've requested it to
		5959	* be enabled.
		5960	*/
5060	serge	5961	static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
4560	Serge	5962	struct i915_power_well *power_well)
		5963	{
		5964	return I915_READ(HSW_PWR_WELL_DRIVER) ==
		5965	(HSW_PWR_WELL_ENABLE_REQUEST \| HSW_PWR_WELL_STATE_ENABLED);
		5966	}
		5967
5060	serge	5968	bool intel_display_power_enabled_unlocked(struct drm_i915_private *dev_priv,
4104	Serge	5969	enum intel_display_power_domain domain)
3746	Serge	5970	{
4560	Serge	5971	struct i915_power_domains *power_domains;
		5972	struct i915_power_well *power_well;
		5973	bool is_enabled;
		5974	int i;
3746	Serge	5975
5060	serge	5976	if (dev_priv->pm.suspended)
		5977	return false;
		5978
4560	Serge	5979	power_domains = &dev_priv->power_domains;
4104	Serge	5980
4560	Serge	5981	is_enabled = true;
		5982
		5983	for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
		5984	if (power_well->always_on)
		5985	continue;
		5986
5060	serge	5987	if (!power_well->hw_enabled) {
4560	Serge	5988	is_enabled = false;
		5989	break;
		5990	}
4104	Serge	5991	}
4560	Serge	5992
		5993	return is_enabled;
3746	Serge	5994	}
		5995
5060	serge	5996	bool intel_display_power_enabled(struct drm_i915_private *dev_priv,
		5997	enum intel_display_power_domain domain)
		5998	{
		5999	struct i915_power_domains *power_domains;
		6000	bool ret;
		6001
		6002	power_domains = &dev_priv->power_domains;
		6003
		6004	mutex_lock(&power_domains->lock);
		6005	ret = intel_display_power_enabled_unlocked(dev_priv, domain);
		6006	mutex_unlock(&power_domains->lock);
		6007
		6008	return ret;
		6009	}
		6010
		6011	/*
		6012	* Starting with Haswell, we have a "Power Down Well" that can be turned off
		6013	* when not needed anymore. We have 4 registers that can request the power well
		6014	* to be enabled, and it will only be disabled if none of the registers is
		6015	* requesting it to be enabled.
		6016	*/
4560	Serge	6017	static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
3031	serge	6018	{
4560	Serge	6019	struct drm_device *dev = dev_priv->dev;
		6020
		6021	/*
		6022	* After we re-enable the power well, if we touch VGA register 0x3d5
		6023	* we'll get unclaimed register interrupts. This stops after we write
		6024	* anything to the VGA MSR register. The vgacon module uses this
		6025	* register all the time, so if we unbind our driver and, as a
		6026	* consequence, bind vgacon, we'll get stuck in an infinite loop at
		6027	* console_unlock(). So make here we touch the VGA MSR register, making
		6028	* sure vgacon can keep working normally without triggering interrupts
		6029	* and error messages.
		6030	*/
		6031	// vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
		6032	outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
		6033	// vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
		6034
5060	serge	6035	if (IS_BROADWELL(dev))
		6036	gen8_irq_power_well_post_enable(dev_priv);
4560	Serge	6037	}
		6038
5060	serge	6039	static void hsw_set_power_well(struct drm_i915_private *dev_priv,
4560	Serge	6040	struct i915_power_well *power_well, bool enable)
		6041	{
3480	Serge	6042	bool is_enabled, enable_requested;
		6043	uint32_t tmp;
3031	serge	6044
3480	Serge	6045	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
4104	Serge	6046	is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
		6047	enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
3031	serge	6048
3480	Serge	6049	if (enable) {
		6050	if (!enable_requested)
4104	Serge	6051	I915_WRITE(HSW_PWR_WELL_DRIVER,
		6052	HSW_PWR_WELL_ENABLE_REQUEST);
3031	serge	6053
3480	Serge	6054	if (!is_enabled) {
		6055	DRM_DEBUG_KMS("Enabling power well\n");
		6056	if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
4104	Serge	6057	HSW_PWR_WELL_STATE_ENABLED), 20))
3480	Serge	6058	DRM_ERROR("Timeout enabling power well\n");
3031	serge	6059	}
4560	Serge	6060
		6061	hsw_power_well_post_enable(dev_priv);
3480	Serge	6062	} else {
		6063	if (enable_requested) {
		6064	I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
4104	Serge	6065	POSTING_READ(HSW_PWR_WELL_DRIVER);
3480	Serge	6066	DRM_DEBUG_KMS("Requesting to disable the power well\n");
4104	Serge	6067	}
		6068	}
		6069	}
		6070
5060	serge	6071	static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
4560	Serge	6072	struct i915_power_well *power_well)
		6073	{
5060	serge	6074	hsw_set_power_well(dev_priv, power_well, power_well->count > 0);
4104	Serge	6075
5060	serge	6076	/*
		6077	* We're taking over the BIOS, so clear any requests made by it since
		6078	* the driver is in charge now.
		6079	*/
		6080	if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
		6081	I915_WRITE(HSW_PWR_WELL_BIOS, 0);
4560	Serge	6082	}
		6083
5060	serge	6084	static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
		6085	struct i915_power_well *power_well)
		6086	{
		6087	hsw_set_power_well(dev_priv, power_well, true);
		6088	}
		6089
		6090	static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
4560	Serge	6091	struct i915_power_well *power_well)
		6092	{
5060	serge	6093	hsw_set_power_well(dev_priv, power_well, false);
		6094	}
4560	Serge	6095
5060	serge	6096	static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
		6097	struct i915_power_well *power_well)
		6098	{
		6099	}
4560	Serge	6100
5060	serge	6101	static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
		6102	struct i915_power_well *power_well)
		6103	{
		6104	return true;
		6105	}
		6106
		6107	static void vlv_set_power_well(struct drm_i915_private *dev_priv,
		6108	struct i915_power_well *power_well, bool enable)
		6109	{
		6110	enum punit_power_well power_well_id = power_well->data;
		6111	u32 mask;
		6112	u32 state;
		6113	u32 ctrl;
		6114
		6115	mask = PUNIT_PWRGT_MASK(power_well_id);
		6116	state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
		6117	PUNIT_PWRGT_PWR_GATE(power_well_id);
		6118
		6119	mutex_lock(&dev_priv->rps.hw_lock);
		6120
		6121	#define COND \
		6122	((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
		6123
		6124	if (COND)
		6125	goto out;
		6126
		6127	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
		6128	ctrl &= ~mask;
		6129	ctrl \|= state;
		6130	vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
		6131
		6132	if (wait_for(COND, 100))
		6133	DRM_ERROR("timout setting power well state %08x (%08x)\n",
		6134	state,
		6135	vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
		6136
		6137	#undef COND
		6138
		6139	out:
		6140	mutex_unlock(&dev_priv->rps.hw_lock);
		6141	}
		6142
		6143	static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv,
		6144	struct i915_power_well *power_well)
		6145	{
		6146	vlv_set_power_well(dev_priv, power_well, power_well->count > 0);
		6147	}
		6148
		6149	static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
		6150	struct i915_power_well *power_well)
		6151	{
		6152	vlv_set_power_well(dev_priv, power_well, true);
		6153	}
		6154
		6155	static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
		6156	struct i915_power_well *power_well)
		6157	{
		6158	vlv_set_power_well(dev_priv, power_well, false);
		6159	}
		6160
		6161	static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
		6162	struct i915_power_well *power_well)
		6163	{
		6164	int power_well_id = power_well->data;
		6165	bool enabled = false;
		6166	u32 mask;
		6167	u32 state;
		6168	u32 ctrl;
		6169
		6170	mask = PUNIT_PWRGT_MASK(power_well_id);
		6171	ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
		6172
		6173	mutex_lock(&dev_priv->rps.hw_lock);
		6174
		6175	state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
		6176	/*
		6177	* We only ever set the power-on and power-gate states, anything
		6178	* else is unexpected.
		6179	*/
		6180	WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
		6181	state != PUNIT_PWRGT_PWR_GATE(power_well_id));
		6182	if (state == ctrl)
		6183	enabled = true;
		6184
		6185	/*
		6186	* A transient state at this point would mean some unexpected party
		6187	* is poking at the power controls too.
		6188	*/
		6189	ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
		6190	WARN_ON(ctrl != state);
		6191
		6192	mutex_unlock(&dev_priv->rps.hw_lock);
		6193
		6194	return enabled;
		6195	}
		6196
		6197	static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
		6198	struct i915_power_well *power_well)
		6199	{
		6200	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
		6201
		6202	vlv_set_power_well(dev_priv, power_well, true);
		6203
		6204	spin_lock_irq(&dev_priv->irq_lock);
		6205	valleyview_enable_display_irqs(dev_priv);
		6206	spin_unlock_irq(&dev_priv->irq_lock);
		6207
		6208	/*
		6209	* During driver initialization/resume we can avoid restoring the
		6210	* part of the HW/SW state that will be inited anyway explicitly.
		6211	*/
		6212	if (dev_priv->power_domains.initializing)
		6213	return;
		6214
		6215	intel_hpd_init(dev_priv->dev);
		6216
		6217	i915_redisable_vga_power_on(dev_priv->dev);
		6218	}
		6219
		6220	static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
		6221	struct i915_power_well *power_well)
		6222	{
		6223	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
		6224
		6225	spin_lock_irq(&dev_priv->irq_lock);
		6226	valleyview_disable_display_irqs(dev_priv);
		6227	spin_unlock_irq(&dev_priv->irq_lock);
		6228
		6229	vlv_set_power_well(dev_priv, power_well, false);
		6230	}
		6231
		6232	static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
		6233	struct i915_power_well *power_well)
		6234	{
		6235	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
		6236
		6237	/*
		6238	* Enable the CRI clock source so we can get at the
		6239	* display and the reference clock for VGA
		6240	* hotplug / manual detection.
		6241	*/
		6242	I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) \|
		6243	DPLL_REFA_CLK_ENABLE_VLV \| DPLL_INTEGRATED_CRI_CLK_VLV);
		6244	udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
		6245
		6246	vlv_set_power_well(dev_priv, power_well, true);
		6247
		6248	/*
		6249	* From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
		6250	* 6. De-assert cmn_reset/side_reset. Same as VLV X0.
		6251	* a. GUnit 0x2110 bit[0] set to 1 (def 0)
		6252	* b. The other bits such as sfr settings / modesel may all
		6253	* be set to 0.
		6254	*
		6255	* This should only be done on init and resume from S3 with
		6256	* both PLLs disabled, or we risk losing DPIO and PLL
		6257	* synchronization.
		6258	*/
		6259	I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) \| DPIO_CMNRST);
		6260	}
		6261
		6262	static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
		6263	struct i915_power_well *power_well)
		6264	{
		6265	struct drm_device *dev = dev_priv->dev;
		6266	enum pipe pipe;
		6267
		6268	WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
		6269
		6270	for_each_pipe(pipe)
		6271	assert_pll_disabled(dev_priv, pipe);
		6272
		6273	/* Assert common reset */
		6274	I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
		6275
		6276	vlv_set_power_well(dev_priv, power_well, false);
		6277	}
		6278
		6279	static void check_power_well_state(struct drm_i915_private *dev_priv,
		6280	struct i915_power_well *power_well)
		6281	{
		6282	bool enabled = power_well->ops->is_enabled(dev_priv, power_well);
		6283
		6284	if (power_well->always_on \|\| !i915.disable_power_well) {
		6285	if (!enabled)
		6286	goto mismatch;
		6287
		6288	return;
4560	Serge	6289	}
5060	serge	6290
		6291	if (enabled != (power_well->count > 0))
		6292	goto mismatch;
		6293
		6294	return;
		6295
		6296	mismatch:
		6297	WARN(1, "state mismatch for '%s' (always_on %d hw state %d use-count %d disable_power_well %d\n",
		6298	power_well->name, power_well->always_on, enabled,
		6299	power_well->count, i915.disable_power_well);
4560	Serge	6300	}
		6301
5060	serge	6302	void intel_display_power_get(struct drm_i915_private *dev_priv,
4560	Serge	6303	enum intel_display_power_domain domain)
		6304	{
		6305	struct i915_power_domains *power_domains;
		6306	struct i915_power_well *power_well;
		6307	int i;
		6308
5060	serge	6309	intel_runtime_pm_get(dev_priv);
		6310
4560	Serge	6311	power_domains = &dev_priv->power_domains;
		6312
		6313	mutex_lock(&power_domains->lock);
		6314
5060	serge	6315	for_each_power_well(i, power_well, BIT(domain), power_domains) {
		6316	if (!power_well->count++) {
		6317	DRM_DEBUG_KMS("enabling %s\n", power_well->name);
		6318	power_well->ops->enable(dev_priv, power_well);
		6319	power_well->hw_enabled = true;
		6320	}
4560	Serge	6321
5060	serge	6322	check_power_well_state(dev_priv, power_well);
		6323	}
		6324
4560	Serge	6325	power_domains->domain_use_count[domain]++;
		6326
		6327	mutex_unlock(&power_domains->lock);
		6328	}
		6329
5060	serge	6330	void intel_display_power_put(struct drm_i915_private *dev_priv,
4560	Serge	6331	enum intel_display_power_domain domain)
		6332	{
		6333	struct i915_power_domains *power_domains;
		6334	struct i915_power_well *power_well;
		6335	int i;
		6336
		6337	power_domains = &dev_priv->power_domains;
		6338
		6339	mutex_lock(&power_domains->lock);
		6340
		6341	WARN_ON(!power_domains->domain_use_count[domain]);
		6342	power_domains->domain_use_count[domain]--;
		6343
5060	serge	6344	for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
		6345	WARN_ON(!power_well->count);
4560	Serge	6346
5060	serge	6347	if (!--power_well->count && i915.disable_power_well) {
		6348	DRM_DEBUG_KMS("disabling %s\n", power_well->name);
		6349	power_well->hw_enabled = false;
		6350	power_well->ops->disable(dev_priv, power_well);
		6351	}
		6352
		6353	check_power_well_state(dev_priv, power_well);
		6354	}
		6355
4560	Serge	6356	mutex_unlock(&power_domains->lock);
5060	serge	6357
		6358	intel_runtime_pm_put(dev_priv);
4560	Serge	6359	}
		6360
		6361	static struct i915_power_domains *hsw_pwr;
		6362
4104	Serge	6363	/* Display audio driver power well request */
5060	serge	6364	int i915_request_power_well(void)
4104	Serge	6365	{
4560	Serge	6366	struct drm_i915_private *dev_priv;
		6367
5060	serge	6368	if (!hsw_pwr)
		6369	return -ENODEV;
4104	Serge	6370
4560	Serge	6371	dev_priv = container_of(hsw_pwr, struct drm_i915_private,
		6372	power_domains);
5060	serge	6373	intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO);
		6374	return 0;
4104	Serge	6375	}
		6376	EXPORT_SYMBOL_GPL(i915_request_power_well);
		6377
		6378	/* Display audio driver power well release */
5060	serge	6379	int i915_release_power_well(void)
4104	Serge	6380	{
4560	Serge	6381	struct drm_i915_private *dev_priv;
		6382
5060	serge	6383	if (!hsw_pwr)
		6384	return -ENODEV;
4104	Serge	6385
4560	Serge	6386	dev_priv = container_of(hsw_pwr, struct drm_i915_private,
		6387	power_domains);
5060	serge	6388	intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO);
		6389	return 0;
4104	Serge	6390	}
		6391	EXPORT_SYMBOL_GPL(i915_release_power_well);
		6392
5060	serge	6393	/*
		6394	* Private interface for the audio driver to get CDCLK in kHz.
		6395	*
		6396	* Caller must request power well using i915_request_power_well() prior to
		6397	* making the call.
		6398	*/
		6399	int i915_get_cdclk_freq(void)
		6400	{
		6401	struct drm_i915_private *dev_priv;
		6402
		6403	if (!hsw_pwr)
		6404	return -ENODEV;
		6405
		6406	dev_priv = container_of(hsw_pwr, struct drm_i915_private,
		6407	power_domains);
		6408
		6409	return intel_ddi_get_cdclk_freq(dev_priv);
		6410	}
		6411	EXPORT_SYMBOL_GPL(i915_get_cdclk_freq);
		6412
		6413
		6414	#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1)
		6415
		6416	#define HSW_ALWAYS_ON_POWER_DOMAINS ( \
		6417	BIT(POWER_DOMAIN_PIPE_A) \| \
		6418	BIT(POWER_DOMAIN_TRANSCODER_EDP) \| \
		6419	BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) \| \
		6420	BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) \| \
		6421	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) \| \
		6422	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) \| \
		6423	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) \| \
		6424	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) \| \
		6425	BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) \| \
		6426	BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) \| \
		6427	BIT(POWER_DOMAIN_PORT_CRT) \| \
		6428	BIT(POWER_DOMAIN_PLLS) \| \
		6429	BIT(POWER_DOMAIN_INIT))
		6430	#define HSW_DISPLAY_POWER_DOMAINS ( \
		6431	(POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) \| \
		6432	BIT(POWER_DOMAIN_INIT))
		6433
		6434	#define BDW_ALWAYS_ON_POWER_DOMAINS ( \
		6435	HSW_ALWAYS_ON_POWER_DOMAINS \| \
		6436	BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER))
		6437	#define BDW_DISPLAY_POWER_DOMAINS ( \
		6438	(POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS) \| \
		6439	BIT(POWER_DOMAIN_INIT))
		6440
		6441	#define VLV_ALWAYS_ON_POWER_DOMAINS BIT(POWER_DOMAIN_INIT)
		6442	#define VLV_DISPLAY_POWER_DOMAINS POWER_DOMAIN_MASK
		6443
		6444	#define VLV_DPIO_CMN_BC_POWER_DOMAINS ( \
		6445	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) \| \
		6446	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) \| \
		6447	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) \| \
		6448	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) \| \
		6449	BIT(POWER_DOMAIN_PORT_CRT) \| \
		6450	BIT(POWER_DOMAIN_INIT))
		6451
		6452	#define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS ( \
		6453	BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) \| \
		6454	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) \| \
		6455	BIT(POWER_DOMAIN_INIT))
		6456
		6457	#define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS ( \
		6458	BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) \| \
		6459	BIT(POWER_DOMAIN_INIT))
		6460
		6461	#define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS ( \
		6462	BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) \| \
		6463	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) \| \
		6464	BIT(POWER_DOMAIN_INIT))
		6465
		6466	#define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS ( \
		6467	BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) \| \
		6468	BIT(POWER_DOMAIN_INIT))
		6469
		6470	static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
		6471	.sync_hw = i9xx_always_on_power_well_noop,
		6472	.enable = i9xx_always_on_power_well_noop,
		6473	.disable = i9xx_always_on_power_well_noop,
		6474	.is_enabled = i9xx_always_on_power_well_enabled,
		6475	};
		6476
4560	Serge	6477	static struct i915_power_well i9xx_always_on_power_well[] = {
		6478	{
		6479	.name = "always-on",
		6480	.always_on = 1,
		6481	.domains = POWER_DOMAIN_MASK,
5060	serge	6482	.ops = &i9xx_always_on_power_well_ops,
4560	Serge	6483	},
		6484	};
		6485
5060	serge	6486	static const struct i915_power_well_ops hsw_power_well_ops = {
		6487	.sync_hw = hsw_power_well_sync_hw,
		6488	.enable = hsw_power_well_enable,
		6489	.disable = hsw_power_well_disable,
		6490	.is_enabled = hsw_power_well_enabled,
		6491	};
		6492
4560	Serge	6493	static struct i915_power_well hsw_power_wells[] = {
		6494	{
		6495	.name = "always-on",
		6496	.always_on = 1,
		6497	.domains = HSW_ALWAYS_ON_POWER_DOMAINS,
5060	serge	6498	.ops = &i9xx_always_on_power_well_ops,
4560	Serge	6499	},
		6500	{
		6501	.name = "display",
5060	serge	6502	.domains = HSW_DISPLAY_POWER_DOMAINS,
		6503	.ops = &hsw_power_well_ops,
4560	Serge	6504	},
		6505	};
		6506
		6507	static struct i915_power_well bdw_power_wells[] = {
		6508	{
		6509	.name = "always-on",
		6510	.always_on = 1,
		6511	.domains = BDW_ALWAYS_ON_POWER_DOMAINS,
5060	serge	6512	.ops = &i9xx_always_on_power_well_ops,
4560	Serge	6513	},
		6514	{
		6515	.name = "display",
5060	serge	6516	.domains = BDW_DISPLAY_POWER_DOMAINS,
		6517	.ops = &hsw_power_well_ops,
4560	Serge	6518	},
		6519	};
		6520
5060	serge	6521	static const struct i915_power_well_ops vlv_display_power_well_ops = {
		6522	.sync_hw = vlv_power_well_sync_hw,
		6523	.enable = vlv_display_power_well_enable,
		6524	.disable = vlv_display_power_well_disable,
		6525	.is_enabled = vlv_power_well_enabled,
		6526	};
		6527
		6528	static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
		6529	.sync_hw = vlv_power_well_sync_hw,
		6530	.enable = vlv_dpio_cmn_power_well_enable,
		6531	.disable = vlv_dpio_cmn_power_well_disable,
		6532	.is_enabled = vlv_power_well_enabled,
		6533	};
		6534
		6535	static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
		6536	.sync_hw = vlv_power_well_sync_hw,
		6537	.enable = vlv_power_well_enable,
		6538	.disable = vlv_power_well_disable,
		6539	.is_enabled = vlv_power_well_enabled,
		6540	};
		6541
		6542	static struct i915_power_well vlv_power_wells[] = {
		6543	{
		6544	.name = "always-on",
		6545	.always_on = 1,
		6546	.domains = VLV_ALWAYS_ON_POWER_DOMAINS,
		6547	.ops = &i9xx_always_on_power_well_ops,
		6548	},
		6549	{
		6550	.name = "display",
		6551	.domains = VLV_DISPLAY_POWER_DOMAINS,
		6552	.data = PUNIT_POWER_WELL_DISP2D,
		6553	.ops = &vlv_display_power_well_ops,
		6554	},
		6555	{
		6556	.name = "dpio-tx-b-01",
		6557	.domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS \|
		6558	VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS \|
		6559	VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS \|
		6560	VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
		6561	.ops = &vlv_dpio_power_well_ops,
		6562	.data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
		6563	},
		6564	{
		6565	.name = "dpio-tx-b-23",
		6566	.domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS \|
		6567	VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS \|
		6568	VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS \|
		6569	VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
		6570	.ops = &vlv_dpio_power_well_ops,
		6571	.data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
		6572	},
		6573	{
		6574	.name = "dpio-tx-c-01",
		6575	.domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS \|
		6576	VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS \|
		6577	VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS \|
		6578	VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
		6579	.ops = &vlv_dpio_power_well_ops,
		6580	.data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
		6581	},
		6582	{
		6583	.name = "dpio-tx-c-23",
		6584	.domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS \|
		6585	VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS \|
		6586	VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS \|
		6587	VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
		6588	.ops = &vlv_dpio_power_well_ops,
		6589	.data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
		6590	},
		6591	{
		6592	.name = "dpio-common",
		6593	.domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
		6594	.data = PUNIT_POWER_WELL_DPIO_CMN_BC,
		6595	.ops = &vlv_dpio_cmn_power_well_ops,
		6596	},
		6597	};
		6598
		6599	static struct i915_power_well lookup_power_well(struct drm_i915_private dev_priv,
		6600	enum punit_power_well power_well_id)
		6601	{
		6602	struct i915_power_domains *power_domains = &dev_priv->power_domains;
		6603	struct i915_power_well *power_well;
		6604	int i;
		6605
		6606	for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
		6607	if (power_well->data == power_well_id)
		6608	return power_well;
		6609	}
		6610
		6611	return NULL;
		6612	}
		6613
4560	Serge	6614	#define set_power_wells(power_domains, __power_wells) ({ \
		6615	(power_domains)->power_wells = (__power_wells); \
		6616	(power_domains)->power_well_count = ARRAY_SIZE(__power_wells); \
		6617	})
		6618
5060	serge	6619	int intel_power_domains_init(struct drm_i915_private *dev_priv)
4104	Serge	6620	{
4560	Serge	6621	struct i915_power_domains *power_domains = &dev_priv->power_domains;
4104	Serge	6622
4560	Serge	6623	mutex_init(&power_domains->lock);
4104	Serge	6624
4560	Serge	6625	/*
		6626	* The enabling order will be from lower to higher indexed wells,
		6627	* the disabling order is reversed.
		6628	*/
5060	serge	6629	if (IS_HASWELL(dev_priv->dev)) {
4560	Serge	6630	set_power_wells(power_domains, hsw_power_wells);
		6631	hsw_pwr = power_domains;
5060	serge	6632	} else if (IS_BROADWELL(dev_priv->dev)) {
4560	Serge	6633	set_power_wells(power_domains, bdw_power_wells);
		6634	hsw_pwr = power_domains;
5060	serge	6635	} else if (IS_VALLEYVIEW(dev_priv->dev)) {
		6636	set_power_wells(power_domains, vlv_power_wells);
4560	Serge	6637	} else {
		6638	set_power_wells(power_domains, i9xx_always_on_power_well);
		6639	}
4104	Serge	6640
		6641	return 0;
		6642	}
		6643
5060	serge	6644	void intel_power_domains_remove(struct drm_i915_private *dev_priv)
4104	Serge	6645	{
		6646	hsw_pwr = NULL;
		6647	}
		6648
5060	serge	6649	static void intel_power_domains_resume(struct drm_i915_private *dev_priv)
4104	Serge	6650	{
4560	Serge	6651	struct i915_power_domains *power_domains = &dev_priv->power_domains;
		6652	struct i915_power_well *power_well;
		6653	int i;
4104	Serge	6654
4560	Serge	6655	mutex_lock(&power_domains->lock);
		6656	for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
5060	serge	6657	power_well->ops->sync_hw(dev_priv, power_well);
		6658	power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
		6659	power_well);
3031	serge	6660	}
4560	Serge	6661	mutex_unlock(&power_domains->lock);
3480	Serge	6662	}
3031	serge	6663
5060	serge	6664	static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
3480	Serge	6665	{
5060	serge	6666	struct i915_power_well *cmn =
		6667	lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
		6668	struct i915_power_well *disp2d =
		6669	lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
3480	Serge	6670
5060	serge	6671	/* nothing to do if common lane is already off */
		6672	if (!cmn->ops->is_enabled(dev_priv, cmn))
		6673	return;
4560	Serge	6674
5060	serge	6675	/* If the display might be already active skip this */
		6676	if (disp2d->ops->is_enabled(dev_priv, disp2d) &&
		6677	I915_READ(DPIO_CTL) & DPIO_CMNRST)
3480	Serge	6678	return;
		6679
5060	serge	6680	DRM_DEBUG_KMS("toggling display PHY side reset\n");
		6681
		6682	/* cmnlane needs DPLL registers */
		6683	disp2d->ops->enable(dev_priv, disp2d);
		6684
		6685	/*
		6686	* From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
		6687	* Need to assert and de-assert PHY SB reset by gating the
		6688	* common lane power, then un-gating it.
		6689	* Simply ungating isn't enough to reset the PHY enough to get
		6690	* ports and lanes running.
		6691	*/
		6692	cmn->ops->disable(dev_priv, cmn);
3031	serge	6693	}
		6694
5060	serge	6695	void intel_power_domains_init_hw(struct drm_i915_private *dev_priv)
		6696	{
		6697	struct drm_device *dev = dev_priv->dev;
		6698	struct i915_power_domains *power_domains = &dev_priv->power_domains;
		6699
		6700	power_domains->initializing = true;
		6701
		6702	if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
		6703	mutex_lock(&power_domains->lock);
		6704	vlv_cmnlane_wa(dev_priv);
		6705	mutex_unlock(&power_domains->lock);
		6706	}
		6707
		6708	/* For now, we need the power well to be always enabled. */
		6709	intel_display_set_init_power(dev_priv, true);
		6710	intel_power_domains_resume(dev_priv);
		6711	power_domains->initializing = false;
		6712	}
		6713
4104	Serge	6714	void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
		6715	{
5060	serge	6716	intel_runtime_pm_get(dev_priv);
4104	Serge	6717	}
		6718
		6719	void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
		6720	{
5060	serge	6721	intel_runtime_pm_put(dev_priv);
4104	Serge	6722	}
		6723
4560	Serge	6724	void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
		6725	{
		6726	struct drm_device *dev = dev_priv->dev;
		6727	struct device *device = &dev->pdev->dev;
		6728
5060	serge	6729	if (!HAS_RUNTIME_PM(dev))
4560	Serge	6730	return;
5060	serge	6731
		6732	// pm_runtime_get_sync(device);
		6733	WARN(dev_priv->pm.suspended, "Device still suspended.\n");
4560	Serge	6734	}
		6735
5060	serge	6736	void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
		6737	{
		6738	struct drm_device *dev = dev_priv->dev;
		6739	struct device *device = &dev->pdev->dev;
		6740
		6741	if (!HAS_RUNTIME_PM(dev))
		6742	return;
		6743
		6744	WARN(dev_priv->pm.suspended, "Getting nosync-ref while suspended.\n");
		6745	// pm_runtime_get_noresume(device);
		6746	}
		6747
4560	Serge	6748	void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
		6749	{
		6750	struct drm_device *dev = dev_priv->dev;
		6751	struct device *device = &dev->pdev->dev;
		6752
5060	serge	6753	if (!HAS_RUNTIME_PM(dev))
4560	Serge	6754	return;
		6755
		6756	}
		6757
		6758	void intel_init_runtime_pm(struct drm_i915_private *dev_priv)
		6759	{
		6760	struct drm_device *dev = dev_priv->dev;
		6761	struct device *device = &dev->pdev->dev;
		6762
5060	serge	6763	if (!HAS_RUNTIME_PM(dev))
		6764	return;
4560	Serge	6765
5060	serge	6766
		6767	/*
		6768	* RPM depends on RC6 to save restore the GT HW context, so make RC6 a
		6769	* requirement.
		6770	*/
		6771	if (!intel_enable_rc6(dev)) {
		6772	DRM_INFO("RC6 disabled, disabling runtime PM support\n");
4560	Serge	6773	return;
5060	serge	6774	}
4560	Serge	6775
5060	serge	6776
4560	Serge	6777	}
		6778
		6779	void intel_fini_runtime_pm(struct drm_i915_private *dev_priv)
		6780	{
		6781	struct drm_device *dev = dev_priv->dev;
		6782	struct device *device = &dev->pdev->dev;
		6783
5060	serge	6784	if (!HAS_RUNTIME_PM(dev))
		6785	return;
		6786
		6787	if (!intel_enable_rc6(dev))
4560	Serge	6788	return;
		6789
		6790	}
		6791
3031	serge	6792	/* Set up chip specific power management-related functions */
		6793	void intel_init_pm(struct drm_device *dev)
		6794	{
		6795	struct drm_i915_private *dev_priv = dev->dev_private;
		6796
4560	Serge	6797	if (HAS_FBC(dev)) {
		6798	if (INTEL_INFO(dev)->gen >= 7) {
3031	serge	6799	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
4560	Serge	6800	dev_priv->display.enable_fbc = gen7_enable_fbc;
3031	serge	6801	dev_priv->display.disable_fbc = ironlake_disable_fbc;
4560	Serge	6802	} else if (INTEL_INFO(dev)->gen >= 5) {
		6803	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
		6804	dev_priv->display.enable_fbc = ironlake_enable_fbc;
		6805	dev_priv->display.disable_fbc = ironlake_disable_fbc;
3031	serge	6806	} else if (IS_GM45(dev)) {
		6807	dev_priv->display.fbc_enabled = g4x_fbc_enabled;
		6808	dev_priv->display.enable_fbc = g4x_enable_fbc;
		6809	dev_priv->display.disable_fbc = g4x_disable_fbc;
4560	Serge	6810	} else {
3031	serge	6811	dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
		6812	dev_priv->display.enable_fbc = i8xx_enable_fbc;
		6813	dev_priv->display.disable_fbc = i8xx_disable_fbc;
4560	Serge	6814
		6815	/* This value was pulled out of someone's hat */
		6816	I915_WRITE(FBC_CONTROL, 500 << FBC_CTL_INTERVAL_SHIFT);
3031	serge	6817	}
		6818	}
		6819
		6820	/* For cxsr */
		6821	if (IS_PINEVIEW(dev))
		6822	i915_pineview_get_mem_freq(dev);
		6823	else if (IS_GEN5(dev))
		6824	i915_ironlake_get_mem_freq(dev);
		6825
		6826	/* For FIFO watermark updates */
		6827	if (HAS_PCH_SPLIT(dev)) {
5060	serge	6828	ilk_setup_wm_latency(dev);
4104	Serge	6829
4560	Serge	6830	if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
		6831	dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) \|\|
		6832	(!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
		6833	dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
		6834	dev_priv->display.update_wm = ilk_update_wm;
		6835	dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
3031	serge	6836	} else {
		6837	DRM_DEBUG_KMS("Failed to read display plane latency. "
		6838	"Disable CxSR\n");
		6839	}
4560	Serge	6840
		6841	if (IS_GEN5(dev))
		6842	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		6843	else if (IS_GEN6(dev))
3031	serge	6844	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
4560	Serge	6845	else if (IS_IVYBRIDGE(dev))
3031	serge	6846	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
4560	Serge	6847	else if (IS_HASWELL(dev))
3031	serge	6848	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
4560	Serge	6849	else if (INTEL_INFO(dev)->gen == 8)
		6850	dev_priv->display.init_clock_gating = gen8_init_clock_gating;
5060	serge	6851	} else if (IS_CHERRYVIEW(dev)) {
		6852	dev_priv->display.update_wm = valleyview_update_wm;
		6853	dev_priv->display.init_clock_gating =
		6854	cherryview_init_clock_gating;
3031	serge	6855	} else if (IS_VALLEYVIEW(dev)) {
		6856	dev_priv->display.update_wm = valleyview_update_wm;
		6857	dev_priv->display.init_clock_gating =
		6858	valleyview_init_clock_gating;
		6859	} else if (IS_PINEVIEW(dev)) {
		6860	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		6861	dev_priv->is_ddr3,
		6862	dev_priv->fsb_freq,
		6863	dev_priv->mem_freq)) {
		6864	DRM_INFO("failed to find known CxSR latency "
		6865	"(found ddr%s fsb freq %d, mem freq %d), "
		6866	"disabling CxSR\n",
		6867	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		6868	dev_priv->fsb_freq, dev_priv->mem_freq);
		6869	/* Disable CxSR and never update its watermark again */
5060	serge	6870	intel_set_memory_cxsr(dev_priv, false);
3031	serge	6871	dev_priv->display.update_wm = NULL;
		6872	} else
		6873	dev_priv->display.update_wm = pineview_update_wm;
		6874	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		6875	} else if (IS_G4X(dev)) {
		6876	dev_priv->display.update_wm = g4x_update_wm;
		6877	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		6878	} else if (IS_GEN4(dev)) {
		6879	dev_priv->display.update_wm = i965_update_wm;
		6880	if (IS_CRESTLINE(dev))
		6881	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		6882	else if (IS_BROADWATER(dev))
		6883	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		6884	} else if (IS_GEN3(dev)) {
		6885	dev_priv->display.update_wm = i9xx_update_wm;
		6886	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		6887	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
4560	Serge	6888	} else if (IS_GEN2(dev)) {
		6889	if (INTEL_INFO(dev)->num_pipes == 1) {
		6890	dev_priv->display.update_wm = i845_update_wm;
		6891	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		6892	} else {
		6893	dev_priv->display.update_wm = i9xx_update_wm;
3031	serge	6894	dev_priv->display.get_fifo_size = i830_get_fifo_size;
4560	Serge	6895	}
		6896
		6897	if (IS_I85X(dev) \|\| IS_I865G(dev))
3031	serge	6898	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
4560	Serge	6899	else
		6900	dev_priv->display.init_clock_gating = i830_init_clock_gating;
3031	serge	6901	} else {
4560	Serge	6902	DRM_ERROR("unexpected fall-through in intel_init_pm\n");
3031	serge	6903	}
		6904	}
		6905
3243	Serge	6906	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u8 mbox, u32 val)
		6907	{
		6908	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	6909
3243	Serge	6910	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		6911	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		6912	return -EAGAIN;
		6913	}
3031	serge	6914
3243	Serge	6915	I915_WRITE(GEN6_PCODE_DATA, *val);
		6916	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		6917
		6918	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		6919	500)) {
		6920	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		6921	return -ETIMEDOUT;
3031	serge	6922	}
3243	Serge	6923
		6924	*val = I915_READ(GEN6_PCODE_DATA);
		6925	I915_WRITE(GEN6_PCODE_DATA, 0);
		6926
		6927	return 0;
		6928	}
		6929
		6930	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
		6931	{
		6932	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		6933
		6934	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		6935	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		6936	return -EAGAIN;
3031	serge	6937	}
3243	Serge	6938
		6939	I915_WRITE(GEN6_PCODE_DATA, val);
		6940	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		6941
		6942	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		6943	500)) {
		6944	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		6945	return -ETIMEDOUT;
3031	serge	6946	}
3243	Serge	6947
		6948	I915_WRITE(GEN6_PCODE_DATA, 0);
		6949
		6950	return 0;
3031	serge	6951	}
3746	Serge	6952
5060	serge	6953	static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
3746	Serge	6954	{
4560	Serge	6955	int div;
3746	Serge	6956
4560	Serge	6957	/* 4 x czclk */
		6958	switch (dev_priv->mem_freq) {
4104	Serge	6959	case 800:
4560	Serge	6960	div = 10;
4104	Serge	6961	break;
		6962	case 1066:
4560	Serge	6963	div = 12;
4104	Serge	6964	break;
		6965	case 1333:
4560	Serge	6966	div = 16;
4104	Serge	6967	break;
		6968	default:
		6969	return -1;
		6970	}
3746	Serge	6971
4560	Serge	6972	return DIV_ROUND_CLOSEST(dev_priv->mem_freq * (val + 6 - 0xbd), 4 * div);
4104	Serge	6973	}
3746	Serge	6974
5060	serge	6975	static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
4104	Serge	6976	{
4560	Serge	6977	int mul;
3746	Serge	6978
4560	Serge	6979	/* 4 x czclk */
		6980	switch (dev_priv->mem_freq) {
4104	Serge	6981	case 800:
4560	Serge	6982	mul = 10;
4104	Serge	6983	break;
		6984	case 1066:
4560	Serge	6985	mul = 12;
4104	Serge	6986	break;
		6987	case 1333:
4560	Serge	6988	mul = 16;
4104	Serge	6989	break;
		6990	default:
		6991	return -1;
3746	Serge	6992	}
		6993
4560	Serge	6994	return DIV_ROUND_CLOSEST(4 * mul * val, dev_priv->mem_freq) + 0xbd - 6;
3746	Serge	6995	}
		6996
5060	serge	6997	static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
		6998	{
		6999	int div, freq;
		7000
		7001	switch (dev_priv->rps.cz_freq) {
		7002	case 200:
		7003	div = 5;
		7004	break;
		7005	case 267:
		7006	div = 6;
		7007	break;
		7008	case 320:
		7009	case 333:
		7010	case 400:
		7011	div = 8;
		7012	break;
		7013	default:
		7014	return -1;
		7015	}
		7016
		7017	freq = (DIV_ROUND_CLOSEST((dev_priv->rps.cz_freq * val), 2 * div) / 2);
		7018
		7019	return freq;
		7020	}
		7021
		7022	static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7023	{
		7024	int mul, opcode;
		7025
		7026	switch (dev_priv->rps.cz_freq) {
		7027	case 200:
		7028	mul = 5;
		7029	break;
		7030	case 267:
		7031	mul = 6;
		7032	break;
		7033	case 320:
		7034	case 333:
		7035	case 400:
		7036	mul = 8;
		7037	break;
		7038	default:
		7039	return -1;
		7040	}
		7041
		7042	opcode = (DIV_ROUND_CLOSEST((val * 2 * mul), dev_priv->rps.cz_freq) * 2);
		7043
		7044	return opcode;
		7045	}
		7046
		7047	int vlv_gpu_freq(struct drm_i915_private *dev_priv, int val)
		7048	{
		7049	int ret = -1;
		7050
		7051	if (IS_CHERRYVIEW(dev_priv->dev))
		7052	ret = chv_gpu_freq(dev_priv, val);
		7053	else if (IS_VALLEYVIEW(dev_priv->dev))
		7054	ret = byt_gpu_freq(dev_priv, val);
		7055
		7056	return ret;
		7057	}
		7058
		7059	int vlv_freq_opcode(struct drm_i915_private *dev_priv, int val)
		7060	{
		7061	int ret = -1;
		7062
		7063	if (IS_CHERRYVIEW(dev_priv->dev))
		7064	ret = chv_freq_opcode(dev_priv, val);
		7065	else if (IS_VALLEYVIEW(dev_priv->dev))
		7066	ret = byt_freq_opcode(dev_priv, val);
		7067
		7068	return ret;
		7069	}
		7070
4560	Serge	7071	void intel_pm_setup(struct drm_device *dev)
3746	Serge	7072	{
4104	Serge	7073	struct drm_i915_private *dev_priv = dev->dev_private;
		7074
4560	Serge	7075	mutex_init(&dev_priv->rps.hw_lock);
		7076
4104	Serge	7077	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		7078	intel_gen6_powersave_work);
5060	serge	7079
		7080	dev_priv->pm.suspended = false;
		7081	dev_priv->pm._irqs_disabled = false;
3746	Serge	7082	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 5060