WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	#define iowrite32(v, addr) writel((v), (addr))
		29	#define ioread32(addr) readl(addr)
		30
		31	//#include
		32	#include "i915_drv.h"
		33	#include "intel_drv.h"
		34	#include
		35	//#include "../../../platform/x86/intel_ips.h"
		36	#include
		37
		38	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		39
		40	#define assert_spin_locked(x)
		41
		42	void getrawmonotonic(struct timespec *ts);
		43	void set_normalized_timespec(struct timespec *ts, time_t sec, long nsec);
		44
		45	static inline struct timespec timespec_sub(struct timespec lhs,
		46	struct timespec rhs)
		47	{
		48	struct timespec ts_delta;
		49	set_normalized_timespec(&ts_delta, lhs.tv_sec - rhs.tv_sec,
		50	lhs.tv_nsec - rhs.tv_nsec);
		51	return ts_delta;
		52	}
		53
		54
		55	/* FBC, or Frame Buffer Compression, is a technique employed to compress the
		56	* framebuffer contents in-memory, aiming at reducing the required bandwidth
		57	* during in-memory transfers and, therefore, reduce the power packet.
		58	*
		59	* The benefits of FBC are mostly visible with solid backgrounds and
		60	* variation-less patterns.
		61	*
		62	* FBC-related functionality can be enabled by the means of the
		63	* i915.i915_enable_fbc parameter
		64	*/
		65
		66	static void i8xx_disable_fbc(struct drm_device *dev)
		67	{
		68	struct drm_i915_private *dev_priv = dev->dev_private;
		69	u32 fbc_ctl;
		70
		71	/* Disable compression */
		72	fbc_ctl = I915_READ(FBC_CONTROL);
		73	if ((fbc_ctl & FBC_CTL_EN) == 0)
		74	return;
		75
		76	fbc_ctl &= ~FBC_CTL_EN;
		77	I915_WRITE(FBC_CONTROL, fbc_ctl);
		78
		79	/* Wait for compressing bit to clear */
		80	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
		81	DRM_DEBUG_KMS("FBC idle timed out\n");
		82	return;
		83	}
		84
		85	DRM_DEBUG_KMS("disabled FBC\n");
		86	}
		87
		88	static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		89	{
		90	struct drm_device *dev = crtc->dev;
		91	struct drm_i915_private *dev_priv = dev->dev_private;
		92	struct drm_framebuffer *fb = crtc->fb;
		93	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		94	struct drm_i915_gem_object *obj = intel_fb->obj;
		95	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		96	int cfb_pitch;
		97	int plane, i;
		98	u32 fbc_ctl, fbc_ctl2;
		99
		100	cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
		101	if (fb->pitches[0] < cfb_pitch)
		102	cfb_pitch = fb->pitches[0];
		103
		104	/* FBC_CTL wants 64B units */
		105	cfb_pitch = (cfb_pitch / 64) - 1;
		106	plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
		107
		108	/* Clear old tags */
		109	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
		110	I915_WRITE(FBC_TAG + (i * 4), 0);
		111
		112	/* Set it up... */
		113	fbc_ctl2 = FBC_CTL_FENCE_DBL \| FBC_CTL_IDLE_IMM \| FBC_CTL_CPU_FENCE;
		114	fbc_ctl2 \|= plane;
		115	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
		116	I915_WRITE(FBC_FENCE_OFF, crtc->y);
		117
		118	/* enable it... */
		119	fbc_ctl = FBC_CTL_EN \| FBC_CTL_PERIODIC;
		120	if (IS_I945GM(dev))
		121	fbc_ctl \|= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
		122	fbc_ctl \|= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
		123	fbc_ctl \|= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
		124	fbc_ctl \|= obj->fence_reg;
		125	I915_WRITE(FBC_CONTROL, fbc_ctl);
		126
		127	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
		128	cfb_pitch, crtc->y, intel_crtc->plane);
		129	}
		130
		131	static bool i8xx_fbc_enabled(struct drm_device *dev)
		132	{
		133	struct drm_i915_private *dev_priv = dev->dev_private;
		134
		135	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
		136	}
		137
		138	static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		139	{
		140	struct drm_device *dev = crtc->dev;
		141	struct drm_i915_private *dev_priv = dev->dev_private;
		142	struct drm_framebuffer *fb = crtc->fb;
		143	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		144	struct drm_i915_gem_object *obj = intel_fb->obj;
		145	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		146	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		147	unsigned long stall_watermark = 200;
		148	u32 dpfc_ctl;
		149
		150	dpfc_ctl = plane \| DPFC_SR_EN \| DPFC_CTL_LIMIT_1X;
		151	dpfc_ctl \|= DPFC_CTL_FENCE_EN \| obj->fence_reg;
		152	I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
		153
		154	I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		155	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		156	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		157	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
		158
		159	/* enable it... */
		160	I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) \| DPFC_CTL_EN);
		161
		162	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		163	}
		164
		165	static void g4x_disable_fbc(struct drm_device *dev)
		166	{
		167	struct drm_i915_private *dev_priv = dev->dev_private;
		168	u32 dpfc_ctl;
		169
		170	/* Disable compression */
		171	dpfc_ctl = I915_READ(DPFC_CONTROL);
		172	if (dpfc_ctl & DPFC_CTL_EN) {
		173	dpfc_ctl &= ~DPFC_CTL_EN;
		174	I915_WRITE(DPFC_CONTROL, dpfc_ctl);
		175
		176	DRM_DEBUG_KMS("disabled FBC\n");
		177	}
		178	}
		179
		180	static bool g4x_fbc_enabled(struct drm_device *dev)
		181	{
		182	struct drm_i915_private *dev_priv = dev->dev_private;
		183
		184	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
		185	}
		186
		187	static void sandybridge_blit_fbc_update(struct drm_device *dev)
		188	{
		189	struct drm_i915_private *dev_priv = dev->dev_private;
		190	u32 blt_ecoskpd;
		191
		192	/* Make sure blitter notifies FBC of writes */
		193	gen6_gt_force_wake_get(dev_priv);
		194	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
		195	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY <<
		196	GEN6_BLITTER_LOCK_SHIFT;
		197	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		198	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY;
		199	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		200	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
		201	GEN6_BLITTER_LOCK_SHIFT);
		202	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		203	POSTING_READ(GEN6_BLITTER_ECOSKPD);
		204	gen6_gt_force_wake_put(dev_priv);
		205	}
		206
		207	static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		208	{
		209	struct drm_device *dev = crtc->dev;
		210	struct drm_i915_private *dev_priv = dev->dev_private;
		211	struct drm_framebuffer *fb = crtc->fb;
		212	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		213	struct drm_i915_gem_object *obj = intel_fb->obj;
		214	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		215	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		216	unsigned long stall_watermark = 200;
		217	u32 dpfc_ctl;
		218
		219	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		220	dpfc_ctl &= DPFC_RESERVED;
		221	dpfc_ctl \|= (plane \| DPFC_CTL_LIMIT_1X);
		222	/* Set persistent mode for front-buffer rendering, ala X. */
		223	dpfc_ctl \|= DPFC_CTL_PERSISTENT_MODE;
		224	dpfc_ctl \|= (DPFC_CTL_FENCE_EN \| obj->fence_reg);
		225	I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
		226
		227	I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		228	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		229	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		230	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
		231	I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset \| ILK_FBC_RT_VALID);
		232	/* enable it... */
		233	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		234
		235	if (IS_GEN6(dev)) {
		236	I915_WRITE(SNB_DPFC_CTL_SA,
		237	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		238	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		239	sandybridge_blit_fbc_update(dev);
		240	}
		241
		242	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		243	}
		244
		245	static void ironlake_disable_fbc(struct drm_device *dev)
		246	{
		247	struct drm_i915_private *dev_priv = dev->dev_private;
		248	u32 dpfc_ctl;
		249
		250	/* Disable compression */
		251	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		252	if (dpfc_ctl & DPFC_CTL_EN) {
		253	dpfc_ctl &= ~DPFC_CTL_EN;
		254	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
		255
		256	DRM_DEBUG_KMS("disabled FBC\n");
		257	}
		258	}
		259
		260	static bool ironlake_fbc_enabled(struct drm_device *dev)
		261	{
		262	struct drm_i915_private *dev_priv = dev->dev_private;
		263
		264	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
		265	}
		266
		267	bool intel_fbc_enabled(struct drm_device *dev)
		268	{
		269	struct drm_i915_private *dev_priv = dev->dev_private;
		270
		271	if (!dev_priv->display.fbc_enabled)
		272	return false;
		273
		274	return dev_priv->display.fbc_enabled(dev);
		275	}
		276
		277	#if 0
		278	static void intel_fbc_work_fn(struct work_struct *__work)
		279	{
		280	struct intel_fbc_work *work =
		281	container_of(to_delayed_work(__work),
		282	struct intel_fbc_work, work);
		283	struct drm_device *dev = work->crtc->dev;
		284	struct drm_i915_private *dev_priv = dev->dev_private;
		285
		286	mutex_lock(&dev->struct_mutex);
		287	if (work == dev_priv->fbc_work) {
		288	/* Double check that we haven't switched fb without cancelling
		289	* the prior work.
		290	*/
		291	if (work->crtc->fb == work->fb) {
		292	dev_priv->display.enable_fbc(work->crtc,
		293	work->interval);
		294
		295	dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
		296	dev_priv->cfb_fb = work->crtc->fb->base.id;
		297	dev_priv->cfb_y = work->crtc->y;
		298	}
		299
		300	dev_priv->fbc_work = NULL;
		301	}
		302	mutex_unlock(&dev->struct_mutex);
		303
		304	kfree(work);
		305	}
		306
		307	static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
		308	{
		309	if (dev_priv->fbc_work == NULL)
		310	return;
		311
		312	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
		313
		314	/* Synchronisation is provided by struct_mutex and checking of
		315	* dev_priv->fbc_work, so we can perform the cancellation
		316	* entirely asynchronously.
		317	*/
		318	if (cancel_delayed_work(&dev_priv->fbc_work->work))
		319	/* tasklet was killed before being run, clean up */
		320	kfree(dev_priv->fbc_work);
		321
		322	/* Mark the work as no longer wanted so that if it does
		323	* wake-up (because the work was already running and waiting
		324	* for our mutex), it will discover that is no longer
		325	* necessary to run.
		326	*/
		327	dev_priv->fbc_work = NULL;
		328	}
		329	#endif
		330
		331	void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		332	{
		333	struct intel_fbc_work *work;
		334	struct drm_device *dev = crtc->dev;
		335	struct drm_i915_private *dev_priv = dev->dev_private;
		336
		337	// if (!dev_priv->display.enable_fbc)
		338	return;
		339	#if 0
		340	intel_cancel_fbc_work(dev_priv);
		341
		342	work = kzalloc(sizeof *work, GFP_KERNEL);
		343	if (work == NULL) {
		344	dev_priv->display.enable_fbc(crtc, interval);
		345	return;
		346	}
		347
		348	work->crtc = crtc;
		349	work->fb = crtc->fb;
		350	work->interval = interval;
		351	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
		352
		353	dev_priv->fbc_work = work;
		354
		355	DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
		356
		357	/* Delay the actual enabling to let pageflipping cease and the
		358	* display to settle before starting the compression. Note that
		359	* this delay also serves a second purpose: it allows for a
		360	* vblank to pass after disabling the FBC before we attempt
		361	* to modify the control registers.
		362	*
		363	* A more complicated solution would involve tracking vblanks
		364	* following the termination of the page-flipping sequence
		365	* and indeed performing the enable as a co-routine and not
		366	* waiting synchronously upon the vblank.
		367	*/
		368	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
		369	#endif
		370
		371	}
		372
		373	void intel_disable_fbc(struct drm_device *dev)
		374	{
		375	struct drm_i915_private *dev_priv = dev->dev_private;
		376
		377	// intel_cancel_fbc_work(dev_priv);
		378
		379	// if (!dev_priv->display.disable_fbc)
		380	// return;
		381
		382	// dev_priv->display.disable_fbc(dev);
		383	dev_priv->cfb_plane = -1;
		384	}
		385
		386	/**
		387	* intel_update_fbc - enable/disable FBC as needed
		388	* @dev: the drm_device
		389	*
		390	* Set up the framebuffer compression hardware at mode set time. We
		391	* enable it if possible:
		392	* - plane A only (on pre-965)
		393	* - no pixel mulitply/line duplication
		394	* - no alpha buffer discard
		395	* - no dual wide
		396	* - framebuffer <= 2048 in width, 1536 in height
		397	*
		398	* We can't assume that any compression will take place (worst case),
		399	* so the compressed buffer has to be the same size as the uncompressed
		400	* one. It also must reside (along with the line length buffer) in
		401	* stolen memory.
		402	*
		403	* We need to enable/disable FBC on a global basis.
		404	*/
		405	void intel_update_fbc(struct drm_device *dev)
		406	{
		407	struct drm_i915_private *dev_priv = dev->dev_private;
		408	struct drm_crtc crtc = NULL, tmp_crtc;
		409	struct intel_crtc *intel_crtc;
		410	struct drm_framebuffer *fb;
		411	struct intel_framebuffer *intel_fb;
		412	struct drm_i915_gem_object *obj;
		413	int enable_fbc;
		414
		415	if (!i915_powersave)
		416	return;
		417
		418	if (!I915_HAS_FBC(dev))
		419	return;
		420
		421	/*
		422	* If FBC is already on, we just have to verify that we can
		423	* keep it that way...
		424	* Need to disable if:
		425	* - more than one pipe is active
		426	* - changing FBC params (stride, fence, mode)
		427	* - new fb is too large to fit in compressed buffer
		428	* - going to an unsupported config (interlace, pixel multiply, etc.)
		429	*/
		430	list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
		431	if (tmp_crtc->enabled &&
		432	!to_intel_crtc(tmp_crtc)->primary_disabled &&
		433	tmp_crtc->fb) {
		434	if (crtc) {
		435	DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
		436	dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
		437	goto out_disable;
		438	}
		439	crtc = tmp_crtc;
		440	}
		441	}
		442
		443	if (!crtc \|\| crtc->fb == NULL) {
		444	DRM_DEBUG_KMS("no output, disabling\n");
		445	dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
		446	goto out_disable;
		447	}
		448
		449	intel_crtc = to_intel_crtc(crtc);
		450	fb = crtc->fb;
		451	intel_fb = to_intel_framebuffer(fb);
		452	obj = intel_fb->obj;
		453
		454	enable_fbc = i915_enable_fbc;
		455	if (enable_fbc < 0) {
		456	DRM_DEBUG_KMS("fbc set to per-chip default\n");
		457	enable_fbc = 1;
		458	if (INTEL_INFO(dev)->gen <= 6)
		459	enable_fbc = 0;
		460	}
		461	if (!enable_fbc) {
		462	DRM_DEBUG_KMS("fbc disabled per module param\n");
		463	dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
		464	goto out_disable;
		465	}
		466	if (intel_fb->obj->base.size > dev_priv->cfb_size) {
		467	DRM_DEBUG_KMS("framebuffer too large, disabling "
		468	"compression\n");
		469	dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
		470	goto out_disable;
		471	}
		472	if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) \|\|
		473	(crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
		474	DRM_DEBUG_KMS("mode incompatible with compression, "
		475	"disabling\n");
		476	dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
		477	goto out_disable;
		478	}
		479	if ((crtc->mode.hdisplay > 2048) \|\|
		480	(crtc->mode.vdisplay > 1536)) {
		481	DRM_DEBUG_KMS("mode too large for compression, disabling\n");
		482	dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
		483	goto out_disable;
		484	}
		485	if ((IS_I915GM(dev) \|\| IS_I945GM(dev)) && intel_crtc->plane != 0) {
		486	DRM_DEBUG_KMS("plane not 0, disabling compression\n");
		487	dev_priv->no_fbc_reason = FBC_BAD_PLANE;
		488	goto out_disable;
		489	}
		490
		491	/* The use of a CPU fence is mandatory in order to detect writes
		492	* by the CPU to the scanout and trigger updates to the FBC.
		493	*/
		494	if (obj->tiling_mode != I915_TILING_X \|\|
		495	obj->fence_reg == I915_FENCE_REG_NONE) {
		496	DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
		497	dev_priv->no_fbc_reason = FBC_NOT_TILED;
		498	goto out_disable;
		499	}
		500
		501	/* If the kernel debugger is active, always disable compression */
		502	if (in_dbg_master())
		503	goto out_disable;
		504
		505	/* If the scanout has not changed, don't modify the FBC settings.
		506	* Note that we make the fundamental assumption that the fb->obj
		507	* cannot be unpinned (and have its GTT offset and fence revoked)
		508	* without first being decoupled from the scanout and FBC disabled.
		509	*/
		510	if (dev_priv->cfb_plane == intel_crtc->plane &&
		511	dev_priv->cfb_fb == fb->base.id &&
		512	dev_priv->cfb_y == crtc->y)
		513	return;
		514
		515	if (intel_fbc_enabled(dev)) {
		516	/* We update FBC along two paths, after changing fb/crtc
		517	* configuration (modeswitching) and after page-flipping
		518	* finishes. For the latter, we know that not only did
		519	* we disable the FBC at the start of the page-flip
		520	* sequence, but also more than one vblank has passed.
		521	*
		522	* For the former case of modeswitching, it is possible
		523	* to switch between two FBC valid configurations
		524	* instantaneously so we do need to disable the FBC
		525	* before we can modify its control registers. We also
		526	* have to wait for the next vblank for that to take
		527	* effect. However, since we delay enabling FBC we can
		528	* assume that a vblank has passed since disabling and
		529	* that we can safely alter the registers in the deferred
		530	* callback.
		531	*
		532	* In the scenario that we go from a valid to invalid
		533	* and then back to valid FBC configuration we have
		534	* no strict enforcement that a vblank occurred since
		535	* disabling the FBC. However, along all current pipe
		536	* disabling paths we do need to wait for a vblank at
		537	* some point. And we wait before enabling FBC anyway.
		538	*/
		539	DRM_DEBUG_KMS("disabling active FBC for update\n");
		540	intel_disable_fbc(dev);
		541	}
		542
		543	intel_enable_fbc(crtc, 500);
		544	return;
		545
		546	out_disable:
		547	/* Multiple disables should be harmless */
		548	if (intel_fbc_enabled(dev)) {
		549	DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
		550	intel_disable_fbc(dev);
		551	}
		552	}
		553
		554	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		555	{
		556	drm_i915_private_t *dev_priv = dev->dev_private;
		557	u32 tmp;
		558
		559	tmp = I915_READ(CLKCFG);
		560
		561	switch (tmp & CLKCFG_FSB_MASK) {
		562	case CLKCFG_FSB_533:
		563	dev_priv->fsb_freq = 533; /* 1334 /
		564	break;
		565	case CLKCFG_FSB_800:
		566	dev_priv->fsb_freq = 800; /* 2004 /
		567	break;
		568	case CLKCFG_FSB_667:
		569	dev_priv->fsb_freq = 667; /* 1674 /
		570	break;
		571	case CLKCFG_FSB_400:
		572	dev_priv->fsb_freq = 400; /* 1004 /
		573	break;
		574	}
		575
		576	switch (tmp & CLKCFG_MEM_MASK) {
		577	case CLKCFG_MEM_533:
		578	dev_priv->mem_freq = 533;
		579	break;
		580	case CLKCFG_MEM_667:
		581	dev_priv->mem_freq = 667;
		582	break;
		583	case CLKCFG_MEM_800:
		584	dev_priv->mem_freq = 800;
		585	break;
		586	}
		587
		588	/* detect pineview DDR3 setting */
		589	tmp = I915_READ(CSHRDDR3CTL);
		590	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		591	}
		592
		593	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		594	{
		595	drm_i915_private_t *dev_priv = dev->dev_private;
		596	u16 ddrpll, csipll;
		597
		598	ddrpll = I915_READ16(DDRMPLL1);
		599	csipll = I915_READ16(CSIPLL0);
		600
		601	switch (ddrpll & 0xff) {
		602	case 0xc:
		603	dev_priv->mem_freq = 800;
		604	break;
		605	case 0x10:
		606	dev_priv->mem_freq = 1066;
		607	break;
		608	case 0x14:
		609	dev_priv->mem_freq = 1333;
		610	break;
		611	case 0x18:
		612	dev_priv->mem_freq = 1600;
		613	break;
		614	default:
		615	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		616	ddrpll & 0xff);
		617	dev_priv->mem_freq = 0;
		618	break;
		619	}
		620
		621	dev_priv->ips.r_t = dev_priv->mem_freq;
		622
		623	switch (csipll & 0x3ff) {
		624	case 0x00c:
		625	dev_priv->fsb_freq = 3200;
		626	break;
		627	case 0x00e:
		628	dev_priv->fsb_freq = 3733;
		629	break;
		630	case 0x010:
		631	dev_priv->fsb_freq = 4266;
		632	break;
		633	case 0x012:
		634	dev_priv->fsb_freq = 4800;
		635	break;
		636	case 0x014:
		637	dev_priv->fsb_freq = 5333;
		638	break;
		639	case 0x016:
		640	dev_priv->fsb_freq = 5866;
		641	break;
		642	case 0x018:
		643	dev_priv->fsb_freq = 6400;
		644	break;
		645	default:
		646	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		647	csipll & 0x3ff);
		648	dev_priv->fsb_freq = 0;
		649	break;
		650	}
		651
		652	if (dev_priv->fsb_freq == 3200) {
		653	dev_priv->ips.c_m = 0;
		654	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		655	dev_priv->ips.c_m = 1;
		656	} else {
		657	dev_priv->ips.c_m = 2;
		658	}
		659	}
		660
		661	static const struct cxsr_latency cxsr_latency_table[] = {
		662	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		663	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		664	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		665	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		666	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		667
		668	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		669	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		670	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		671	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		672	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		673
		674	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		675	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		676	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		677	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		678	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		679
		680	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		681	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		682	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		683	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		684	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		685
		686	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		687	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		688	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		689	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		690	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		691
		692	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		693	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		694	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		695	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		696	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		697	};
		698
		699	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		700	int is_ddr3,
		701	int fsb,
		702	int mem)
		703	{
		704	const struct cxsr_latency *latency;
		705	int i;
		706
		707	if (fsb == 0 \|\| mem == 0)
		708	return NULL;
		709
		710	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		711	latency = &cxsr_latency_table[i];
		712	if (is_desktop == latency->is_desktop &&
		713	is_ddr3 == latency->is_ddr3 &&
		714	fsb == latency->fsb_freq && mem == latency->mem_freq)
		715	return latency;
		716	}
		717
		718	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		719
		720	return NULL;
		721	}
		722
		723	static void pineview_disable_cxsr(struct drm_device *dev)
		724	{
		725	struct drm_i915_private *dev_priv = dev->dev_private;
		726
		727	/* deactivate cxsr */
		728	I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
		729	}
		730
		731	/*
		732	* Latency for FIFO fetches is dependent on several factors:
		733	* - memory configuration (speed, channels)
		734	* - chipset
		735	* - current MCH state
		736	* It can be fairly high in some situations, so here we assume a fairly
		737	* pessimal value. It's a tradeoff between extra memory fetches (if we
		738	* set this value too high, the FIFO will fetch frequently to stay full)
		739	* and power consumption (set it too low to save power and we might see
		740	* FIFO underruns and display "flicker").
		741	*
		742	* A value of 5us seems to be a good balance; safe for very low end
		743	* platforms but not overly aggressive on lower latency configs.
		744	*/
		745	static const int latency_ns = 5000;
		746
		747	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		748	{
		749	struct drm_i915_private *dev_priv = dev->dev_private;
		750	uint32_t dsparb = I915_READ(DSPARB);
		751	int size;
		752
		753	size = dsparb & 0x7f;
		754	if (plane)
		755	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		756
		757	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		758	plane ? "B" : "A", size);
		759
		760	return size;
		761	}
		762
		763	static int i85x_get_fifo_size(struct drm_device *dev, int plane)
		764	{
		765	struct drm_i915_private *dev_priv = dev->dev_private;
		766	uint32_t dsparb = I915_READ(DSPARB);
		767	int size;
		768
		769	size = dsparb & 0x1ff;
		770	if (plane)
		771	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		772	size >>= 1; /* Convert to cachelines */
		773
		774	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		775	plane ? "B" : "A", size);
		776
		777	return size;
		778	}
		779
		780	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		781	{
		782	struct drm_i915_private *dev_priv = dev->dev_private;
		783	uint32_t dsparb = I915_READ(DSPARB);
		784	int size;
		785
		786	size = dsparb & 0x7f;
		787	size >>= 2; /* Convert to cachelines */
		788
		789	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		790	plane ? "B" : "A",
		791	size);
		792
		793	return size;
		794	}
		795
		796	static int i830_get_fifo_size(struct drm_device *dev, int plane)
		797	{
		798	struct drm_i915_private *dev_priv = dev->dev_private;
		799	uint32_t dsparb = I915_READ(DSPARB);
		800	int size;
		801
		802	size = dsparb & 0x7f;
		803	size >>= 1; /* Convert to cachelines */
		804
		805	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		806	plane ? "B" : "A", size);
		807
		808	return size;
		809	}
		810
		811	/* Pineview has different values for various configs */
		812	static const struct intel_watermark_params pineview_display_wm = {
		813	PINEVIEW_DISPLAY_FIFO,
		814	PINEVIEW_MAX_WM,
		815	PINEVIEW_DFT_WM,
		816	PINEVIEW_GUARD_WM,
		817	PINEVIEW_FIFO_LINE_SIZE
		818	};
		819	static const struct intel_watermark_params pineview_display_hplloff_wm = {
		820	PINEVIEW_DISPLAY_FIFO,
		821	PINEVIEW_MAX_WM,
		822	PINEVIEW_DFT_HPLLOFF_WM,
		823	PINEVIEW_GUARD_WM,
		824	PINEVIEW_FIFO_LINE_SIZE
		825	};
		826	static const struct intel_watermark_params pineview_cursor_wm = {
		827	PINEVIEW_CURSOR_FIFO,
		828	PINEVIEW_CURSOR_MAX_WM,
		829	PINEVIEW_CURSOR_DFT_WM,
		830	PINEVIEW_CURSOR_GUARD_WM,
		831	PINEVIEW_FIFO_LINE_SIZE,
		832	};
		833	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
		834	PINEVIEW_CURSOR_FIFO,
		835	PINEVIEW_CURSOR_MAX_WM,
		836	PINEVIEW_CURSOR_DFT_WM,
		837	PINEVIEW_CURSOR_GUARD_WM,
		838	PINEVIEW_FIFO_LINE_SIZE
		839	};
		840	static const struct intel_watermark_params g4x_wm_info = {
		841	G4X_FIFO_SIZE,
		842	G4X_MAX_WM,
		843	G4X_MAX_WM,
		844	2,
		845	G4X_FIFO_LINE_SIZE,
		846	};
		847	static const struct intel_watermark_params g4x_cursor_wm_info = {
		848	I965_CURSOR_FIFO,
		849	I965_CURSOR_MAX_WM,
		850	I965_CURSOR_DFT_WM,
		851	2,
		852	G4X_FIFO_LINE_SIZE,
		853	};
		854	static const struct intel_watermark_params valleyview_wm_info = {
		855	VALLEYVIEW_FIFO_SIZE,
		856	VALLEYVIEW_MAX_WM,
		857	VALLEYVIEW_MAX_WM,
		858	2,
		859	G4X_FIFO_LINE_SIZE,
		860	};
		861	static const struct intel_watermark_params valleyview_cursor_wm_info = {
		862	I965_CURSOR_FIFO,
		863	VALLEYVIEW_CURSOR_MAX_WM,
		864	I965_CURSOR_DFT_WM,
		865	2,
		866	G4X_FIFO_LINE_SIZE,
		867	};
		868	static const struct intel_watermark_params i965_cursor_wm_info = {
		869	I965_CURSOR_FIFO,
		870	I965_CURSOR_MAX_WM,
		871	I965_CURSOR_DFT_WM,
		872	2,
		873	I915_FIFO_LINE_SIZE,
		874	};
		875	static const struct intel_watermark_params i945_wm_info = {
		876	I945_FIFO_SIZE,
		877	I915_MAX_WM,
		878	1,
		879	2,
		880	I915_FIFO_LINE_SIZE
		881	};
		882	static const struct intel_watermark_params i915_wm_info = {
		883	I915_FIFO_SIZE,
		884	I915_MAX_WM,
		885	1,
		886	2,
		887	I915_FIFO_LINE_SIZE
		888	};
		889	static const struct intel_watermark_params i855_wm_info = {
		890	I855GM_FIFO_SIZE,
		891	I915_MAX_WM,
		892	1,
		893	2,
		894	I830_FIFO_LINE_SIZE
		895	};
		896	static const struct intel_watermark_params i830_wm_info = {
		897	I830_FIFO_SIZE,
		898	I915_MAX_WM,
		899	1,
		900	2,
		901	I830_FIFO_LINE_SIZE
		902	};
		903
		904	static const struct intel_watermark_params ironlake_display_wm_info = {
		905	ILK_DISPLAY_FIFO,
		906	ILK_DISPLAY_MAXWM,
		907	ILK_DISPLAY_DFTWM,
		908	2,
		909	ILK_FIFO_LINE_SIZE
		910	};
		911	static const struct intel_watermark_params ironlake_cursor_wm_info = {
		912	ILK_CURSOR_FIFO,
		913	ILK_CURSOR_MAXWM,
		914	ILK_CURSOR_DFTWM,
		915	2,
		916	ILK_FIFO_LINE_SIZE
		917	};
		918	static const struct intel_watermark_params ironlake_display_srwm_info = {
		919	ILK_DISPLAY_SR_FIFO,
		920	ILK_DISPLAY_MAX_SRWM,
		921	ILK_DISPLAY_DFT_SRWM,
		922	2,
		923	ILK_FIFO_LINE_SIZE
		924	};
		925	static const struct intel_watermark_params ironlake_cursor_srwm_info = {
		926	ILK_CURSOR_SR_FIFO,
		927	ILK_CURSOR_MAX_SRWM,
		928	ILK_CURSOR_DFT_SRWM,
		929	2,
		930	ILK_FIFO_LINE_SIZE
		931	};
		932
		933	static const struct intel_watermark_params sandybridge_display_wm_info = {
		934	SNB_DISPLAY_FIFO,
		935	SNB_DISPLAY_MAXWM,
		936	SNB_DISPLAY_DFTWM,
		937	2,
		938	SNB_FIFO_LINE_SIZE
		939	};
		940	static const struct intel_watermark_params sandybridge_cursor_wm_info = {
		941	SNB_CURSOR_FIFO,
		942	SNB_CURSOR_MAXWM,
		943	SNB_CURSOR_DFTWM,
		944	2,
		945	SNB_FIFO_LINE_SIZE
		946	};
		947	static const struct intel_watermark_params sandybridge_display_srwm_info = {
		948	SNB_DISPLAY_SR_FIFO,
		949	SNB_DISPLAY_MAX_SRWM,
		950	SNB_DISPLAY_DFT_SRWM,
		951	2,
		952	SNB_FIFO_LINE_SIZE
		953	};
		954	static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
		955	SNB_CURSOR_SR_FIFO,
		956	SNB_CURSOR_MAX_SRWM,
		957	SNB_CURSOR_DFT_SRWM,
		958	2,
		959	SNB_FIFO_LINE_SIZE
		960	};
		961
		962
		963	/**
		964	* intel_calculate_wm - calculate watermark level
		965	* @clock_in_khz: pixel clock
		966	* @wm: chip FIFO params
		967	* @pixel_size: display pixel size
		968	* @latency_ns: memory latency for the platform
		969	*
		970	* Calculate the watermark level (the level at which the display plane will
		971	* start fetching from memory again). Each chip has a different display
		972	* FIFO size and allocation, so the caller needs to figure that out and pass
		973	* in the correct intel_watermark_params structure.
		974	*
		975	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		976	* on the pixel size. When it reaches the watermark level, it'll start
		977	* fetching FIFO line sized based chunks from memory until the FIFO fills
		978	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		979	* will occur, and a display engine hang could result.
		980	*/
		981	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		982	const struct intel_watermark_params *wm,
		983	int fifo_size,
		984	int pixel_size,
		985	unsigned long latency_ns)
		986	{
		987	long entries_required, wm_size;
		988
		989	/*
		990	* Note: we need to make sure we don't overflow for various clock &
		991	* latency values.
		992	* clocks go from a few thousand to several hundred thousand.
		993	* latency is usually a few thousand
		994	*/
		995	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		996	1000;
		997	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		998
		999	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		1000
		1001	wm_size = fifo_size - (entries_required + wm->guard_size);
		1002
		1003	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		1004
		1005	/* Don't promote wm_size to unsigned... */
		1006	if (wm_size > (long)wm->max_wm)
		1007	wm_size = wm->max_wm;
		1008	if (wm_size <= 0)
		1009	wm_size = wm->default_wm;
		1010	return wm_size;
		1011	}
		1012
		1013	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		1014	{
		1015	struct drm_crtc crtc, enabled = NULL;
		1016
		1017	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
		1018	if (crtc->enabled && crtc->fb) {
		1019	if (enabled)
		1020	return NULL;
		1021	enabled = crtc;
		1022	}
		1023	}
		1024
		1025	return enabled;
		1026	}
		1027
		1028	static void pineview_update_wm(struct drm_device *dev)
		1029	{
		1030	struct drm_i915_private *dev_priv = dev->dev_private;
		1031	struct drm_crtc *crtc;
		1032	const struct cxsr_latency *latency;
		1033	u32 reg;
		1034	unsigned long wm;
		1035
		1036	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		1037	dev_priv->fsb_freq, dev_priv->mem_freq);
		1038	if (!latency) {
		1039	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		1040	pineview_disable_cxsr(dev);
		1041	return;
		1042	}
		1043
		1044	crtc = single_enabled_crtc(dev);
		1045	if (crtc) {
		1046	int clock = crtc->mode.clock;
		1047	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1048
		1049	/* Display SR */
		1050	wm = intel_calculate_wm(clock, &pineview_display_wm,
		1051	pineview_display_wm.fifo_size,
		1052	pixel_size, latency->display_sr);
		1053	reg = I915_READ(DSPFW1);
		1054	reg &= ~DSPFW_SR_MASK;
		1055	reg \|= wm << DSPFW_SR_SHIFT;
		1056	I915_WRITE(DSPFW1, reg);
		1057	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		1058
		1059	/* cursor SR */
		1060	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		1061	pineview_display_wm.fifo_size,
		1062	pixel_size, latency->cursor_sr);
		1063	reg = I915_READ(DSPFW3);
		1064	reg &= ~DSPFW_CURSOR_SR_MASK;
		1065	reg \|= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
		1066	I915_WRITE(DSPFW3, reg);
		1067
		1068	/* Display HPLL off SR */
		1069	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		1070	pineview_display_hplloff_wm.fifo_size,
		1071	pixel_size, latency->display_hpll_disable);
		1072	reg = I915_READ(DSPFW3);
		1073	reg &= ~DSPFW_HPLL_SR_MASK;
		1074	reg \|= wm & DSPFW_HPLL_SR_MASK;
		1075	I915_WRITE(DSPFW3, reg);
		1076
		1077	/* cursor HPLL off SR */
		1078	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		1079	pineview_display_hplloff_wm.fifo_size,
		1080	pixel_size, latency->cursor_hpll_disable);
		1081	reg = I915_READ(DSPFW3);
		1082	reg &= ~DSPFW_HPLL_CURSOR_MASK;
		1083	reg \|= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
		1084	I915_WRITE(DSPFW3, reg);
		1085	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		1086
		1087	/* activate cxsr */
		1088	I915_WRITE(DSPFW3,
		1089	I915_READ(DSPFW3) \| PINEVIEW_SELF_REFRESH_EN);
		1090	DRM_DEBUG_KMS("Self-refresh is enabled\n");
		1091	} else {
		1092	pineview_disable_cxsr(dev);
		1093	DRM_DEBUG_KMS("Self-refresh is disabled\n");
		1094	}
		1095	}
		1096
		1097	static bool g4x_compute_wm0(struct drm_device *dev,
		1098	int plane,
		1099	const struct intel_watermark_params *display,
		1100	int display_latency_ns,
		1101	const struct intel_watermark_params *cursor,
		1102	int cursor_latency_ns,
		1103	int *plane_wm,
		1104	int *cursor_wm)
		1105	{
		1106	struct drm_crtc *crtc;
		1107
		1108	int htotal, hdisplay, clock, pixel_size;
		1109	int line_time_us, line_count;
		1110	int entries, tlb_miss;
		1111
		1112	// ENTER();
		1113
		1114	// dbgprintf("plane %d display %x cursor %x \n", plane, display, cursor);
		1115	// dbgprintf("plane_wm %x cursor_wm %x \n", plane_wm, cursor_wm);
		1116
		1117	crtc = intel_get_crtc_for_plane(dev, plane);
		1118	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		1119
		1120	// dbgprintf("CRTC %d\n, fb %x, enabled %d\n",
		1121	// crtc->base.id, crtc->fb, crtc->enabled );
		1122
		1123
		1124
		1125	if (crtc->fb == NULL \|\| !crtc->enabled \|\| !intel_crtc->active) {
		1126	*cursor_wm = cursor->guard_size;
		1127	*plane_wm = display->guard_size;
		1128	return false;
		1129	}
		1130
		1131	htotal = crtc->mode.htotal;
		1132	hdisplay = crtc->mode.hdisplay;
		1133	clock = crtc->mode.clock;
		1134	pixel_size = crtc->fb->bits_per_pixel / 8;
		1135
		1136	// dbgprintf("mark 1\n");
		1137
		1138	/* Use the small buffer method to calculate plane watermark */
		1139	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1140	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		1141	if (tlb_miss > 0)
		1142	entries += tlb_miss;
		1143	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1144	*plane_wm = entries + display->guard_size;
		1145	if (*plane_wm > (int)display->max_wm)
		1146	*plane_wm = display->max_wm;
		1147
		1148	// dbgprintf("clock %d line_time_us %d\n",clock, line_time_us );
		1149
		1150	/* Use the large buffer method to calculate cursor watermark */
		1151	line_time_us = ((htotal * 1000) / clock);
		1152	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
		1153
		1154	entries = line_count * 64 * pixel_size;
		1155
		1156	// dbgprintf("mark 3\n");
		1157
		1158	// dbgprintf("fifo size %d line size %d\n",
		1159	// cursor->fifo_size, cursor->cacheline_size);
		1160
		1161	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		1162
		1163	if (tlb_miss > 0)
		1164	entries += tlb_miss;
		1165
		1166	// dbgprintf("mark 4\n");
		1167
		1168	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1169
		1170	// dbgprintf("entries %d \n",entries);
		1171
		1172	*cursor_wm = entries + cursor->guard_size;
		1173
		1174	if (*cursor_wm > (int)cursor->max_wm)
		1175	*cursor_wm = (int)cursor->max_wm;
		1176
		1177	// LEAVE();
		1178
		1179	return true;
		1180	}
		1181
		1182	/*
		1183	* Check the wm result.
		1184	*
		1185	* If any calculated watermark values is larger than the maximum value that
		1186	* can be programmed into the associated watermark register, that watermark
		1187	* must be disabled.
		1188	*/
		1189	static bool g4x_check_srwm(struct drm_device *dev,
		1190	int display_wm, int cursor_wm,
		1191	const struct intel_watermark_params *display,
		1192	const struct intel_watermark_params *cursor)
		1193	{
		1194	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		1195	display_wm, cursor_wm);
		1196
		1197	if (display_wm > display->max_wm) {
		1198	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		1199	display_wm, display->max_wm);
		1200	return false;
		1201	}
		1202
		1203	if (cursor_wm > cursor->max_wm) {
		1204	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		1205	cursor_wm, cursor->max_wm);
		1206	return false;
		1207	}
		1208
		1209	if (!(display_wm \|\| cursor_wm)) {
		1210	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		1211	return false;
		1212	}
		1213
		1214	return true;
		1215	}
		1216
		1217	static bool g4x_compute_srwm(struct drm_device *dev,
		1218	int plane,
		1219	int latency_ns,
		1220	const struct intel_watermark_params *display,
		1221	const struct intel_watermark_params *cursor,
		1222	int display_wm, int cursor_wm)
		1223	{
		1224	struct drm_crtc *crtc;
		1225	int hdisplay, htotal, pixel_size, clock;
		1226	unsigned long line_time_us;
		1227	int line_count, line_size;
		1228	int small, large;
		1229	int entries;
		1230
		1231	if (!latency_ns) {
		1232	display_wm = cursor_wm = 0;
		1233	return false;
		1234	}
		1235
		1236	crtc = intel_get_crtc_for_plane(dev, plane);
		1237	hdisplay = crtc->mode.hdisplay;
		1238	htotal = crtc->mode.htotal;
		1239	clock = crtc->mode.clock;
		1240	pixel_size = crtc->fb->bits_per_pixel / 8;
		1241
		1242	line_time_us = (htotal * 1000) / clock;
		1243	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1244	line_size = hdisplay * pixel_size;
		1245
		1246	/* Use the minimum of the small and large buffer method for primary */
		1247	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1248	large = line_count * line_size;
		1249
		1250	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1251	*display_wm = entries + display->guard_size;
		1252
		1253	/* calculate the self-refresh watermark for display cursor */
		1254	entries = line_count * pixel_size * 64;
		1255	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1256	*cursor_wm = entries + cursor->guard_size;
		1257
		1258	return g4x_check_srwm(dev,
		1259	display_wm, cursor_wm,
		1260	display, cursor);
		1261	}
		1262
		1263	static bool vlv_compute_drain_latency(struct drm_device *dev,
		1264	int plane,
		1265	int *plane_prec_mult,
		1266	int *plane_dl,
		1267	int *cursor_prec_mult,
		1268	int *cursor_dl)
		1269	{
		1270	struct drm_crtc *crtc;
		1271	int clock, pixel_size;
		1272	int entries;
		1273
		1274	crtc = intel_get_crtc_for_plane(dev, plane);
		1275	if (crtc->fb == NULL \|\| !crtc->enabled)
		1276	return false;
		1277
		1278	clock = crtc->mode.clock; /* VESA DOT Clock */
		1279	pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */
		1280
		1281	entries = (clock / 1000) * pixel_size;
		1282	*plane_prec_mult = (entries > 256) ?
		1283	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1284	plane_dl = (64 (plane_prec_mult) 4) / ((clock / 1000) *
		1285	pixel_size);
		1286
		1287	entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
		1288	*cursor_prec_mult = (entries > 256) ?
		1289	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1290	cursor_dl = (64 (cursor_prec_mult) 4) / ((clock / 1000) * 4);
		1291
		1292	return true;
		1293	}
		1294
		1295	/*
		1296	* Update drain latency registers of memory arbiter
		1297	*
		1298	* Valleyview SoC has a new memory arbiter and needs drain latency registers
		1299	* to be programmed. Each plane has a drain latency multiplier and a drain
		1300	* latency value.
		1301	*/
		1302
		1303	static void vlv_update_drain_latency(struct drm_device *dev)
		1304	{
		1305	struct drm_i915_private *dev_priv = dev->dev_private;
		1306	int planea_prec, planea_dl, planeb_prec, planeb_dl;
		1307	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
		1308	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
		1309	either 16 or 32 */
		1310
		1311	/* For plane A, Cursor A */
		1312	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
		1313	&cursor_prec_mult, &cursora_dl)) {
		1314	cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1315	DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
		1316	planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1317	DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
		1318
		1319	I915_WRITE(VLV_DDL1, cursora_prec \|
		1320	(cursora_dl << DDL_CURSORA_SHIFT) \|
		1321	planea_prec \| planea_dl);
		1322	}
		1323
		1324	/* For plane B, Cursor B */
		1325	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
		1326	&cursor_prec_mult, &cursorb_dl)) {
		1327	cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1328	DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
		1329	planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1330	DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
		1331
		1332	I915_WRITE(VLV_DDL2, cursorb_prec \|
		1333	(cursorb_dl << DDL_CURSORB_SHIFT) \|
		1334	planeb_prec \| planeb_dl);
		1335	}
		1336	}
		1337
		1338	#define single_plane_enabled(mask) is_power_of_2(mask)
		1339
		1340	static void valleyview_update_wm(struct drm_device *dev)
		1341	{
		1342	static const int sr_latency_ns = 12000;
		1343	struct drm_i915_private *dev_priv = dev->dev_private;
		1344	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1345	int plane_sr, cursor_sr;
		1346	unsigned int enabled = 0;
		1347
		1348	vlv_update_drain_latency(dev);
		1349
		1350	if (g4x_compute_wm0(dev, 0,
		1351	&valleyview_wm_info, latency_ns,
		1352	&valleyview_cursor_wm_info, latency_ns,
		1353	&planea_wm, &cursora_wm))
		1354	enabled \|= 1;
		1355
		1356	if (g4x_compute_wm0(dev, 1,
		1357	&valleyview_wm_info, latency_ns,
		1358	&valleyview_cursor_wm_info, latency_ns,
		1359	&planeb_wm, &cursorb_wm))
		1360	enabled \|= 2;
		1361
		1362	plane_sr = cursor_sr = 0;
		1363	if (single_plane_enabled(enabled) &&
		1364	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1365	sr_latency_ns,
		1366	&valleyview_wm_info,
		1367	&valleyview_cursor_wm_info,
		1368	&plane_sr, &cursor_sr))
		1369	I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
		1370	else
		1371	I915_WRITE(FW_BLC_SELF_VLV,
		1372	I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
		1373
		1374	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1375	planea_wm, cursora_wm,
		1376	planeb_wm, cursorb_wm,
		1377	plane_sr, cursor_sr);
		1378
		1379	I915_WRITE(DSPFW1,
		1380	(plane_sr << DSPFW_SR_SHIFT) \|
		1381	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1382	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1383	planea_wm);
		1384	I915_WRITE(DSPFW2,
		1385	(I915_READ(DSPFW2) & DSPFW_CURSORA_MASK) \|
		1386	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1387	I915_WRITE(DSPFW3,
		1388	(I915_READ(DSPFW3) \| (cursor_sr << DSPFW_CURSOR_SR_SHIFT)));
		1389	}
		1390
		1391	static void g4x_update_wm(struct drm_device *dev)
		1392	{
		1393	static const int sr_latency_ns = 12000;
		1394	struct drm_i915_private *dev_priv = dev->dev_private;
		1395	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1396	int plane_sr, cursor_sr;
		1397	unsigned int enabled = 0;
		1398
		1399	if (g4x_compute_wm0(dev, 0,
		1400	&g4x_wm_info, latency_ns,
		1401	&g4x_cursor_wm_info, latency_ns,
		1402	&planea_wm, &cursora_wm))
		1403	enabled \|= 1;
		1404
		1405	if (g4x_compute_wm0(dev, 1,
		1406	&g4x_wm_info, latency_ns,
		1407	&g4x_cursor_wm_info, latency_ns,
		1408	&planeb_wm, &cursorb_wm))
		1409	enabled \|= 2;
		1410
		1411	plane_sr = cursor_sr = 0;
		1412	if (single_plane_enabled(enabled) &&
		1413	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1414	sr_latency_ns,
		1415	&g4x_wm_info,
		1416	&g4x_cursor_wm_info,
		1417	&plane_sr, &cursor_sr))
		1418	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
		1419	else
		1420	I915_WRITE(FW_BLC_SELF,
		1421	I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
		1422
		1423	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1424	planea_wm, cursora_wm,
		1425	planeb_wm, cursorb_wm,
		1426	plane_sr, cursor_sr);
		1427
		1428	I915_WRITE(DSPFW1,
		1429	(plane_sr << DSPFW_SR_SHIFT) \|
		1430	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1431	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1432	planea_wm);
		1433	I915_WRITE(DSPFW2,
		1434	(I915_READ(DSPFW2) & DSPFW_CURSORA_MASK) \|
		1435	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1436	/* HPLL off in SR has some issues on G4x... disable it */
		1437	I915_WRITE(DSPFW3,
		1438	(I915_READ(DSPFW3) & ~DSPFW_HPLL_SR_EN) \|
		1439	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1440	}
		1441
		1442	static void i965_update_wm(struct drm_device *dev)
		1443	{
		1444	struct drm_i915_private *dev_priv = dev->dev_private;
		1445	struct drm_crtc *crtc;
		1446	int srwm = 1;
		1447	int cursor_sr = 16;
		1448
		1449	/* Calc sr entries for one plane configs */
		1450	crtc = single_enabled_crtc(dev);
		1451	if (crtc) {
		1452	/* self-refresh has much higher latency */
		1453	static const int sr_latency_ns = 12000;
		1454	int clock = crtc->mode.clock;
		1455	int htotal = crtc->mode.htotal;
		1456	int hdisplay = crtc->mode.hdisplay;
		1457	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1458	unsigned long line_time_us;
		1459	int entries;
		1460
		1461	line_time_us = ((htotal * 1000) / clock);
		1462
		1463	/* Use ns/us then divide to preserve precision */
		1464	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1465	pixel_size * hdisplay;
		1466	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1467	srwm = I965_FIFO_SIZE - entries;
		1468	if (srwm < 0)
		1469	srwm = 1;
		1470	srwm &= 0x1ff;
		1471	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1472	entries, srwm);
		1473
		1474	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1475	pixel_size * 64;
		1476	entries = DIV_ROUND_UP(entries,
		1477	i965_cursor_wm_info.cacheline_size);
		1478	cursor_sr = i965_cursor_wm_info.fifo_size -
		1479	(entries + i965_cursor_wm_info.guard_size);
		1480
		1481	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1482	cursor_sr = i965_cursor_wm_info.max_wm;
		1483
		1484	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1485	"cursor %d\n", srwm, cursor_sr);
		1486
		1487	if (IS_CRESTLINE(dev))
		1488	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
		1489	} else {
		1490	/* Turn off self refresh if both pipes are enabled */
		1491	if (IS_CRESTLINE(dev))
		1492	I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
		1493	& ~FW_BLC_SELF_EN);
		1494	}
		1495
		1496	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1497	srwm);
		1498
		1499	/* 965 has limitations... */
		1500	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) \|
		1501	(8 << 16) \| (8 << 8) \| (8 << 0));
		1502	I915_WRITE(DSPFW2, (8 << 8) \| (8 << 0));
		1503	/* update cursor SR watermark */
		1504	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1505	}
		1506
		1507	static void i9xx_update_wm(struct drm_device *dev)
		1508	{
		1509	struct drm_i915_private *dev_priv = dev->dev_private;
		1510	const struct intel_watermark_params *wm_info;
		1511	uint32_t fwater_lo;
		1512	uint32_t fwater_hi;
		1513	int cwm, srwm = 1;
		1514	int fifo_size;
		1515	int planea_wm, planeb_wm;
		1516	struct drm_crtc crtc, enabled = NULL;
		1517
		1518	if (IS_I945GM(dev))
		1519	wm_info = &i945_wm_info;
		1520	else if (!IS_GEN2(dev))
		1521	wm_info = &i915_wm_info;
		1522	else
		1523	wm_info = &i855_wm_info;
		1524
		1525	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1526	crtc = intel_get_crtc_for_plane(dev, 0);
		1527	if (crtc->enabled && crtc->fb) {
		1528	planea_wm = intel_calculate_wm(crtc->mode.clock,
		1529	wm_info, fifo_size,
		1530	crtc->fb->bits_per_pixel / 8,
		1531	latency_ns);
		1532	enabled = crtc;
		1533	} else
		1534	planea_wm = fifo_size - wm_info->guard_size;
		1535
		1536	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1537	crtc = intel_get_crtc_for_plane(dev, 1);
		1538	if (crtc->enabled && crtc->fb) {
		1539	planeb_wm = intel_calculate_wm(crtc->mode.clock,
		1540	wm_info, fifo_size,
		1541	crtc->fb->bits_per_pixel / 8,
		1542	latency_ns);
		1543	if (enabled == NULL)
		1544	enabled = crtc;
		1545	else
		1546	enabled = NULL;
		1547	} else
		1548	planeb_wm = fifo_size - wm_info->guard_size;
		1549
		1550	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1551
		1552	/*
		1553	* Overlay gets an aggressive default since video jitter is bad.
		1554	*/
		1555	cwm = 2;
		1556
		1557	/* Play safe and disable self-refresh before adjusting watermarks. */
		1558	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1559	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK \| 0);
		1560	else if (IS_I915GM(dev))
		1561	I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
		1562
		1563	/* Calc sr entries for one plane configs */
		1564	if (HAS_FW_BLC(dev) && enabled) {
		1565	/* self-refresh has much higher latency */
		1566	static const int sr_latency_ns = 6000;
		1567	int clock = enabled->mode.clock;
		1568	int htotal = enabled->mode.htotal;
		1569	int hdisplay = enabled->mode.hdisplay;
		1570	int pixel_size = enabled->fb->bits_per_pixel / 8;
		1571	unsigned long line_time_us;
		1572	int entries;
		1573
		1574	line_time_us = (htotal * 1000) / clock;
		1575
		1576	/* Use ns/us then divide to preserve precision */
		1577	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1578	pixel_size * hdisplay;
		1579	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1580	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1581	srwm = wm_info->fifo_size - entries;
		1582	if (srwm < 0)
		1583	srwm = 1;
		1584
		1585	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1586	I915_WRITE(FW_BLC_SELF,
		1587	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1588	else if (IS_I915GM(dev))
		1589	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1590	}
		1591
		1592	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1593	planea_wm, planeb_wm, cwm, srwm);
		1594
		1595	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1596	fwater_hi = (cwm & 0x1f);
		1597
		1598	/* Set request length to 8 cachelines per fetch */
		1599	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1600	fwater_hi = fwater_hi \| (1 << 8);
		1601
		1602	I915_WRITE(FW_BLC, fwater_lo);
		1603	I915_WRITE(FW_BLC2, fwater_hi);
		1604
		1605	if (HAS_FW_BLC(dev)) {
		1606	if (enabled) {
		1607	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1608	I915_WRITE(FW_BLC_SELF,
		1609	FW_BLC_SELF_EN_MASK \| FW_BLC_SELF_EN);
		1610	else if (IS_I915GM(dev))
		1611	I915_WRITE(INSTPM, I915_READ(INSTPM) \| INSTPM_SELF_EN);
		1612	DRM_DEBUG_KMS("memory self refresh enabled\n");
		1613	} else
		1614	DRM_DEBUG_KMS("memory self refresh disabled\n");
		1615	}
		1616	}
		1617
		1618	static void i830_update_wm(struct drm_device *dev)
		1619	{
		1620	struct drm_i915_private *dev_priv = dev->dev_private;
		1621	struct drm_crtc *crtc;
		1622	uint32_t fwater_lo;
		1623	int planea_wm;
		1624
		1625	crtc = single_enabled_crtc(dev);
		1626	if (crtc == NULL)
		1627	return;
		1628
		1629	planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
		1630	dev_priv->display.get_fifo_size(dev, 0),
		1631	crtc->fb->bits_per_pixel / 8,
		1632	latency_ns);
		1633	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1634	fwater_lo \|= (3<<8) \| planea_wm;
		1635
		1636	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1637
		1638	I915_WRITE(FW_BLC, fwater_lo);
		1639	}
		1640
		1641	#define ILK_LP0_PLANE_LATENCY 700
		1642	#define ILK_LP0_CURSOR_LATENCY 1300
		1643
		1644	/*
		1645	* Check the wm result.
		1646	*
		1647	* If any calculated watermark values is larger than the maximum value that
		1648	* can be programmed into the associated watermark register, that watermark
		1649	* must be disabled.
		1650	*/
		1651	static bool ironlake_check_srwm(struct drm_device *dev, int level,
		1652	int fbc_wm, int display_wm, int cursor_wm,
		1653	const struct intel_watermark_params *display,
		1654	const struct intel_watermark_params *cursor)
		1655	{
		1656	struct drm_i915_private *dev_priv = dev->dev_private;
		1657
		1658	DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
		1659	" cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
		1660
		1661	if (fbc_wm > SNB_FBC_MAX_SRWM) {
		1662	DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
		1663	fbc_wm, SNB_FBC_MAX_SRWM, level);
		1664
		1665	/* fbc has it's own way to disable FBC WM */
		1666	I915_WRITE(DISP_ARB_CTL,
		1667	I915_READ(DISP_ARB_CTL) \| DISP_FBC_WM_DIS);
		1668	return false;
		1669	}
		1670
		1671	if (display_wm > display->max_wm) {
		1672	DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
		1673	display_wm, SNB_DISPLAY_MAX_SRWM, level);
		1674	return false;
		1675	}
		1676
		1677	if (cursor_wm > cursor->max_wm) {
		1678	DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
		1679	cursor_wm, SNB_CURSOR_MAX_SRWM, level);
		1680	return false;
		1681	}
		1682
		1683	if (!(fbc_wm \|\| display_wm \|\| cursor_wm)) {
		1684	DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
		1685	return false;
		1686	}
		1687
		1688	return true;
		1689	}
		1690
		1691	/*
		1692	* Compute watermark values of WM[1-3],
		1693	*/
		1694	static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
		1695	int latency_ns,
		1696	const struct intel_watermark_params *display,
		1697	const struct intel_watermark_params *cursor,
		1698	int fbc_wm, int display_wm, int *cursor_wm)
		1699	{
		1700	struct drm_crtc *crtc;
		1701	unsigned long line_time_us;
		1702	int hdisplay, htotal, pixel_size, clock;
		1703	int line_count, line_size;
		1704	int small, large;
		1705	int entries;
		1706
		1707	if (!latency_ns) {
		1708	fbc_wm = display_wm = *cursor_wm = 0;
		1709	return false;
		1710	}
		1711
		1712	crtc = intel_get_crtc_for_plane(dev, plane);
		1713	hdisplay = crtc->mode.hdisplay;
		1714	htotal = crtc->mode.htotal;
		1715	clock = crtc->mode.clock;
		1716	pixel_size = crtc->fb->bits_per_pixel / 8;
		1717
		1718	line_time_us = (htotal * 1000) / clock;
		1719	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1720	line_size = hdisplay * pixel_size;
		1721
		1722	/* Use the minimum of the small and large buffer method for primary */
		1723	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1724	large = line_count * line_size;
		1725
		1726	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1727	*display_wm = entries + display->guard_size;
		1728
		1729	/*
		1730	* Spec says:
		1731	* FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
		1732	*/
		1733	fbc_wm = DIV_ROUND_UP(display_wm * 64, line_size) + 2;
		1734
		1735	/* calculate the self-refresh watermark for display cursor */
		1736	entries = line_count * pixel_size * 64;
		1737	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1738	*cursor_wm = entries + cursor->guard_size;
		1739
		1740	return ironlake_check_srwm(dev, level,
		1741	fbc_wm, display_wm, *cursor_wm,
		1742	display, cursor);
		1743	}
		1744
		1745	static void ironlake_update_wm(struct drm_device *dev)
		1746	{
		1747	struct drm_i915_private *dev_priv = dev->dev_private;
		1748	int fbc_wm, plane_wm, cursor_wm;
		1749	unsigned int enabled;
		1750
		1751	enabled = 0;
		1752	if (g4x_compute_wm0(dev, 0,
		1753	&ironlake_display_wm_info,
		1754	ILK_LP0_PLANE_LATENCY,
		1755	&ironlake_cursor_wm_info,
		1756	ILK_LP0_CURSOR_LATENCY,
		1757	&plane_wm, &cursor_wm)) {
		1758	I915_WRITE(WM0_PIPEA_ILK,
		1759	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1760	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1761	" plane %d, " "cursor: %d\n",
		1762	plane_wm, cursor_wm);
		1763	enabled \|= 1;
		1764	}
		1765
		1766	if (g4x_compute_wm0(dev, 1,
		1767	&ironlake_display_wm_info,
		1768	ILK_LP0_PLANE_LATENCY,
		1769	&ironlake_cursor_wm_info,
		1770	ILK_LP0_CURSOR_LATENCY,
		1771	&plane_wm, &cursor_wm)) {
		1772	I915_WRITE(WM0_PIPEB_ILK,
		1773	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1774	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1775	" plane %d, cursor: %d\n",
		1776	plane_wm, cursor_wm);
		1777	enabled \|= 2;
		1778	}
		1779
		1780	/*
		1781	* Calculate and update the self-refresh watermark only when one
		1782	* display plane is used.
		1783	*/
		1784	I915_WRITE(WM3_LP_ILK, 0);
		1785	I915_WRITE(WM2_LP_ILK, 0);
		1786	I915_WRITE(WM1_LP_ILK, 0);
		1787
		1788	if (!single_plane_enabled(enabled))
		1789	return;
		1790	enabled = ffs(enabled) - 1;
		1791
		1792	/* WM1 */
		1793	if (!ironlake_compute_srwm(dev, 1, enabled,
		1794	ILK_READ_WM1_LATENCY() * 500,
		1795	&ironlake_display_srwm_info,
		1796	&ironlake_cursor_srwm_info,
		1797	&fbc_wm, &plane_wm, &cursor_wm))
		1798	return;
		1799
		1800	I915_WRITE(WM1_LP_ILK,
		1801	WM1_LP_SR_EN \|
		1802	(ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1803	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1804	(plane_wm << WM1_LP_SR_SHIFT) \|
		1805	cursor_wm);
		1806
		1807	/* WM2 */
		1808	if (!ironlake_compute_srwm(dev, 2, enabled,
		1809	ILK_READ_WM2_LATENCY() * 500,
		1810	&ironlake_display_srwm_info,
		1811	&ironlake_cursor_srwm_info,
		1812	&fbc_wm, &plane_wm, &cursor_wm))
		1813	return;
		1814
		1815	I915_WRITE(WM2_LP_ILK,
		1816	WM2_LP_EN \|
		1817	(ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1818	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1819	(plane_wm << WM1_LP_SR_SHIFT) \|
		1820	cursor_wm);
		1821
		1822	/*
		1823	* WM3 is unsupported on ILK, probably because we don't have latency
		1824	* data for that power state
		1825	*/
		1826	}
		1827
		1828	static void sandybridge_update_wm(struct drm_device *dev)
		1829	{
		1830	struct drm_i915_private *dev_priv = dev->dev_private;
		1831	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		1832	u32 val;
		1833	int fbc_wm, plane_wm, cursor_wm;
		1834	unsigned int enabled;
		1835
		1836	enabled = 0;
		1837	if (g4x_compute_wm0(dev, 0,
		1838	&sandybridge_display_wm_info, latency,
		1839	&sandybridge_cursor_wm_info, latency,
		1840	&plane_wm, &cursor_wm)) {
		1841	val = I915_READ(WM0_PIPEA_ILK);
		1842	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1843	I915_WRITE(WM0_PIPEA_ILK, val \|
		1844	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1845	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1846	" plane %d, " "cursor: %d\n",
		1847	plane_wm, cursor_wm);
		1848	enabled \|= 1;
		1849	}
		1850
		1851	if (g4x_compute_wm0(dev, 1,
		1852	&sandybridge_display_wm_info, latency,
		1853	&sandybridge_cursor_wm_info, latency,
		1854	&plane_wm, &cursor_wm)) {
		1855	val = I915_READ(WM0_PIPEB_ILK);
		1856	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1857	I915_WRITE(WM0_PIPEB_ILK, val \|
		1858	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1859	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1860	" plane %d, cursor: %d\n",
		1861	plane_wm, cursor_wm);
		1862	enabled \|= 2;
		1863	}
		1864
		1865	if ((dev_priv->num_pipe == 3) &&
		1866	g4x_compute_wm0(dev, 2,
		1867	&sandybridge_display_wm_info, latency,
		1868	&sandybridge_cursor_wm_info, latency,
		1869	&plane_wm, &cursor_wm)) {
		1870	val = I915_READ(WM0_PIPEC_IVB);
		1871	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1872	I915_WRITE(WM0_PIPEC_IVB, val \|
		1873	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1874	DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
		1875	" plane %d, cursor: %d\n",
		1876	plane_wm, cursor_wm);
		1877	enabled \|= 3;
		1878	}
		1879
		1880	/*
		1881	* Calculate and update the self-refresh watermark only when one
		1882	* display plane is used.
		1883	*
		1884	* SNB support 3 levels of watermark.
		1885	*
		1886	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1887	* and disabled in the descending order
		1888	*
		1889	*/
		1890	I915_WRITE(WM3_LP_ILK, 0);
		1891	I915_WRITE(WM2_LP_ILK, 0);
		1892	I915_WRITE(WM1_LP_ILK, 0);
		1893
		1894	if (!single_plane_enabled(enabled) \|\|
		1895	dev_priv->sprite_scaling_enabled)
		1896	return;
		1897	enabled = ffs(enabled) - 1;
		1898
		1899	/* WM1 */
		1900	if (!ironlake_compute_srwm(dev, 1, enabled,
		1901	SNB_READ_WM1_LATENCY() * 500,
		1902	&sandybridge_display_srwm_info,
		1903	&sandybridge_cursor_srwm_info,
		1904	&fbc_wm, &plane_wm, &cursor_wm))
		1905	return;
		1906
		1907	I915_WRITE(WM1_LP_ILK,
		1908	WM1_LP_SR_EN \|
		1909	(SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1910	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1911	(plane_wm << WM1_LP_SR_SHIFT) \|
		1912	cursor_wm);
		1913
		1914	/* WM2 */
		1915	if (!ironlake_compute_srwm(dev, 2, enabled,
		1916	SNB_READ_WM2_LATENCY() * 500,
		1917	&sandybridge_display_srwm_info,
		1918	&sandybridge_cursor_srwm_info,
		1919	&fbc_wm, &plane_wm, &cursor_wm))
		1920	return;
		1921
		1922	I915_WRITE(WM2_LP_ILK,
		1923	WM2_LP_EN \|
		1924	(SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1925	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1926	(plane_wm << WM1_LP_SR_SHIFT) \|
		1927	cursor_wm);
		1928
		1929	/* WM3 */
		1930	if (!ironlake_compute_srwm(dev, 3, enabled,
		1931	SNB_READ_WM3_LATENCY() * 500,
		1932	&sandybridge_display_srwm_info,
		1933	&sandybridge_cursor_srwm_info,
		1934	&fbc_wm, &plane_wm, &cursor_wm))
		1935	return;
		1936
		1937	I915_WRITE(WM3_LP_ILK,
		1938	WM3_LP_EN \|
		1939	(SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1940	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1941	(plane_wm << WM1_LP_SR_SHIFT) \|
		1942	cursor_wm);
		1943
		1944	}
		1945
		1946	static void
		1947	haswell_update_linetime_wm(struct drm_device *dev, int pipe,
		1948	struct drm_display_mode *mode)
		1949	{
		1950	struct drm_i915_private *dev_priv = dev->dev_private;
		1951	u32 temp;
		1952
		1953	temp = I915_READ(PIPE_WM_LINETIME(pipe));
		1954	temp &= ~PIPE_WM_LINETIME_MASK;
		1955
		1956	/* The WM are computed with base on how long it takes to fill a single
		1957	* row at the given clock rate, multiplied by 8.
		1958	* */
		1959	temp \|= PIPE_WM_LINETIME_TIME(
		1960	((mode->crtc_hdisplay * 1000) / mode->clock) * 8);
		1961
		1962	/* IPS watermarks are only used by pipe A, and are ignored by
		1963	* pipes B and C. They are calculated similarly to the common
		1964	* linetime values, except that we are using CD clock frequency
		1965	* in MHz instead of pixel rate for the division.
		1966	*
		1967	* This is a placeholder for the IPS watermark calculation code.
		1968	*/
		1969
		1970	I915_WRITE(PIPE_WM_LINETIME(pipe), temp);
		1971	}
		1972
		1973	static bool
		1974	sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
		1975	uint32_t sprite_width, int pixel_size,
		1976	const struct intel_watermark_params *display,
		1977	int display_latency_ns, int *sprite_wm)
		1978	{
		1979	struct drm_crtc *crtc;
		1980	int clock;
		1981	int entries, tlb_miss;
		1982
		1983	crtc = intel_get_crtc_for_plane(dev, plane);
		1984	if (crtc->fb == NULL \|\| !crtc->enabled) {
		1985	*sprite_wm = display->guard_size;
		1986	return false;
		1987	}
		1988
		1989	clock = crtc->mode.clock;
		1990
		1991	/* Use the small buffer method to calculate the sprite watermark */
		1992	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1993	tlb_miss = display->fifo_size*display->cacheline_size -
		1994	sprite_width * 8;
		1995	if (tlb_miss > 0)
		1996	entries += tlb_miss;
		1997	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1998	*sprite_wm = entries + display->guard_size;
		1999	if (*sprite_wm > (int)display->max_wm)
		2000	*sprite_wm = display->max_wm;
		2001
		2002	return true;
		2003	}
		2004
		2005	static bool
		2006	sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
		2007	uint32_t sprite_width, int pixel_size,
		2008	const struct intel_watermark_params *display,
		2009	int latency_ns, int *sprite_wm)
		2010	{
		2011	struct drm_crtc *crtc;
		2012	unsigned long line_time_us;
		2013	int clock;
		2014	int line_count, line_size;
		2015	int small, large;
		2016	int entries;
		2017
		2018	if (!latency_ns) {
		2019	*sprite_wm = 0;
		2020	return false;
		2021	}
		2022
		2023	crtc = intel_get_crtc_for_plane(dev, plane);
		2024	clock = crtc->mode.clock;
		2025	if (!clock) {
		2026	*sprite_wm = 0;
		2027	return false;
		2028	}
		2029
		2030	line_time_us = (sprite_width * 1000) / clock;
		2031	if (!line_time_us) {
		2032	*sprite_wm = 0;
		2033	return false;
		2034	}
		2035
		2036	line_count = (latency_ns / line_time_us + 1000) / 1000;
		2037	line_size = sprite_width * pixel_size;
		2038
		2039	/* Use the minimum of the small and large buffer method for primary */
		2040	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		2041	large = line_count * line_size;
		2042
		2043	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		2044	*sprite_wm = entries + display->guard_size;
		2045
		2046	return *sprite_wm > 0x3ff ? false : true;
		2047	}
		2048
		2049	static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
		2050	uint32_t sprite_width, int pixel_size)
		2051	{
		2052	struct drm_i915_private *dev_priv = dev->dev_private;
		2053	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		2054	u32 val;
		2055	int sprite_wm, reg;
		2056	int ret;
		2057
		2058	switch (pipe) {
		2059	case 0:
		2060	reg = WM0_PIPEA_ILK;
		2061	break;
		2062	case 1:
		2063	reg = WM0_PIPEB_ILK;
		2064	break;
		2065	case 2:
		2066	reg = WM0_PIPEC_IVB;
		2067	break;
		2068	default:
		2069	return; /* bad pipe */
		2070	}
		2071
		2072	ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
		2073	&sandybridge_display_wm_info,
		2074	latency, &sprite_wm);
		2075	if (!ret) {
		2076	DRM_DEBUG_KMS("failed to compute sprite wm for pipe %d\n",
		2077	pipe);
		2078	return;
		2079	}
		2080
		2081	val = I915_READ(reg);
		2082	val &= ~WM0_PIPE_SPRITE_MASK;
		2083	I915_WRITE(reg, val \| (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
		2084	DRM_DEBUG_KMS("sprite watermarks For pipe %d - %d\n", pipe, sprite_wm);
		2085
		2086
		2087	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2088	pixel_size,
		2089	&sandybridge_display_srwm_info,
		2090	SNB_READ_WM1_LATENCY() * 500,
		2091	&sprite_wm);
		2092	if (!ret) {
		2093	DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %d\n",
		2094	pipe);
		2095	return;
		2096	}
		2097	I915_WRITE(WM1S_LP_ILK, sprite_wm);
		2098
		2099	/* Only IVB has two more LP watermarks for sprite */
		2100	if (!IS_IVYBRIDGE(dev))
		2101	return;
		2102
		2103	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2104	pixel_size,
		2105	&sandybridge_display_srwm_info,
		2106	SNB_READ_WM2_LATENCY() * 500,
		2107	&sprite_wm);
		2108	if (!ret) {
		2109	DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %d\n",
		2110	pipe);
		2111	return;
		2112	}
		2113	I915_WRITE(WM2S_LP_IVB, sprite_wm);
		2114
		2115	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2116	pixel_size,
		2117	&sandybridge_display_srwm_info,
		2118	SNB_READ_WM3_LATENCY() * 500,
		2119	&sprite_wm);
		2120	if (!ret) {
		2121	DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %d\n",
		2122	pipe);
		2123	return;
		2124	}
		2125	I915_WRITE(WM3S_LP_IVB, sprite_wm);
		2126	}
		2127
		2128	/**
		2129	* intel_update_watermarks - update FIFO watermark values based on current modes
		2130	*
		2131	* Calculate watermark values for the various WM regs based on current mode
		2132	* and plane configuration.
		2133	*
		2134	* There are several cases to deal with here:
		2135	* - normal (i.e. non-self-refresh)
		2136	* - self-refresh (SR) mode
		2137	* - lines are large relative to FIFO size (buffer can hold up to 2)
		2138	* - lines are small relative to FIFO size (buffer can hold more than 2
		2139	* lines), so need to account for TLB latency
		2140	*
		2141	* The normal calculation is:
		2142	* watermark = dotclock * bytes per pixel * latency
		2143	* where latency is platform & configuration dependent (we assume pessimal
		2144	* values here).
		2145	*
		2146	* The SR calculation is:
		2147	* watermark = (trunc(latency/line time)+1) * surface width *
		2148	* bytes per pixel
		2149	* where
		2150	* line time = htotal / dotclock
		2151	* surface width = hdisplay for normal plane and 64 for cursor
		2152	* and latency is assumed to be high, as above.
		2153	*
		2154	* The final value programmed to the register should always be rounded up,
		2155	* and include an extra 2 entries to account for clock crossings.
		2156	*
		2157	* We don't use the sprite, so we can ignore that. And on Crestline we have
		2158	* to set the non-SR watermarks to 8.
		2159	*/
		2160	void intel_update_watermarks(struct drm_device *dev)
		2161	{
		2162	struct drm_i915_private *dev_priv = dev->dev_private;
		2163
		2164	if (dev_priv->display.update_wm)
		2165	dev_priv->display.update_wm(dev);
		2166
		2167	}
		2168
		2169	void intel_update_linetime_watermarks(struct drm_device *dev,
		2170	int pipe, struct drm_display_mode *mode)
		2171	{
		2172	struct drm_i915_private *dev_priv = dev->dev_private;
		2173
		2174	if (dev_priv->display.update_linetime_wm)
		2175	dev_priv->display.update_linetime_wm(dev, pipe, mode);
		2176	}
		2177
		2178	void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
		2179	uint32_t sprite_width, int pixel_size)
		2180	{
		2181	struct drm_i915_private *dev_priv = dev->dev_private;
		2182
		2183	if (dev_priv->display.update_sprite_wm)
		2184	dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
		2185	pixel_size);
		2186	}
		2187
		2188	static struct drm_i915_gem_object *
		2189	intel_alloc_context_page(struct drm_device *dev)
		2190	{
		2191	struct drm_i915_gem_object *ctx;
		2192	int ret;
		2193
		2194	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2195
		2196	ctx = i915_gem_alloc_object(dev, 4096);
		2197	if (!ctx) {
		2198	DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
		2199	return NULL;
		2200	}
		2201
		2202	ret = i915_gem_object_pin(ctx, 4096, true, false);
		2203	if (ret) {
		2204	DRM_ERROR("failed to pin power context: %d\n", ret);
		2205	goto err_unref;
		2206	}
		2207
		2208	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
		2209	if (ret) {
		2210	DRM_ERROR("failed to set-domain on power context: %d\n", ret);
		2211	goto err_unpin;
		2212	}
		2213
		2214	return ctx;
		2215
		2216	err_unpin:
		2217	i915_gem_object_unpin(ctx);
		2218	err_unref:
		2219	drm_gem_object_unreference(&ctx->base);
		2220	mutex_unlock(&dev->struct_mutex);
		2221	return NULL;
		2222	}
		2223
		2224	/**
		2225	* Lock protecting IPS related data structures
		2226	*/
		2227	DEFINE_SPINLOCK(mchdev_lock);
		2228
		2229	/* Global for IPS driver to get at the current i915 device. Protected by
		2230	* mchdev_lock. */
		2231	static struct drm_i915_private *i915_mch_dev;
		2232
		2233	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		2234	{
		2235	struct drm_i915_private *dev_priv = dev->dev_private;
		2236	u16 rgvswctl;
		2237
		2238	assert_spin_locked(&mchdev_lock);
		2239
		2240	rgvswctl = I915_READ16(MEMSWCTL);
		2241	if (rgvswctl & MEMCTL_CMD_STS) {
		2242	DRM_DEBUG("gpu busy, RCS change rejected\n");
		2243	return false; /* still busy with another command */
		2244	}
		2245
		2246	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		2247	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		2248	I915_WRITE16(MEMSWCTL, rgvswctl);
		2249	POSTING_READ16(MEMSWCTL);
		2250
		2251	rgvswctl \|= MEMCTL_CMD_STS;
		2252	I915_WRITE16(MEMSWCTL, rgvswctl);
		2253
		2254	return true;
		2255	}
		2256
		2257	static void ironlake_enable_drps(struct drm_device *dev)
		2258	{
		2259	struct drm_i915_private *dev_priv = dev->dev_private;
		2260	u32 rgvmodectl = I915_READ(MEMMODECTL);
		2261	u8 fmax, fmin, fstart, vstart;
		2262
		2263	spin_lock_irq(&mchdev_lock);
		2264
		2265	/* Enable temp reporting */
		2266	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		2267	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		2268
		2269	/* 100ms RC evaluation intervals */
		2270	I915_WRITE(RCUPEI, 100000);
		2271	I915_WRITE(RCDNEI, 100000);
		2272
		2273	/* Set max/min thresholds to 90ms and 80ms respectively */
		2274	I915_WRITE(RCBMAXAVG, 90000);
		2275	I915_WRITE(RCBMINAVG, 80000);
		2276
		2277	I915_WRITE(MEMIHYST, 1);
		2278
		2279	/* Set up min, max, and cur for interrupt handling */
		2280	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		2281	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		2282	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		2283	MEMMODE_FSTART_SHIFT;
		2284
		2285	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
		2286	PXVFREQ_PX_SHIFT;
		2287
		2288	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		2289	dev_priv->ips.fstart = fstart;
		2290
		2291	dev_priv->ips.max_delay = fstart;
		2292	dev_priv->ips.min_delay = fmin;
		2293	dev_priv->ips.cur_delay = fstart;
		2294
		2295	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		2296	fmax, fmin, fstart);
		2297
		2298	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		2299
		2300	/*
		2301	* Interrupts will be enabled in ironlake_irq_postinstall
		2302	*/
		2303
		2304	I915_WRITE(VIDSTART, vstart);
		2305	POSTING_READ(VIDSTART);
		2306
		2307	rgvmodectl \|= MEMMODE_SWMODE_EN;
		2308	I915_WRITE(MEMMODECTL, rgvmodectl);
		2309
		2310	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		2311	DRM_ERROR("stuck trying to change perf mode\n");
		2312	mdelay(1);
		2313
		2314	ironlake_set_drps(dev, fstart);
		2315
		2316	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
		2317	I915_READ(0x112e0);
		2318	dev_priv->ips.last_time1 = jiffies_to_msecs(GetTimerTicks());
		2319	dev_priv->ips.last_count2 = I915_READ(0x112f4);
		2320	// getrawmonotonic(&dev_priv->ips.last_time2);
		2321
		2322	spin_unlock_irq(&mchdev_lock);
		2323	}
		2324
		2325	static void ironlake_disable_drps(struct drm_device *dev)
		2326	{
		2327	struct drm_i915_private *dev_priv = dev->dev_private;
		2328	u16 rgvswctl;
		2329
		2330	spin_lock_irq(&mchdev_lock);
		2331
		2332	rgvswctl = I915_READ16(MEMSWCTL);
		2333
		2334	/* Ack interrupts, disable EFC interrupt */
		2335	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		2336	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		2337	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		2338	I915_WRITE(DEIIR, DE_PCU_EVENT);
		2339	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		2340
		2341	/* Go back to the starting frequency */
		2342	ironlake_set_drps(dev, dev_priv->ips.fstart);
		2343	mdelay(1);
		2344	rgvswctl \|= MEMCTL_CMD_STS;
		2345	I915_WRITE(MEMSWCTL, rgvswctl);
		2346	mdelay(1);
		2347
		2348	spin_unlock_irq(&mchdev_lock);
		2349	}
		2350
		2351	/* There's a funny hw issue where the hw returns all 0 when reading from
		2352	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		2353	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		2354	* all limits and the gpu stuck at whatever frequency it is at atm).
		2355	*/
		2356	static u32 gen6_rps_limits(struct drm_i915_private dev_priv, u8 val)
		2357	{
		2358	u32 limits;
		2359
		2360	limits = 0;
		2361
		2362	if (*val >= dev_priv->rps.max_delay)
		2363	*val = dev_priv->rps.max_delay;
		2364	limits \|= dev_priv->rps.max_delay << 24;
		2365
		2366	/* Only set the down limit when we've reached the lowest level to avoid
		2367	* getting more interrupts, otherwise leave this clear. This prevents a
		2368	* race in the hw when coming out of rc6: There's a tiny window where
		2369	* the hw runs at the minimal clock before selecting the desired
		2370	* frequency, if the down threshold expires in that window we will not
		2371	* receive a down interrupt. */
		2372	if (*val <= dev_priv->rps.min_delay) {
		2373	*val = dev_priv->rps.min_delay;
		2374	limits \|= dev_priv->rps.min_delay << 16;
		2375	}
		2376
		2377	return limits;
		2378	}
		2379
		2380	void gen6_set_rps(struct drm_device *dev, u8 val)
		2381	{
		2382	struct drm_i915_private *dev_priv = dev->dev_private;
		2383	u32 limits = gen6_rps_limits(dev_priv, &val);
		2384
		2385	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2386	WARN_ON(val > dev_priv->rps.max_delay);
		2387	WARN_ON(val < dev_priv->rps.min_delay);
		2388
		2389	if (val == dev_priv->rps.cur_delay)
		2390	return;
		2391
		2392	I915_WRITE(GEN6_RPNSWREQ,
		2393	GEN6_FREQUENCY(val) \|
		2394	GEN6_OFFSET(0) \|
		2395	GEN6_AGGRESSIVE_TURBO);
		2396
		2397	/* Make sure we continue to get interrupts
		2398	* until we hit the minimum or maximum frequencies.
		2399	*/
		2400	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
		2401
		2402	POSTING_READ(GEN6_RPNSWREQ);
		2403
		2404	dev_priv->rps.cur_delay = val;
		2405
		2406	trace_intel_gpu_freq_change(val * 50);
		2407	}
		2408
		2409	static void gen6_disable_rps(struct drm_device *dev)
		2410	{
		2411	struct drm_i915_private *dev_priv = dev->dev_private;
		2412
		2413	I915_WRITE(GEN6_RC_CONTROL, 0);
		2414	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		2415	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
		2416	I915_WRITE(GEN6_PMIER, 0);
		2417	/* Complete PM interrupt masking here doesn't race with the rps work
		2418	* item again unmasking PM interrupts because that is using a different
		2419	* register (PMIMR) to mask PM interrupts. The only risk is in leaving
		2420	* stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
		2421
		2422	spin_lock_irq(&dev_priv->rps.lock);
		2423	dev_priv->rps.pm_iir = 0;
		2424	spin_unlock_irq(&dev_priv->rps.lock);
		2425
		2426	I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
		2427	}
		2428
		2429	int intel_enable_rc6(const struct drm_device *dev)
		2430	{
		2431	/* Respect the kernel parameter if it is set */
		2432	if (i915_enable_rc6 >= 0)
		2433	return i915_enable_rc6;
		2434
		2435	if (INTEL_INFO(dev)->gen == 5) {
		2436	#ifdef CONFIG_INTEL_IOMMU
		2437	/* Disable rc6 on ilk if VT-d is on. */
		2438	if (intel_iommu_gfx_mapped)
		2439	return false;
		2440	#endif
		2441	DRM_DEBUG_DRIVER("Ironlake: only RC6 available\n");
		2442	return INTEL_RC6_ENABLE;
		2443	}
		2444
		2445	if (IS_HASWELL(dev)) {
		2446	DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
		2447	return INTEL_RC6_ENABLE;
		2448	}
		2449
		2450	/* snb/ivb have more than one rc6 state. */
		2451	if (INTEL_INFO(dev)->gen == 6) {
		2452	DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
		2453	return INTEL_RC6_ENABLE;
		2454	}
		2455
		2456	DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
		2457	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
		2458	}
		2459
		2460	static void gen6_enable_rps(struct drm_device *dev)
		2461	{
		2462	struct drm_i915_private *dev_priv = dev->dev_private;
		2463	struct intel_ring_buffer *ring;
		2464	u32 rp_state_cap;
		2465	u32 gt_perf_status;
		2466	u32 pcu_mbox, rc6_mask = 0;
		2467	u32 gtfifodbg;
		2468	int rc6_mode;
		2469	int i;
		2470
		2471	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2472
		2473	/* Here begins a magic sequence of register writes to enable
		2474	* auto-downclocking.
		2475	*
		2476	* Perhaps there might be some value in exposing these to
		2477	* userspace...
		2478	*/
		2479	I915_WRITE(GEN6_RC_STATE, 0);
		2480
		2481	/* Clear the DBG now so we don't confuse earlier errors */
		2482	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		2483	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		2484	I915_WRITE(GTFIFODBG, gtfifodbg);
		2485	}
		2486
		2487	gen6_gt_force_wake_get(dev_priv);
		2488
		2489	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		2490	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
		2491
		2492	/* In units of 100MHz */
		2493	dev_priv->rps.max_delay = rp_state_cap & 0xff;
		2494	dev_priv->rps.min_delay = (rp_state_cap & 0xff0000) >> 16;
		2495	dev_priv->rps.cur_delay = 0;
		2496
		2497	/* disable the counters and set deterministic thresholds */
		2498	I915_WRITE(GEN6_RC_CONTROL, 0);
		2499
		2500	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		2501	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		2502	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		2503	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		2504	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		2505
		2506	for_each_ring(ring, dev_priv, i)
		2507	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		2508
		2509	I915_WRITE(GEN6_RC_SLEEP, 0);
		2510	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
		2511	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
		2512	I915_WRITE(GEN6_RC6p_THRESHOLD, 100000);
		2513	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		2514
		2515	/* Check if we are enabling RC6 */
		2516	rc6_mode = intel_enable_rc6(dev_priv->dev);
		2517	if (rc6_mode & INTEL_RC6_ENABLE)
		2518	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		2519
		2520	/* We don't use those on Haswell */
		2521	if (!IS_HASWELL(dev)) {
		2522	if (rc6_mode & INTEL_RC6p_ENABLE)
		2523	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		2524
		2525	if (rc6_mode & INTEL_RC6pp_ENABLE)
		2526	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		2527	}
		2528
		2529	DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
		2530	(rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		2531	(rc6_mask & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		2532	(rc6_mask & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
		2533
		2534	I915_WRITE(GEN6_RC_CONTROL,
		2535	rc6_mask \|
		2536	GEN6_RC_CTL_EI_MODE(1) \|
		2537	GEN6_RC_CTL_HW_ENABLE);
		2538
		2539	I915_WRITE(GEN6_RPNSWREQ,
		2540	GEN6_FREQUENCY(10) \|
		2541	GEN6_OFFSET(0) \|
		2542	GEN6_AGGRESSIVE_TURBO);
		2543	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		2544	GEN6_FREQUENCY(12));
		2545
		2546	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
		2547	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
		2548	dev_priv->rps.max_delay << 24 \|
		2549	dev_priv->rps.min_delay << 16);
		2550
		2551	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		2552	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		2553	I915_WRITE(GEN6_RP_UP_EI, 66000);
		2554	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		2555
		2556	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		2557	I915_WRITE(GEN6_RP_CONTROL,
		2558	GEN6_RP_MEDIA_TURBO \|
		2559	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		2560	GEN6_RP_MEDIA_IS_GFX \|
		2561	GEN6_RP_ENABLE \|
		2562	GEN6_RP_UP_BUSY_AVG \|
		2563	(IS_HASWELL(dev) ? GEN7_RP_DOWN_IDLE_AVG : GEN6_RP_DOWN_IDLE_CONT));
		2564
		2565	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		2566	500))
		2567	DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
		2568
		2569	I915_WRITE(GEN6_PCODE_DATA, 0);
		2570	I915_WRITE(GEN6_PCODE_MAILBOX,
		2571	GEN6_PCODE_READY \|
		2572	GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
		2573	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		2574	500))
		2575	DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
		2576
		2577	/* Check for overclock support */
		2578	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		2579	500))
		2580	DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
		2581	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_READ_OC_PARAMS);
		2582	pcu_mbox = I915_READ(GEN6_PCODE_DATA);
		2583	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		2584	500))
		2585	DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
		2586	if (pcu_mbox & (1<<31)) { /* OC supported */
		2587	dev_priv->rps.max_delay = pcu_mbox & 0xff;
		2588	DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
		2589	}
		2590
		2591	gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
		2592
		2593	/* requires MSI enabled */
		2594	I915_WRITE(GEN6_PMIER, GEN6_PM_DEFERRED_EVENTS);
		2595	spin_lock_irq(&dev_priv->rps.lock);
		2596	WARN_ON(dev_priv->rps.pm_iir != 0);
		2597	I915_WRITE(GEN6_PMIMR, 0);
		2598	spin_unlock_irq(&dev_priv->rps.lock);
		2599	/* enable all PM interrupts */
		2600	I915_WRITE(GEN6_PMINTRMSK, 0);
		2601
		2602	gen6_gt_force_wake_put(dev_priv);
		2603	}
		2604
		2605	#if 0
		2606	static void gen6_update_ring_freq(struct drm_device *dev)
		2607	{
		2608	struct drm_i915_private *dev_priv = dev->dev_private;
		2609	int min_freq = 15;
		2610	int gpu_freq, ia_freq, max_ia_freq;
		2611	int scaling_factor = 180;
		2612
		2613	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2614
		2615	max_ia_freq = cpufreq_quick_get_max(0);
		2616	/*
		2617	* Default to measured freq if none found, PCU will ensure we don't go
		2618	* over
		2619	*/
		2620	if (!max_ia_freq)
		2621	max_ia_freq = tsc_khz;
		2622
		2623	/* Convert from kHz to MHz */
		2624	max_ia_freq /= 1000;
		2625
		2626	/*
		2627	* For each potential GPU frequency, load a ring frequency we'd like
		2628	* to use for memory access. We do this by specifying the IA frequency
		2629	* the PCU should use as a reference to determine the ring frequency.
		2630	*/
		2631	for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
		2632	gpu_freq--) {
		2633	int diff = dev_priv->rps.max_delay - gpu_freq;
		2634
		2635	/*
		2636	* For GPU frequencies less than 750MHz, just use the lowest
		2637	* ring freq.
		2638	*/
		2639	if (gpu_freq < min_freq)
		2640	ia_freq = 800;
		2641	else
		2642	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		2643	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
		2644
		2645	I915_WRITE(GEN6_PCODE_DATA,
		2646	(ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT) \|
		2647	gpu_freq);
		2648	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \|
		2649	GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
		2650	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
		2651	GEN6_PCODE_READY) == 0, 10)) {
		2652	DRM_ERROR("pcode write of freq table timed out\n");
		2653	continue;
		2654	}
		2655	}
		2656	}
		2657	#endif
		2658
		2659	void ironlake_teardown_rc6(struct drm_device *dev)
		2660	{
		2661	struct drm_i915_private *dev_priv = dev->dev_private;
		2662
		2663	if (dev_priv->renderctx) {
		2664	i915_gem_object_unpin(dev_priv->renderctx);
		2665	drm_gem_object_unreference(&dev_priv->renderctx->base);
		2666	dev_priv->renderctx = NULL;
		2667	}
		2668
		2669	if (dev_priv->pwrctx) {
		2670	i915_gem_object_unpin(dev_priv->pwrctx);
		2671	drm_gem_object_unreference(&dev_priv->pwrctx->base);
		2672	dev_priv->pwrctx = NULL;
		2673	}
		2674	}
		2675
		2676	static void ironlake_disable_rc6(struct drm_device *dev)
		2677	{
		2678	struct drm_i915_private *dev_priv = dev->dev_private;
		2679
		2680	if (I915_READ(PWRCTXA)) {
		2681	/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
		2682	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) \| RCX_SW_EXIT);
		2683	wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
		2684	50);
		2685
		2686	I915_WRITE(PWRCTXA, 0);
		2687	POSTING_READ(PWRCTXA);
		2688
		2689	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		2690	POSTING_READ(RSTDBYCTL);
		2691	}
		2692	}
		2693
		2694	static int ironlake_setup_rc6(struct drm_device *dev)
		2695	{
		2696	struct drm_i915_private *dev_priv = dev->dev_private;
		2697
		2698	if (dev_priv->renderctx == NULL)
		2699	dev_priv->renderctx = intel_alloc_context_page(dev);
		2700	if (!dev_priv->renderctx)
		2701	return -ENOMEM;
		2702
		2703	if (dev_priv->pwrctx == NULL)
		2704	dev_priv->pwrctx = intel_alloc_context_page(dev);
		2705	if (!dev_priv->pwrctx) {
		2706	ironlake_teardown_rc6(dev);
		2707	return -ENOMEM;
		2708	}
		2709
		2710	return 0;
		2711	}
		2712
		2713	static void ironlake_enable_rc6(struct drm_device *dev)
		2714	{
		2715	struct drm_i915_private *dev_priv = dev->dev_private;
		2716	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
		2717	int ret;
		2718
		2719	/* rc6 disabled by default due to repeated reports of hanging during
		2720	* boot and resume.
		2721	*/
		2722	if (!intel_enable_rc6(dev))
		2723	return;
		2724
		2725	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2726
		2727	ret = ironlake_setup_rc6(dev);
		2728	if (ret)
		2729	return;
		2730
		2731	/*
		2732	* GPU can automatically power down the render unit if given a page
		2733	* to save state.
		2734	*/
		2735	ret = intel_ring_begin(ring, 6);
		2736	if (ret) {
		2737	ironlake_teardown_rc6(dev);
		2738	return;
		2739	}
		2740
		2741	intel_ring_emit(ring, MI_SUSPEND_FLUSH \| MI_SUSPEND_FLUSH_EN);
		2742	intel_ring_emit(ring, MI_SET_CONTEXT);
		2743	intel_ring_emit(ring, dev_priv->renderctx->gtt_offset \|
		2744	MI_MM_SPACE_GTT \|
		2745	MI_SAVE_EXT_STATE_EN \|
		2746	MI_RESTORE_EXT_STATE_EN \|
		2747	MI_RESTORE_INHIBIT);
		2748	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
		2749	intel_ring_emit(ring, MI_NOOP);
		2750	intel_ring_emit(ring, MI_FLUSH);
		2751	intel_ring_advance(ring);
		2752
		2753	/*
		2754	* Wait for the command parser to advance past MI_SET_CONTEXT. The HW
		2755	* does an implicit flush, combined with MI_FLUSH above, it should be
		2756	* safe to assume that renderctx is valid
		2757	*/
		2758	ret = intel_wait_ring_idle(ring);
		2759	if (ret) {
		2760	DRM_ERROR("failed to enable ironlake power power savings\n");
		2761	ironlake_teardown_rc6(dev);
		2762	return;
		2763	}
		2764
		2765	I915_WRITE(PWRCTXA, dev_priv->pwrctx->gtt_offset \| PWRCTX_EN);
		2766	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		2767	}
		2768
		2769	static unsigned long intel_pxfreq(u32 vidfreq)
		2770	{
		2771	unsigned long freq;
		2772	int div = (vidfreq & 0x3f0000) >> 16;
		2773	int post = (vidfreq & 0x3000) >> 12;
		2774	int pre = (vidfreq & 0x7);
		2775
		2776	if (!pre)
		2777	return 0;
		2778
		2779	freq = ((div * 133333) / ((1<
		2780
		2781	return freq;
		2782	}
		2783
		2784	static const struct cparams {
		2785	u16 i;
		2786	u16 t;
		2787	u16 m;
		2788	u16 c;
		2789	} cparams[] = {
		2790	{ 1, 1333, 301, 28664 },
		2791	{ 1, 1066, 294, 24460 },
		2792	{ 1, 800, 294, 25192 },
		2793	{ 0, 1333, 276, 27605 },
		2794	{ 0, 1066, 276, 27605 },
		2795	{ 0, 800, 231, 23784 },
		2796	};
		2797
		2798	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		2799	{
		2800	u64 total_count, diff, ret;
		2801	u32 count1, count2, count3, m = 0, c = 0;
		2802	unsigned long now = jiffies_to_msecs(GetTimerTicks()), diff1;
		2803	int i;
		2804
		2805	assert_spin_locked(&mchdev_lock);
		2806
		2807	diff1 = now - dev_priv->ips.last_time1;
		2808
		2809	/* Prevent division-by-zero if we are asking too fast.
		2810	* Also, we don't get interesting results if we are polling
		2811	* faster than once in 10ms, so just return the saved value
		2812	* in such cases.
		2813	*/
		2814	if (diff1 <= 10)
		2815	return dev_priv->ips.chipset_power;
		2816
		2817	count1 = I915_READ(DMIEC);
		2818	count2 = I915_READ(DDREC);
		2819	count3 = I915_READ(CSIEC);
		2820
		2821	total_count = count1 + count2 + count3;
		2822
		2823	/* FIXME: handle per-counter overflow */
		2824	if (total_count < dev_priv->ips.last_count1) {
		2825	diff = ~0UL - dev_priv->ips.last_count1;
		2826	diff += total_count;
		2827	} else {
		2828	diff = total_count - dev_priv->ips.last_count1;
		2829	}
		2830
		2831	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		2832	if (cparams[i].i == dev_priv->ips.c_m &&
		2833	cparams[i].t == dev_priv->ips.r_t) {
		2834	m = cparams[i].m;
		2835	c = cparams[i].c;
		2836	break;
		2837	}
		2838	}
		2839
		2840	diff = div_u64(diff, diff1);
		2841	ret = ((m * diff) + c);
		2842	ret = div_u64(ret, 10);
		2843
		2844	dev_priv->ips.last_count1 = total_count;
		2845	dev_priv->ips.last_time1 = now;
		2846
		2847	dev_priv->ips.chipset_power = ret;
		2848
		2849	return ret;
		2850	}
		2851
		2852	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		2853	{
		2854	unsigned long val;
		2855
		2856	if (dev_priv->info->gen != 5)
		2857	return 0;
		2858
		2859	spin_lock_irq(&mchdev_lock);
		2860
		2861	val = __i915_chipset_val(dev_priv);
		2862
		2863	spin_unlock_irq(&mchdev_lock);
		2864
		2865	return val;
		2866	}
		2867
		2868	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		2869	{
		2870	unsigned long m, x, b;
		2871	u32 tsfs;
		2872
		2873	tsfs = I915_READ(TSFS);
		2874
		2875	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		2876	x = I915_READ8(TR1);
		2877
		2878	b = tsfs & TSFS_INTR_MASK;
		2879
		2880	return ((m * x) / 127) - b;
		2881	}
		2882
		2883	static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		2884	{
		2885	static const struct v_table {
		2886	u16 vd; /* in .1 mil */
		2887	u16 vm; /* in .1 mil */
		2888	} v_table[] = {
		2889	{ 0, 0, },
		2890	{ 375, 0, },
		2891	{ 500, 0, },
		2892	{ 625, 0, },
		2893	{ 750, 0, },
		2894	{ 875, 0, },
		2895	{ 1000, 0, },
		2896	{ 1125, 0, },
		2897	{ 4125, 3000, },
		2898	{ 4125, 3000, },
		2899	{ 4125, 3000, },
		2900	{ 4125, 3000, },
		2901	{ 4125, 3000, },
		2902	{ 4125, 3000, },
		2903	{ 4125, 3000, },
		2904	{ 4125, 3000, },
		2905	{ 4125, 3000, },
		2906	{ 4125, 3000, },
		2907	{ 4125, 3000, },
		2908	{ 4125, 3000, },
		2909	{ 4125, 3000, },
		2910	{ 4125, 3000, },
		2911	{ 4125, 3000, },
		2912	{ 4125, 3000, },
		2913	{ 4125, 3000, },
		2914	{ 4125, 3000, },
		2915	{ 4125, 3000, },
		2916	{ 4125, 3000, },
		2917	{ 4125, 3000, },
		2918	{ 4125, 3000, },
		2919	{ 4125, 3000, },
		2920	{ 4125, 3000, },
		2921	{ 4250, 3125, },
		2922	{ 4375, 3250, },
		2923	{ 4500, 3375, },
		2924	{ 4625, 3500, },
		2925	{ 4750, 3625, },
		2926	{ 4875, 3750, },
		2927	{ 5000, 3875, },
		2928	{ 5125, 4000, },
		2929	{ 5250, 4125, },
		2930	{ 5375, 4250, },
		2931	{ 5500, 4375, },
		2932	{ 5625, 4500, },
		2933	{ 5750, 4625, },
		2934	{ 5875, 4750, },
		2935	{ 6000, 4875, },
		2936	{ 6125, 5000, },
		2937	{ 6250, 5125, },
		2938	{ 6375, 5250, },
		2939	{ 6500, 5375, },
		2940	{ 6625, 5500, },
		2941	{ 6750, 5625, },
		2942	{ 6875, 5750, },
		2943	{ 7000, 5875, },
		2944	{ 7125, 6000, },
		2945	{ 7250, 6125, },
		2946	{ 7375, 6250, },
		2947	{ 7500, 6375, },
		2948	{ 7625, 6500, },
		2949	{ 7750, 6625, },
		2950	{ 7875, 6750, },
		2951	{ 8000, 6875, },
		2952	{ 8125, 7000, },
		2953	{ 8250, 7125, },
		2954	{ 8375, 7250, },
		2955	{ 8500, 7375, },
		2956	{ 8625, 7500, },
		2957	{ 8750, 7625, },
		2958	{ 8875, 7750, },
		2959	{ 9000, 7875, },
		2960	{ 9125, 8000, },
		2961	{ 9250, 8125, },
		2962	{ 9375, 8250, },
		2963	{ 9500, 8375, },
		2964	{ 9625, 8500, },
		2965	{ 9750, 8625, },
		2966	{ 9875, 8750, },
		2967	{ 10000, 8875, },
		2968	{ 10125, 9000, },
		2969	{ 10250, 9125, },
		2970	{ 10375, 9250, },
		2971	{ 10500, 9375, },
		2972	{ 10625, 9500, },
		2973	{ 10750, 9625, },
		2974	{ 10875, 9750, },
		2975	{ 11000, 9875, },
		2976	{ 11125, 10000, },
		2977	{ 11250, 10125, },
		2978	{ 11375, 10250, },
		2979	{ 11500, 10375, },
		2980	{ 11625, 10500, },
		2981	{ 11750, 10625, },
		2982	{ 11875, 10750, },
		2983	{ 12000, 10875, },
		2984	{ 12125, 11000, },
		2985	{ 12250, 11125, },
		2986	{ 12375, 11250, },
		2987	{ 12500, 11375, },
		2988	{ 12625, 11500, },
		2989	{ 12750, 11625, },
		2990	{ 12875, 11750, },
		2991	{ 13000, 11875, },
		2992	{ 13125, 12000, },
		2993	{ 13250, 12125, },
		2994	{ 13375, 12250, },
		2995	{ 13500, 12375, },
		2996	{ 13625, 12500, },
		2997	{ 13750, 12625, },
		2998	{ 13875, 12750, },
		2999	{ 14000, 12875, },
		3000	{ 14125, 13000, },
		3001	{ 14250, 13125, },
		3002	{ 14375, 13250, },
		3003	{ 14500, 13375, },
		3004	{ 14625, 13500, },
		3005	{ 14750, 13625, },
		3006	{ 14875, 13750, },
		3007	{ 15000, 13875, },
		3008	{ 15125, 14000, },
		3009	{ 15250, 14125, },
		3010	{ 15375, 14250, },
		3011	{ 15500, 14375, },
		3012	{ 15625, 14500, },
		3013	{ 15750, 14625, },
		3014	{ 15875, 14750, },
		3015	{ 16000, 14875, },
		3016	{ 16125, 15000, },
		3017	};
		3018	if (dev_priv->info->is_mobile)
		3019	return v_table[pxvid].vm;
		3020	else
		3021	return v_table[pxvid].vd;
		3022	}
		3023
		3024	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		3025	{
		3026	struct timespec now, diff1;
		3027	u64 diff;
		3028	unsigned long diffms;
		3029	u32 count;
		3030
		3031	assert_spin_locked(&mchdev_lock);
		3032
		3033	getrawmonotonic(&now);
		3034	diff1 = timespec_sub(now, dev_priv->ips.last_time2);
		3035
		3036	/* Don't divide by 0 */
		3037	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
		3038	if (!diffms)
		3039	return;
		3040
		3041	count = I915_READ(GFXEC);
		3042
		3043	if (count < dev_priv->ips.last_count2) {
		3044	diff = ~0UL - dev_priv->ips.last_count2;
		3045	diff += count;
		3046	} else {
		3047	diff = count - dev_priv->ips.last_count2;
		3048	}
		3049
		3050	dev_priv->ips.last_count2 = count;
		3051	dev_priv->ips.last_time2 = now;
		3052
		3053	/* More magic constants... */
		3054	diff = diff * 1181;
		3055	diff = div_u64(diff, diffms * 10);
		3056	dev_priv->ips.gfx_power = diff;
		3057	}
		3058
		3059	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		3060	{
		3061	if (dev_priv->info->gen != 5)
		3062	return;
		3063
		3064	spin_lock_irq(&mchdev_lock);
		3065
		3066	__i915_update_gfx_val(dev_priv);
		3067
		3068	spin_unlock_irq(&mchdev_lock);
		3069	}
		3070
		3071	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		3072	{
		3073	unsigned long t, corr, state1, corr2, state2;
		3074	u32 pxvid, ext_v;
		3075
		3076	assert_spin_locked(&mchdev_lock);
		3077
		3078	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
		3079	pxvid = (pxvid >> 24) & 0x7f;
		3080	ext_v = pvid_to_extvid(dev_priv, pxvid);
		3081
		3082	state1 = ext_v;
		3083
		3084	t = i915_mch_val(dev_priv);
		3085
		3086	/* Revel in the empirically derived constants */
		3087
		3088	/* Correction factor in 1/100000 units */
		3089	if (t > 80)
		3090	corr = ((t * 2349) + 135940);
		3091	else if (t >= 50)
		3092	corr = ((t * 964) + 29317);
		3093	else /* < 50 */
		3094	corr = ((t * 301) + 1004);
		3095
		3096	corr = corr * ((150142 * state1) / 10000 - 78642);
		3097	corr /= 100000;
		3098	corr2 = (corr * dev_priv->ips.corr);
		3099
		3100	state2 = (corr2 * state1) / 10000;
		3101	state2 /= 100; /* convert to mW */
		3102
		3103	__i915_update_gfx_val(dev_priv);
		3104
		3105	return dev_priv->ips.gfx_power + state2;
		3106	}
		3107
		3108	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		3109	{
		3110	unsigned long val;
		3111
		3112	if (dev_priv->info->gen != 5)
		3113	return 0;
		3114
		3115	spin_lock_irq(&mchdev_lock);
		3116
		3117	val = __i915_gfx_val(dev_priv);
		3118
		3119	spin_unlock_irq(&mchdev_lock);
		3120
		3121	return val;
		3122	}
		3123
		3124	/**
		3125	* i915_read_mch_val - return value for IPS use
		3126	*
		3127	* Calculate and return a value for the IPS driver to use when deciding whether
		3128	* we have thermal and power headroom to increase CPU or GPU power budget.
		3129	*/
		3130	unsigned long i915_read_mch_val(void)
		3131	{
		3132	struct drm_i915_private *dev_priv;
		3133	unsigned long chipset_val, graphics_val, ret = 0;
		3134
		3135	spin_lock_irq(&mchdev_lock);
		3136	if (!i915_mch_dev)
		3137	goto out_unlock;
		3138	dev_priv = i915_mch_dev;
		3139
		3140	chipset_val = __i915_chipset_val(dev_priv);
		3141	graphics_val = __i915_gfx_val(dev_priv);
		3142
		3143	ret = chipset_val + graphics_val;
		3144
		3145	out_unlock:
		3146	spin_unlock_irq(&mchdev_lock);
		3147
		3148	return ret;
		3149	}
		3150	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		3151
		3152	/**
		3153	* i915_gpu_raise - raise GPU frequency limit
		3154	*
		3155	* Raise the limit; IPS indicates we have thermal headroom.
		3156	*/
		3157	bool i915_gpu_raise(void)
		3158	{
		3159	struct drm_i915_private *dev_priv;
		3160	bool ret = true;
		3161
		3162	spin_lock_irq(&mchdev_lock);
		3163	if (!i915_mch_dev) {
		3164	ret = false;
		3165	goto out_unlock;
		3166	}
		3167	dev_priv = i915_mch_dev;
		3168
		3169	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		3170	dev_priv->ips.max_delay--;
		3171
		3172	out_unlock:
		3173	spin_unlock_irq(&mchdev_lock);
		3174
		3175	return ret;
		3176	}
		3177	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		3178
		3179	/**
		3180	* i915_gpu_lower - lower GPU frequency limit
		3181	*
		3182	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		3183	* frequency maximum.
		3184	*/
		3185	bool i915_gpu_lower(void)
		3186	{
		3187	struct drm_i915_private *dev_priv;
		3188	bool ret = true;
		3189
		3190	spin_lock_irq(&mchdev_lock);
		3191	if (!i915_mch_dev) {
		3192	ret = false;
		3193	goto out_unlock;
		3194	}
		3195	dev_priv = i915_mch_dev;
		3196
		3197	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		3198	dev_priv->ips.max_delay++;
		3199
		3200	out_unlock:
		3201	spin_unlock_irq(&mchdev_lock);
		3202
		3203	return ret;
		3204	}
		3205	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		3206
		3207	/**
		3208	* i915_gpu_busy - indicate GPU business to IPS
		3209	*
		3210	* Tell the IPS driver whether or not the GPU is busy.
		3211	*/
		3212	bool i915_gpu_busy(void)
		3213	{
		3214	struct drm_i915_private *dev_priv;
		3215	struct intel_ring_buffer *ring;
		3216	bool ret = false;
		3217	int i;
		3218
		3219	spin_lock_irq(&mchdev_lock);
		3220	if (!i915_mch_dev)
		3221	goto out_unlock;
		3222	dev_priv = i915_mch_dev;
		3223
		3224	for_each_ring(ring, dev_priv, i)
		3225	ret \|= !list_empty(&ring->request_list);
		3226
		3227	out_unlock:
		3228	spin_unlock_irq(&mchdev_lock);
		3229
		3230	return ret;
		3231	}
		3232	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		3233
		3234	/**
		3235	* i915_gpu_turbo_disable - disable graphics turbo
		3236	*
		3237	* Disable graphics turbo by resetting the max frequency and setting the
		3238	* current frequency to the default.
		3239	*/
		3240	bool i915_gpu_turbo_disable(void)
		3241	{
		3242	struct drm_i915_private *dev_priv;
		3243	bool ret = true;
		3244
		3245	spin_lock_irq(&mchdev_lock);
		3246	if (!i915_mch_dev) {
		3247	ret = false;
		3248	goto out_unlock;
		3249	}
		3250	dev_priv = i915_mch_dev;
		3251
		3252	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		3253
		3254	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		3255	ret = false;
		3256
		3257	out_unlock:
		3258	spin_unlock_irq(&mchdev_lock);
		3259
		3260	return ret;
		3261	}
		3262	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		3263
		3264	/**
		3265	* Tells the intel_ips driver that the i915 driver is now loaded, if
		3266	* IPS got loaded first.
		3267	*
		3268	* This awkward dance is so that neither module has to depend on the
		3269	* other in order for IPS to do the appropriate communication of
		3270	* GPU turbo limits to i915.
		3271	*/
		3272	static void
		3273	ips_ping_for_i915_load(void)
		3274	{
		3275	void (*link)(void);
		3276
		3277	// link = symbol_get(ips_link_to_i915_driver);
		3278	// if (link) {
		3279	// link();
		3280	// symbol_put(ips_link_to_i915_driver);
		3281	// }
		3282	}
		3283
		3284	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		3285	{
		3286	/* We only register the i915 ips part with intel-ips once everything is
		3287	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		3288	spin_lock_irq(&mchdev_lock);
		3289	i915_mch_dev = dev_priv;
		3290	spin_unlock_irq(&mchdev_lock);
		3291
		3292	ips_ping_for_i915_load();
		3293	}
		3294
		3295	void intel_gpu_ips_teardown(void)
		3296	{
		3297	spin_lock_irq(&mchdev_lock);
		3298	i915_mch_dev = NULL;
		3299	spin_unlock_irq(&mchdev_lock);
		3300	}
		3301	static void intel_init_emon(struct drm_device *dev)
		3302	{
		3303	struct drm_i915_private *dev_priv = dev->dev_private;
		3304	u32 lcfuse;
		3305	u8 pxw[16];
		3306	int i;
		3307
		3308	/* Disable to program */
		3309	I915_WRITE(ECR, 0);
		3310	POSTING_READ(ECR);
		3311
		3312	/* Program energy weights for various events */
		3313	I915_WRITE(SDEW, 0x15040d00);
		3314	I915_WRITE(CSIEW0, 0x007f0000);
		3315	I915_WRITE(CSIEW1, 0x1e220004);
		3316	I915_WRITE(CSIEW2, 0x04000004);
		3317
		3318	for (i = 0; i < 5; i++)
		3319	I915_WRITE(PEW + (i * 4), 0);
		3320	for (i = 0; i < 3; i++)
		3321	I915_WRITE(DEW + (i * 4), 0);
		3322
		3323	/* Program P-state weights to account for frequency power adjustment */
		3324	for (i = 0; i < 16; i++) {
		3325	u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
		3326	unsigned long freq = intel_pxfreq(pxvidfreq);
		3327	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		3328	PXVFREQ_PX_SHIFT;
		3329	unsigned long val;
		3330
		3331	val = vid * vid;
		3332	val *= (freq / 1000);
		3333	val *= 255;
		3334	val /= (127127900);
		3335	if (val > 0xff)
		3336	DRM_ERROR("bad pxval: %ld\n", val);
		3337	pxw[i] = val;
		3338	}
		3339	/* Render standby states get 0 weight */
		3340	pxw[14] = 0;
		3341	pxw[15] = 0;
		3342
		3343	for (i = 0; i < 4; i++) {
		3344	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		3345	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
		3346	I915_WRITE(PXW + (i * 4), val);
		3347	}
		3348
		3349	/* Adjust magic regs to magic values (more experimental results) */
		3350	I915_WRITE(OGW0, 0);
		3351	I915_WRITE(OGW1, 0);
		3352	I915_WRITE(EG0, 0x00007f00);
		3353	I915_WRITE(EG1, 0x0000000e);
		3354	I915_WRITE(EG2, 0x000e0000);
		3355	I915_WRITE(EG3, 0x68000300);
		3356	I915_WRITE(EG4, 0x42000000);
		3357	I915_WRITE(EG5, 0x00140031);
		3358	I915_WRITE(EG6, 0);
		3359	I915_WRITE(EG7, 0);
		3360
		3361	for (i = 0; i < 8; i++)
		3362	I915_WRITE(PXWL + (i * 4), 0);
		3363
		3364	/* Enable PMON + select events */
		3365	I915_WRITE(ECR, 0x80000019);
		3366
		3367	lcfuse = I915_READ(LCFUSE02);
		3368
		3369	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		3370	}
		3371
		3372	void intel_disable_gt_powersave(struct drm_device *dev)
		3373	{
		3374	if (IS_IRONLAKE_M(dev)) {
		3375	ironlake_disable_drps(dev);
		3376	ironlake_disable_rc6(dev);
		3377	} else if (INTEL_INFO(dev)->gen >= 6 && !IS_VALLEYVIEW(dev)) {
		3378	gen6_disable_rps(dev);
		3379	}
		3380	}
		3381
		3382	void intel_enable_gt_powersave(struct drm_device *dev)
		3383	{
		3384	if (IS_IRONLAKE_M(dev)) {
		3385	ironlake_enable_drps(dev);
		3386	ironlake_enable_rc6(dev);
		3387	intel_init_emon(dev);
		3388	} else if ((IS_GEN6(dev) \|\| IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) {
		3389	// gen6_enable_rps(dev);
		3390	// gen6_update_ring_freq(dev);
		3391	}
		3392	}
		3393
		3394	static void ironlake_init_clock_gating(struct drm_device *dev)
		3395	{
		3396	struct drm_i915_private *dev_priv = dev->dev_private;
		3397	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
		3398
		3399	/* Required for FBC */
		3400	dspclk_gate \|= DPFCUNIT_CLOCK_GATE_DISABLE \|
		3401	DPFCRUNIT_CLOCK_GATE_DISABLE \|
		3402	DPFDUNIT_CLOCK_GATE_DISABLE;
		3403	/* Required for CxSR */
		3404	dspclk_gate \|= DPARBUNIT_CLOCK_GATE_DISABLE;
		3405
		3406	I915_WRITE(PCH_3DCGDIS0,
		3407	MARIUNIT_CLOCK_GATE_DISABLE \|
		3408	SVSMUNIT_CLOCK_GATE_DISABLE);
		3409	I915_WRITE(PCH_3DCGDIS1,
		3410	VFMUNIT_CLOCK_GATE_DISABLE);
		3411
		3412	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
		3413
		3414	/*
		3415	* According to the spec the following bits should be set in
		3416	* order to enable memory self-refresh
		3417	* The bit 22/21 of 0x42004
		3418	* The bit 5 of 0x42020
		3419	* The bit 15 of 0x45000
		3420	*/
		3421	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3422	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3423	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
		3424	I915_WRITE(ILK_DSPCLK_GATE,
		3425	(I915_READ(ILK_DSPCLK_GATE) \|
		3426	ILK_DPARB_CLK_GATE));
		3427	I915_WRITE(DISP_ARB_CTL,
		3428	(I915_READ(DISP_ARB_CTL) \|
		3429	DISP_FBC_WM_DIS));
		3430	I915_WRITE(WM3_LP_ILK, 0);
		3431	I915_WRITE(WM2_LP_ILK, 0);
		3432	I915_WRITE(WM1_LP_ILK, 0);
		3433
		3434	/*
		3435	* Based on the document from hardware guys the following bits
		3436	* should be set unconditionally in order to enable FBC.
		3437	* The bit 22 of 0x42000
		3438	* The bit 22 of 0x42004
		3439	* The bit 7,8,9 of 0x42020.
		3440	*/
		3441	if (IS_IRONLAKE_M(dev)) {
		3442	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		3443	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		3444	ILK_FBCQ_DIS);
		3445	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3446	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3447	ILK_DPARB_GATE);
		3448	I915_WRITE(ILK_DSPCLK_GATE,
		3449	I915_READ(ILK_DSPCLK_GATE) \|
		3450	ILK_DPFC_DIS1 \|
		3451	ILK_DPFC_DIS2 \|
		3452	ILK_CLK_FBC);
		3453	}
		3454
		3455	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3456	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3457	ILK_ELPIN_409_SELECT);
		3458	I915_WRITE(_3D_CHICKEN2,
		3459	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		3460	_3D_CHICKEN2_WM_READ_PIPELINED);
		3461	}
		3462
		3463	static void gen6_init_clock_gating(struct drm_device *dev)
		3464	{
		3465	struct drm_i915_private *dev_priv = dev->dev_private;
		3466	int pipe;
		3467	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
		3468
		3469	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
		3470
		3471	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3472	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3473	ILK_ELPIN_409_SELECT);
		3474
		3475	I915_WRITE(WM3_LP_ILK, 0);
		3476	I915_WRITE(WM2_LP_ILK, 0);
		3477	I915_WRITE(WM1_LP_ILK, 0);
		3478
		3479	I915_WRITE(CACHE_MODE_0,
		3480	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		3481
		3482	I915_WRITE(GEN6_UCGCTL1,
		3483	I915_READ(GEN6_UCGCTL1) \|
		3484	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		3485	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		3486
		3487	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3488	* gating disable must be set. Failure to set it results in
		3489	* flickering pixels due to Z write ordering failures after
		3490	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3491	* Sanctuary and Tropics, and apparently anything else with
		3492	* alpha test or pixel discard.
		3493	*
		3494	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3495	* but we didn't debug actual testcases to find it out.
		3496	*
		3497	* Also apply WaDisableVDSUnitClockGating and
		3498	* WaDisableRCPBUnitClockGating.
		3499	*/
		3500	I915_WRITE(GEN6_UCGCTL2,
		3501	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		3502	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		3503	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3504
		3505	/* Bspec says we need to always set all mask bits. */
		3506	I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) \|
		3507	_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
		3508
		3509	/*
		3510	* According to the spec the following bits should be
		3511	* set in order to enable memory self-refresh and fbc:
		3512	* The bit21 and bit22 of 0x42000
		3513	* The bit21 and bit22 of 0x42004
		3514	* The bit5 and bit7 of 0x42020
		3515	* The bit14 of 0x70180
		3516	* The bit14 of 0x71180
		3517	*/
		3518	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		3519	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		3520	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		3521	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3522	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3523	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
		3524	I915_WRITE(ILK_DSPCLK_GATE,
		3525	I915_READ(ILK_DSPCLK_GATE) \|
		3526	ILK_DPARB_CLK_GATE \|
		3527	ILK_DPFD_CLK_GATE);
		3528
		3529	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3530	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3531
		3532	for_each_pipe(pipe) {
		3533	I915_WRITE(DSPCNTR(pipe),
		3534	I915_READ(DSPCNTR(pipe)) \|
		3535	DISPPLANE_TRICKLE_FEED_DISABLE);
		3536	intel_flush_display_plane(dev_priv, pipe);
		3537	}
		3538
		3539	/* The default value should be 0x200 according to docs, but the two
		3540	* platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
		3541	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
		3542	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
		3543	}
		3544
		3545	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		3546	{
		3547	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		3548
		3549	reg &= ~GEN7_FF_SCHED_MASK;
		3550	reg \|= GEN7_FF_TS_SCHED_HW;
		3551	reg \|= GEN7_FF_VS_SCHED_HW;
		3552	reg \|= GEN7_FF_DS_SCHED_HW;
		3553
		3554	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		3555	}
		3556
		3557	static void haswell_init_clock_gating(struct drm_device *dev)
		3558	{
		3559	struct drm_i915_private *dev_priv = dev->dev_private;
		3560	int pipe;
		3561	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
		3562
		3563	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
		3564
		3565	I915_WRITE(WM3_LP_ILK, 0);
		3566	I915_WRITE(WM2_LP_ILK, 0);
		3567	I915_WRITE(WM1_LP_ILK, 0);
		3568
		3569	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3570	* This implements the WaDisableRCZUnitClockGating workaround.
		3571	*/
		3572	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
		3573
		3574	I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
		3575
		3576	I915_WRITE(IVB_CHICKEN3,
		3577	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3578	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3579
		3580	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3581	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3582	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3583
		3584	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3585	I915_WRITE(GEN7_L3CNTLREG1,
		3586	GEN7_WA_FOR_GEN7_L3_CONTROL);
		3587	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		3588	GEN7_WA_L3_CHICKEN_MODE);
		3589
		3590	/* This is required by WaCatErrorRejectionIssue */
		3591	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3592	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3593	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3594
		3595	for_each_pipe(pipe) {
		3596	I915_WRITE(DSPCNTR(pipe),
		3597	I915_READ(DSPCNTR(pipe)) \|
		3598	DISPPLANE_TRICKLE_FEED_DISABLE);
		3599	intel_flush_display_plane(dev_priv, pipe);
		3600	}
		3601
		3602	gen7_setup_fixed_func_scheduler(dev_priv);
		3603
		3604	/* WaDisable4x2SubspanOptimization */
		3605	I915_WRITE(CACHE_MODE_1,
		3606	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3607
		3608	/* XXX: This is a workaround for early silicon revisions and should be
		3609	* removed later.
		3610	*/
		3611	I915_WRITE(WM_DBG,
		3612	I915_READ(WM_DBG) \|
		3613	WM_DBG_DISALLOW_MULTIPLE_LP \|
		3614	WM_DBG_DISALLOW_SPRITE \|
		3615	WM_DBG_DISALLOW_MAXFIFO);
		3616
		3617	}
		3618
		3619	static void ivybridge_init_clock_gating(struct drm_device *dev)
		3620	{
		3621	struct drm_i915_private *dev_priv = dev->dev_private;
		3622	int pipe;
		3623	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
		3624	uint32_t snpcr;
		3625
		3626	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
		3627
		3628	I915_WRITE(WM3_LP_ILK, 0);
		3629	I915_WRITE(WM2_LP_ILK, 0);
		3630	I915_WRITE(WM1_LP_ILK, 0);
		3631
		3632	I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
		3633
		3634	I915_WRITE(IVB_CHICKEN3,
		3635	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3636	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3637
		3638	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3639	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3640	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3641
		3642	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3643	I915_WRITE(GEN7_L3CNTLREG1,
		3644	GEN7_WA_FOR_GEN7_L3_CONTROL);
		3645	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		3646	GEN7_WA_L3_CHICKEN_MODE);
		3647
		3648	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3649	* gating disable must be set. Failure to set it results in
		3650	* flickering pixels due to Z write ordering failures after
		3651	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3652	* Sanctuary and Tropics, and apparently anything else with
		3653	* alpha test or pixel discard.
		3654	*
		3655	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3656	* but we didn't debug actual testcases to find it out.
		3657	*
		3658	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3659	* This implements the WaDisableRCZUnitClockGating workaround.
		3660	*/
		3661	I915_WRITE(GEN6_UCGCTL2,
		3662	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		3663	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3664
		3665	/* This is required by WaCatErrorRejectionIssue */
		3666	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3667	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3668	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3669
		3670	for_each_pipe(pipe) {
		3671	I915_WRITE(DSPCNTR(pipe),
		3672	I915_READ(DSPCNTR(pipe)) \|
		3673	DISPPLANE_TRICKLE_FEED_DISABLE);
		3674	intel_flush_display_plane(dev_priv, pipe);
		3675	}
		3676
		3677	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3678	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3679
		3680	gen7_setup_fixed_func_scheduler(dev_priv);
		3681
		3682	/* WaDisable4x2SubspanOptimization */
		3683	I915_WRITE(CACHE_MODE_1,
		3684	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3685
		3686	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		3687	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		3688	snpcr \|= GEN6_MBC_SNPCR_MED;
		3689	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
		3690	}
		3691
		3692	static void valleyview_init_clock_gating(struct drm_device *dev)
		3693	{
		3694	struct drm_i915_private *dev_priv = dev->dev_private;
		3695	int pipe;
		3696	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
		3697
		3698	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
		3699
		3700	I915_WRITE(WM3_LP_ILK, 0);
		3701	I915_WRITE(WM2_LP_ILK, 0);
		3702	I915_WRITE(WM1_LP_ILK, 0);
		3703
		3704	I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
		3705
		3706	I915_WRITE(IVB_CHICKEN3,
		3707	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3708	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3709
		3710	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3711	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3712	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3713
		3714	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3715	I915_WRITE(GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
		3716	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
		3717
		3718	/* This is required by WaCatErrorRejectionIssue */
		3719	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3720	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3721	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3722
		3723	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3724	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3725
		3726
		3727	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3728	* gating disable must be set. Failure to set it results in
		3729	* flickering pixels due to Z write ordering failures after
		3730	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3731	* Sanctuary and Tropics, and apparently anything else with
		3732	* alpha test or pixel discard.
		3733	*
		3734	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3735	* but we didn't debug actual testcases to find it out.
		3736	*
		3737	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3738	* This implements the WaDisableRCZUnitClockGating workaround.
		3739	*
		3740	* Also apply WaDisableVDSUnitClockGating and
		3741	* WaDisableRCPBUnitClockGating.
		3742	*/
		3743	I915_WRITE(GEN6_UCGCTL2,
		3744	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		3745	GEN7_TDLUNIT_CLOCK_GATE_DISABLE \|
		3746	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		3747	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		3748	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3749
		3750	I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
		3751
		3752	for_each_pipe(pipe) {
		3753	I915_WRITE(DSPCNTR(pipe),
		3754	I915_READ(DSPCNTR(pipe)) \|
		3755	DISPPLANE_TRICKLE_FEED_DISABLE);
		3756	intel_flush_display_plane(dev_priv, pipe);
		3757	}
		3758
		3759	I915_WRITE(CACHE_MODE_1,
		3760	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3761
		3762	/*
		3763	* On ValleyView, the GUnit needs to signal the GT
		3764	* when flip and other events complete. So enable
		3765	* all the GUnit->GT interrupts here
		3766	*/
		3767	I915_WRITE(VLV_DPFLIPSTAT, PIPEB_LINE_COMPARE_INT_EN \|
		3768	PIPEB_HLINE_INT_EN \| PIPEB_VBLANK_INT_EN \|
		3769	SPRITED_FLIPDONE_INT_EN \| SPRITEC_FLIPDONE_INT_EN \|
		3770	PLANEB_FLIPDONE_INT_EN \| PIPEA_LINE_COMPARE_INT_EN \|
		3771	PIPEA_HLINE_INT_EN \| PIPEA_VBLANK_INT_EN \|
		3772	SPRITEB_FLIPDONE_INT_EN \| SPRITEA_FLIPDONE_INT_EN \|
		3773	PLANEA_FLIPDONE_INT_EN);
		3774	}
		3775
		3776	static void g4x_init_clock_gating(struct drm_device *dev)
		3777	{
		3778	struct drm_i915_private *dev_priv = dev->dev_private;
		3779	uint32_t dspclk_gate;
		3780
		3781	I915_WRITE(RENCLK_GATE_D1, 0);
		3782	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		3783	GS_UNIT_CLOCK_GATE_DISABLE \|
		3784	CL_UNIT_CLOCK_GATE_DISABLE);
		3785	I915_WRITE(RAMCLK_GATE_D, 0);
		3786	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		3787	OVRUNIT_CLOCK_GATE_DISABLE \|
		3788	OVCUNIT_CLOCK_GATE_DISABLE;
		3789	if (IS_GM45(dev))
		3790	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		3791	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
		3792	}
		3793
		3794	static void crestline_init_clock_gating(struct drm_device *dev)
		3795	{
		3796	struct drm_i915_private *dev_priv = dev->dev_private;
		3797
		3798	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		3799	I915_WRITE(RENCLK_GATE_D2, 0);
		3800	I915_WRITE(DSPCLK_GATE_D, 0);
		3801	I915_WRITE(RAMCLK_GATE_D, 0);
		3802	I915_WRITE16(DEUC, 0);
		3803	}
		3804
		3805	static void broadwater_init_clock_gating(struct drm_device *dev)
		3806	{
		3807	struct drm_i915_private *dev_priv = dev->dev_private;
		3808
		3809	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		3810	I965_RCC_CLOCK_GATE_DISABLE \|
		3811	I965_RCPB_CLOCK_GATE_DISABLE \|
		3812	I965_ISC_CLOCK_GATE_DISABLE \|
		3813	I965_FBC_CLOCK_GATE_DISABLE);
		3814	I915_WRITE(RENCLK_GATE_D2, 0);
		3815	}
		3816
		3817	static void gen3_init_clock_gating(struct drm_device *dev)
		3818	{
		3819	struct drm_i915_private *dev_priv = dev->dev_private;
		3820	u32 dstate = I915_READ(D_STATE);
		3821
		3822	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		3823	DSTATE_DOT_CLOCK_GATING;
		3824	I915_WRITE(D_STATE, dstate);
		3825
		3826	if (IS_PINEVIEW(dev))
		3827	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		3828
		3829	/* IIR "flip pending" means done if this bit is set */
		3830	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
		3831	}
		3832
		3833	static void i85x_init_clock_gating(struct drm_device *dev)
		3834	{
		3835	struct drm_i915_private *dev_priv = dev->dev_private;
		3836
		3837	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
		3838	}
		3839
		3840	static void i830_init_clock_gating(struct drm_device *dev)
		3841	{
		3842	struct drm_i915_private *dev_priv = dev->dev_private;
		3843
		3844	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
		3845	}
		3846
		3847	static void ibx_init_clock_gating(struct drm_device *dev)
		3848	{
		3849	struct drm_i915_private *dev_priv = dev->dev_private;
		3850
		3851	/*
		3852	* On Ibex Peak and Cougar Point, we need to disable clock
		3853	* gating for the panel power sequencer or it will fail to
		3854	* start up when no ports are active.
		3855	*/
		3856	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		3857	}
		3858
		3859	static void cpt_init_clock_gating(struct drm_device *dev)
		3860	{
		3861	struct drm_i915_private *dev_priv = dev->dev_private;
		3862	int pipe;
		3863
		3864	/*
		3865	* On Ibex Peak and Cougar Point, we need to disable clock
		3866	* gating for the panel power sequencer or it will fail to
		3867	* start up when no ports are active.
		3868	*/
		3869	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		3870	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		3871	DPLS_EDP_PPS_FIX_DIS);
		3872	/* Without this, mode sets may fail silently on FDI */
		3873	for_each_pipe(pipe)
		3874	I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_AUTOTRAIN_GEN_STALL_DIS);
		3875	}
		3876
		3877	void intel_init_clock_gating(struct drm_device *dev)
		3878	{
		3879	struct drm_i915_private *dev_priv = dev->dev_private;
		3880
		3881	dev_priv->display.init_clock_gating(dev);
		3882
		3883	if (dev_priv->display.init_pch_clock_gating)
		3884	dev_priv->display.init_pch_clock_gating(dev);
		3885	}
		3886
		3887	/* Starting with Haswell, we have different power wells for
		3888	* different parts of the GPU. This attempts to enable them all.
		3889	*/
		3890	void intel_init_power_wells(struct drm_device *dev)
		3891	{
		3892	struct drm_i915_private *dev_priv = dev->dev_private;
		3893	unsigned long power_wells[] = {
		3894	HSW_PWR_WELL_CTL1,
		3895	HSW_PWR_WELL_CTL2,
		3896	HSW_PWR_WELL_CTL4
		3897	};
		3898	int i;
		3899
		3900	if (!IS_HASWELL(dev))
		3901	return;
		3902
		3903	mutex_lock(&dev->struct_mutex);
		3904
		3905	for (i = 0; i < ARRAY_SIZE(power_wells); i++) {
		3906	int well = I915_READ(power_wells[i]);
		3907
		3908	if ((well & HSW_PWR_WELL_STATE) == 0) {
		3909	I915_WRITE(power_wells[i], well & HSW_PWR_WELL_ENABLE);
		3910	if (wait_for(I915_READ(power_wells[i] & HSW_PWR_WELL_STATE), 20))
		3911	DRM_ERROR("Error enabling power well %lx\n", power_wells[i]);
		3912	}
		3913	}
		3914
		3915	mutex_unlock(&dev->struct_mutex);
		3916	}
		3917
		3918	/* Set up chip specific power management-related functions */
		3919	void intel_init_pm(struct drm_device *dev)
		3920	{
		3921	struct drm_i915_private *dev_priv = dev->dev_private;
		3922
		3923	if (I915_HAS_FBC(dev)) {
		3924	if (HAS_PCH_SPLIT(dev)) {
		3925	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
		3926	dev_priv->display.enable_fbc = ironlake_enable_fbc;
		3927	dev_priv->display.disable_fbc = ironlake_disable_fbc;
		3928	} else if (IS_GM45(dev)) {
		3929	dev_priv->display.fbc_enabled = g4x_fbc_enabled;
		3930	dev_priv->display.enable_fbc = g4x_enable_fbc;
		3931	dev_priv->display.disable_fbc = g4x_disable_fbc;
		3932	} else if (IS_CRESTLINE(dev)) {
		3933	dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
		3934	dev_priv->display.enable_fbc = i8xx_enable_fbc;
		3935	dev_priv->display.disable_fbc = i8xx_disable_fbc;
		3936	}
		3937	/* 855GM needs testing */
		3938	}
		3939
		3940	/* For cxsr */
		3941	if (IS_PINEVIEW(dev))
		3942	i915_pineview_get_mem_freq(dev);
		3943	else if (IS_GEN5(dev))
		3944	i915_ironlake_get_mem_freq(dev);
		3945
		3946	/* For FIFO watermark updates */
		3947	if (HAS_PCH_SPLIT(dev)) {
		3948	if (HAS_PCH_IBX(dev))
		3949	dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
		3950	else if (HAS_PCH_CPT(dev))
		3951	dev_priv->display.init_pch_clock_gating = cpt_init_clock_gating;
		3952
		3953	if (IS_GEN5(dev)) {
		3954	if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
		3955	dev_priv->display.update_wm = ironlake_update_wm;
		3956	else {
		3957	DRM_DEBUG_KMS("Failed to get proper latency. "
		3958	"Disable CxSR\n");
		3959	dev_priv->display.update_wm = NULL;
		3960	}
		3961	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		3962	} else if (IS_GEN6(dev)) {
		3963	if (SNB_READ_WM0_LATENCY()) {
		3964	dev_priv->display.update_wm = sandybridge_update_wm;
		3965	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		3966	} else {
		3967	DRM_DEBUG_KMS("Failed to read display plane latency. "
		3968	"Disable CxSR\n");
		3969	dev_priv->display.update_wm = NULL;
		3970	}
		3971	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
		3972	} else if (IS_IVYBRIDGE(dev)) {
		3973	/* FIXME: detect B0+ stepping and use auto training */
		3974	if (SNB_READ_WM0_LATENCY()) {
		3975	dev_priv->display.update_wm = sandybridge_update_wm;
		3976	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		3977	} else {
		3978	DRM_DEBUG_KMS("Failed to read display plane latency. "
		3979	"Disable CxSR\n");
		3980	dev_priv->display.update_wm = NULL;
		3981	}
		3982	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
		3983	} else if (IS_HASWELL(dev)) {
		3984	if (SNB_READ_WM0_LATENCY()) {
		3985	dev_priv->display.update_wm = sandybridge_update_wm;
		3986	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		3987	dev_priv->display.update_linetime_wm = haswell_update_linetime_wm;
		3988	} else {
		3989	DRM_DEBUG_KMS("Failed to read display plane latency. "
		3990	"Disable CxSR\n");
		3991	dev_priv->display.update_wm = NULL;
		3992	}
		3993	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
		3994	} else
		3995	dev_priv->display.update_wm = NULL;
		3996	} else if (IS_VALLEYVIEW(dev)) {
		3997	dev_priv->display.update_wm = valleyview_update_wm;
		3998	dev_priv->display.init_clock_gating =
		3999	valleyview_init_clock_gating;
		4000	} else if (IS_PINEVIEW(dev)) {
		4001	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		4002	dev_priv->is_ddr3,
		4003	dev_priv->fsb_freq,
		4004	dev_priv->mem_freq)) {
		4005	DRM_INFO("failed to find known CxSR latency "
		4006	"(found ddr%s fsb freq %d, mem freq %d), "
		4007	"disabling CxSR\n",
		4008	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		4009	dev_priv->fsb_freq, dev_priv->mem_freq);
		4010	/* Disable CxSR and never update its watermark again */
		4011	pineview_disable_cxsr(dev);
		4012	dev_priv->display.update_wm = NULL;
		4013	} else
		4014	dev_priv->display.update_wm = pineview_update_wm;
		4015	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		4016	} else if (IS_G4X(dev)) {
		4017	dev_priv->display.update_wm = g4x_update_wm;
		4018	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		4019	} else if (IS_GEN4(dev)) {
		4020	dev_priv->display.update_wm = i965_update_wm;
		4021	if (IS_CRESTLINE(dev))
		4022	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		4023	else if (IS_BROADWATER(dev))
		4024	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		4025	} else if (IS_GEN3(dev)) {
		4026	dev_priv->display.update_wm = i9xx_update_wm;
		4027	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		4028	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		4029	} else if (IS_I865G(dev)) {
		4030	dev_priv->display.update_wm = i830_update_wm;
		4031	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		4032	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		4033	} else if (IS_I85X(dev)) {
		4034	dev_priv->display.update_wm = i9xx_update_wm;
		4035	dev_priv->display.get_fifo_size = i85x_get_fifo_size;
		4036	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		4037	} else {
		4038	dev_priv->display.update_wm = i830_update_wm;
		4039	dev_priv->display.init_clock_gating = i830_init_clock_gating;
		4040	if (IS_845G(dev))
		4041	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		4042	else
		4043	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		4044	}
		4045	}
		4046
		4047	static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
		4048	{
		4049	u32 gt_thread_status_mask;
		4050
		4051	if (IS_HASWELL(dev_priv->dev))
		4052	gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
		4053	else
		4054	gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
		4055
		4056	/* w/a for a sporadic read returning 0 by waiting for the GT
		4057	* thread to wake up.
		4058	*/
		4059	if (wait_for_atomic_us((I915_READ_NOTRACE(GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
		4060	DRM_ERROR("GT thread status wait timed out\n");
		4061	}
		4062
		4063	static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
		4064	{
		4065	u32 forcewake_ack;
		4066
		4067	if (IS_HASWELL(dev_priv->dev))
		4068	forcewake_ack = FORCEWAKE_ACK_HSW;
		4069	else
		4070	forcewake_ack = FORCEWAKE_ACK;
		4071
		4072	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1) == 0,
		4073	FORCEWAKE_ACK_TIMEOUT_MS))
		4074	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4075
		4076	I915_WRITE_NOTRACE(FORCEWAKE, 1);
		4077	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
		4078
		4079	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
		4080	FORCEWAKE_ACK_TIMEOUT_MS))
		4081	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4082
		4083	__gen6_gt_wait_for_thread_c0(dev_priv);
		4084	}
		4085
		4086	static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
		4087	{
		4088	u32 forcewake_ack;
		4089
		4090	if (IS_HASWELL(dev_priv->dev))
		4091	forcewake_ack = FORCEWAKE_ACK_HSW;
		4092	else
		4093	forcewake_ack = FORCEWAKE_MT_ACK;
		4094
		4095	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1) == 0,
		4096	FORCEWAKE_ACK_TIMEOUT_MS))
		4097	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4098
		4099	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(1));
		4100	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
		4101
		4102	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
		4103	FORCEWAKE_ACK_TIMEOUT_MS))
		4104	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4105
		4106	__gen6_gt_wait_for_thread_c0(dev_priv);
		4107	}
		4108
		4109	/*
		4110	* Generally this is called implicitly by the register read function. However,
		4111	* if some sequence requires the GT to not power down then this function should
		4112	* be called at the beginning of the sequence followed by a call to
		4113	* gen6_gt_force_wake_put() at the end of the sequence.
		4114	*/
		4115	void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
		4116	{
		4117	unsigned long irqflags;
		4118
		4119	spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
		4120	if (dev_priv->forcewake_count++ == 0)
		4121	dev_priv->gt.force_wake_get(dev_priv);
		4122	spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
		4123	}
		4124
		4125	void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
		4126	{
		4127	u32 gtfifodbg;
		4128	gtfifodbg = I915_READ_NOTRACE(GTFIFODBG);
		4129	if (WARN(gtfifodbg & GT_FIFO_CPU_ERROR_MASK,
		4130	"MMIO read or write has been dropped %x\n", gtfifodbg))
		4131	I915_WRITE_NOTRACE(GTFIFODBG, GT_FIFO_CPU_ERROR_MASK);
		4132	}
		4133
		4134	static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
		4135	{
		4136	I915_WRITE_NOTRACE(FORCEWAKE, 0);
		4137	/* gen6_gt_check_fifodbg doubles as the POSTING_READ */
		4138	gen6_gt_check_fifodbg(dev_priv);
		4139	}
		4140
		4141	static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
		4142	{
		4143	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(1));
		4144	/* gen6_gt_check_fifodbg doubles as the POSTING_READ */
		4145	gen6_gt_check_fifodbg(dev_priv);
		4146	}
		4147
		4148	/*
		4149	* see gen6_gt_force_wake_get()
		4150	*/
		4151	void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
		4152	{
		4153	unsigned long irqflags;
		4154
		4155	spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
		4156	if (--dev_priv->forcewake_count == 0)
		4157	dev_priv->gt.force_wake_put(dev_priv);
		4158	spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
		4159	}
		4160
		4161	int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
		4162	{
		4163	int ret = 0;
		4164
		4165	if (dev_priv->gt_fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
		4166	int loop = 500;
		4167	u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
		4168	while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
		4169	udelay(10);
		4170	fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
		4171	}
		4172	if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
		4173	++ret;
		4174	dev_priv->gt_fifo_count = fifo;
		4175	}
		4176	dev_priv->gt_fifo_count--;
		4177
		4178	return ret;
		4179	}
		4180
		4181	static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
		4182	{
		4183	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1) == 0,
		4184	FORCEWAKE_ACK_TIMEOUT_MS))
		4185	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4186
		4187	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(1));
		4188
		4189	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1),
		4190	FORCEWAKE_ACK_TIMEOUT_MS))
		4191	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4192
		4193	__gen6_gt_wait_for_thread_c0(dev_priv);
		4194	}
		4195
		4196	static void vlv_force_wake_put(struct drm_i915_private *dev_priv)
		4197	{
		4198	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(1));
		4199	/* The below doubles as a POSTING_READ */
		4200	gen6_gt_check_fifodbg(dev_priv);
		4201	}
		4202
		4203	void intel_gt_init(struct drm_device *dev)
		4204	{
		4205	struct drm_i915_private *dev_priv = dev->dev_private;
		4206
		4207	spin_lock_init(&dev_priv->gt_lock);
		4208
		4209	if (IS_VALLEYVIEW(dev)) {
		4210	dev_priv->gt.force_wake_get = vlv_force_wake_get;
		4211	dev_priv->gt.force_wake_put = vlv_force_wake_put;
		4212	} else if (INTEL_INFO(dev)->gen >= 6) {
		4213	dev_priv->gt.force_wake_get = __gen6_gt_force_wake_get;
		4214	dev_priv->gt.force_wake_put = __gen6_gt_force_wake_put;
		4215
		4216	/* IVB configs may use multi-threaded forcewake */
		4217	if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev)) {
		4218	u32 ecobus;
		4219
		4220	/* A small trick here - if the bios hasn't configured
		4221	* MT forcewake, and if the device is in RC6, then
		4222	* force_wake_mt_get will not wake the device and the
		4223	* ECOBUS read will return zero. Which will be
		4224	* (correctly) interpreted by the test below as MT
		4225	* forcewake being disabled.
		4226	*/
		4227	mutex_lock(&dev->struct_mutex);
		4228	__gen6_gt_force_wake_mt_get(dev_priv);
		4229	ecobus = I915_READ_NOTRACE(ECOBUS);
		4230	__gen6_gt_force_wake_mt_put(dev_priv);
		4231	mutex_unlock(&dev->struct_mutex);
		4232
		4233	if (ecobus & FORCEWAKE_MT_ENABLE) {
		4234	DRM_DEBUG_KMS("Using MT version of forcewake\n");
		4235	dev_priv->gt.force_wake_get =
		4236	__gen6_gt_force_wake_mt_get;
		4237	dev_priv->gt.force_wake_put =
		4238	__gen6_gt_force_wake_mt_put;
		4239	}
		4240	}
		4241	}
		4242	}
		4243

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 3031