WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	//#include
		29	#include "i915_drv.h"
		30	#include "intel_drv.h"
		31	#include
		32	//#include "../../../platform/x86/intel_ips.h"
		33	#include
		34
		35	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		36
		37	#define assert_spin_locked(x)
		38
		39	void getrawmonotonic(struct timespec *ts);
		40	void set_normalized_timespec(struct timespec *ts, time_t sec, long nsec);
		41
		42	static inline struct timespec timespec_sub(struct timespec lhs,
		43	struct timespec rhs)
		44	{
		45	struct timespec ts_delta;
		46	set_normalized_timespec(&ts_delta, lhs.tv_sec - rhs.tv_sec,
		47	lhs.tv_nsec - rhs.tv_nsec);
		48	return ts_delta;
		49	}
		50
		51
		52	/* FBC, or Frame Buffer Compression, is a technique employed to compress the
		53	* framebuffer contents in-memory, aiming at reducing the required bandwidth
		54	* during in-memory transfers and, therefore, reduce the power packet.
		55	*
		56	* The benefits of FBC are mostly visible with solid backgrounds and
		57	* variation-less patterns.
		58	*
		59	* FBC-related functionality can be enabled by the means of the
		60	* i915.i915_enable_fbc parameter
		61	*/
		62
3243	Serge	63	static bool intel_crtc_active(struct drm_crtc *crtc)
		64	{
		65	/* Be paranoid as we can arrive here with only partial
		66	* state retrieved from the hardware during setup.
		67	*/
		68	return to_intel_crtc(crtc)->active && crtc->fb && crtc->mode.clock;
		69	}
		70
3031	serge	71	static void i8xx_disable_fbc(struct drm_device *dev)
		72	{
		73	struct drm_i915_private *dev_priv = dev->dev_private;
		74	u32 fbc_ctl;
		75
		76	/* Disable compression */
		77	fbc_ctl = I915_READ(FBC_CONTROL);
		78	if ((fbc_ctl & FBC_CTL_EN) == 0)
		79	return;
		80
		81	fbc_ctl &= ~FBC_CTL_EN;
		82	I915_WRITE(FBC_CONTROL, fbc_ctl);
		83
		84	/* Wait for compressing bit to clear */
		85	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
		86	DRM_DEBUG_KMS("FBC idle timed out\n");
		87	return;
		88	}
		89
		90	DRM_DEBUG_KMS("disabled FBC\n");
		91	}
		92
		93	static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		94	{
		95	struct drm_device *dev = crtc->dev;
		96	struct drm_i915_private *dev_priv = dev->dev_private;
		97	struct drm_framebuffer *fb = crtc->fb;
		98	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		99	struct drm_i915_gem_object *obj = intel_fb->obj;
		100	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		101	int cfb_pitch;
		102	int plane, i;
		103	u32 fbc_ctl, fbc_ctl2;
		104
		105	cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
		106	if (fb->pitches[0] < cfb_pitch)
		107	cfb_pitch = fb->pitches[0];
		108
		109	/* FBC_CTL wants 64B units */
		110	cfb_pitch = (cfb_pitch / 64) - 1;
		111	plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
		112
		113	/* Clear old tags */
		114	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
		115	I915_WRITE(FBC_TAG + (i * 4), 0);
		116
		117	/* Set it up... */
		118	fbc_ctl2 = FBC_CTL_FENCE_DBL \| FBC_CTL_IDLE_IMM \| FBC_CTL_CPU_FENCE;
		119	fbc_ctl2 \|= plane;
		120	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
		121	I915_WRITE(FBC_FENCE_OFF, crtc->y);
		122
		123	/* enable it... */
		124	fbc_ctl = FBC_CTL_EN \| FBC_CTL_PERIODIC;
		125	if (IS_I945GM(dev))
		126	fbc_ctl \|= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
		127	fbc_ctl \|= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
		128	fbc_ctl \|= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
		129	fbc_ctl \|= obj->fence_reg;
		130	I915_WRITE(FBC_CONTROL, fbc_ctl);
		131
		132	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
		133	cfb_pitch, crtc->y, intel_crtc->plane);
		134	}
		135
		136	static bool i8xx_fbc_enabled(struct drm_device *dev)
		137	{
		138	struct drm_i915_private *dev_priv = dev->dev_private;
		139
		140	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
		141	}
		142
		143	static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		144	{
		145	struct drm_device *dev = crtc->dev;
		146	struct drm_i915_private *dev_priv = dev->dev_private;
		147	struct drm_framebuffer *fb = crtc->fb;
		148	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		149	struct drm_i915_gem_object *obj = intel_fb->obj;
		150	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		151	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		152	unsigned long stall_watermark = 200;
		153	u32 dpfc_ctl;
		154
		155	dpfc_ctl = plane \| DPFC_SR_EN \| DPFC_CTL_LIMIT_1X;
		156	dpfc_ctl \|= DPFC_CTL_FENCE_EN \| obj->fence_reg;
		157	I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
		158
		159	I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		160	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		161	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		162	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
		163
		164	/* enable it... */
		165	I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) \| DPFC_CTL_EN);
		166
		167	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		168	}
		169
		170	static void g4x_disable_fbc(struct drm_device *dev)
		171	{
		172	struct drm_i915_private *dev_priv = dev->dev_private;
		173	u32 dpfc_ctl;
		174
		175	/* Disable compression */
		176	dpfc_ctl = I915_READ(DPFC_CONTROL);
		177	if (dpfc_ctl & DPFC_CTL_EN) {
		178	dpfc_ctl &= ~DPFC_CTL_EN;
		179	I915_WRITE(DPFC_CONTROL, dpfc_ctl);
		180
		181	DRM_DEBUG_KMS("disabled FBC\n");
		182	}
		183	}
		184
		185	static bool g4x_fbc_enabled(struct drm_device *dev)
		186	{
		187	struct drm_i915_private *dev_priv = dev->dev_private;
		188
		189	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
		190	}
		191
		192	static void sandybridge_blit_fbc_update(struct drm_device *dev)
		193	{
		194	struct drm_i915_private *dev_priv = dev->dev_private;
		195	u32 blt_ecoskpd;
		196
		197	/* Make sure blitter notifies FBC of writes */
		198	gen6_gt_force_wake_get(dev_priv);
		199	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
		200	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY <<
		201	GEN6_BLITTER_LOCK_SHIFT;
		202	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		203	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY;
		204	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		205	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
		206	GEN6_BLITTER_LOCK_SHIFT);
		207	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		208	POSTING_READ(GEN6_BLITTER_ECOSKPD);
		209	gen6_gt_force_wake_put(dev_priv);
		210	}
		211
		212	static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		213	{
		214	struct drm_device *dev = crtc->dev;
		215	struct drm_i915_private *dev_priv = dev->dev_private;
		216	struct drm_framebuffer *fb = crtc->fb;
		217	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		218	struct drm_i915_gem_object *obj = intel_fb->obj;
		219	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		220	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		221	unsigned long stall_watermark = 200;
		222	u32 dpfc_ctl;
		223
		224	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		225	dpfc_ctl &= DPFC_RESERVED;
		226	dpfc_ctl \|= (plane \| DPFC_CTL_LIMIT_1X);
		227	/* Set persistent mode for front-buffer rendering, ala X. */
		228	dpfc_ctl \|= DPFC_CTL_PERSISTENT_MODE;
		229	dpfc_ctl \|= (DPFC_CTL_FENCE_EN \| obj->fence_reg);
		230	I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
		231
		232	I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		233	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		234	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		235	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
		236	I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset \| ILK_FBC_RT_VALID);
		237	/* enable it... */
		238	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		239
		240	if (IS_GEN6(dev)) {
		241	I915_WRITE(SNB_DPFC_CTL_SA,
		242	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		243	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		244	sandybridge_blit_fbc_update(dev);
		245	}
		246
		247	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		248	}
		249
		250	static void ironlake_disable_fbc(struct drm_device *dev)
		251	{
		252	struct drm_i915_private *dev_priv = dev->dev_private;
		253	u32 dpfc_ctl;
		254
		255	/* Disable compression */
		256	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		257	if (dpfc_ctl & DPFC_CTL_EN) {
		258	dpfc_ctl &= ~DPFC_CTL_EN;
		259	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
		260
		261	DRM_DEBUG_KMS("disabled FBC\n");
		262	}
		263	}
		264
		265	static bool ironlake_fbc_enabled(struct drm_device *dev)
		266	{
		267	struct drm_i915_private *dev_priv = dev->dev_private;
		268
		269	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
		270	}
		271
		272	bool intel_fbc_enabled(struct drm_device *dev)
		273	{
		274	struct drm_i915_private *dev_priv = dev->dev_private;
		275
		276	if (!dev_priv->display.fbc_enabled)
		277	return false;
		278
		279	return dev_priv->display.fbc_enabled(dev);
		280	}
		281
		282	static void intel_fbc_work_fn(struct work_struct *__work)
		283	{
		284	struct intel_fbc_work *work =
		285	container_of(to_delayed_work(__work),
		286	struct intel_fbc_work, work);
		287	struct drm_device *dev = work->crtc->dev;
		288	struct drm_i915_private *dev_priv = dev->dev_private;
		289
		290	mutex_lock(&dev->struct_mutex);
		291	if (work == dev_priv->fbc_work) {
		292	/* Double check that we haven't switched fb without cancelling
		293	* the prior work.
		294	*/
		295	if (work->crtc->fb == work->fb) {
		296	dev_priv->display.enable_fbc(work->crtc,
		297	work->interval);
		298
		299	dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
		300	dev_priv->cfb_fb = work->crtc->fb->base.id;
		301	dev_priv->cfb_y = work->crtc->y;
		302	}
		303
		304	dev_priv->fbc_work = NULL;
		305	}
		306	mutex_unlock(&dev->struct_mutex);
		307
		308	kfree(work);
		309	}
		310
		311	static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
		312	{
		313	if (dev_priv->fbc_work == NULL)
		314	return;
		315
		316	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
		317
		318	/* Synchronisation is provided by struct_mutex and checking of
		319	* dev_priv->fbc_work, so we can perform the cancellation
		320	* entirely asynchronously.
		321	*/
3482	Serge	322	// if (cancel_delayed_work(&dev_priv->fbc_work->work))
3031	serge	323	/* tasklet was killed before being run, clean up */
3482	Serge	324	// kfree(dev_priv->fbc_work);
3031	serge	325
		326	/* Mark the work as no longer wanted so that if it does
		327	* wake-up (because the work was already running and waiting
		328	* for our mutex), it will discover that is no longer
		329	* necessary to run.
		330	*/
		331	dev_priv->fbc_work = NULL;
		332	}
		333
		334	void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		335	{
		336	struct intel_fbc_work *work;
		337	struct drm_device *dev = crtc->dev;
		338	struct drm_i915_private *dev_priv = dev->dev_private;
		339
3482	Serge	340	if (!dev_priv->display.enable_fbc)
3031	serge	341	return;
3482	Serge	342
3031	serge	343	intel_cancel_fbc_work(dev_priv);
		344
		345	work = kzalloc(sizeof *work, GFP_KERNEL);
		346	if (work == NULL) {
		347	dev_priv->display.enable_fbc(crtc, interval);
		348	return;
		349	}
		350
		351	work->crtc = crtc;
		352	work->fb = crtc->fb;
		353	work->interval = interval;
		354	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
		355
		356	dev_priv->fbc_work = work;
		357
		358	DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
		359
		360	/* Delay the actual enabling to let pageflipping cease and the
		361	* display to settle before starting the compression. Note that
		362	* this delay also serves a second purpose: it allows for a
		363	* vblank to pass after disabling the FBC before we attempt
		364	* to modify the control registers.
		365	*
		366	* A more complicated solution would involve tracking vblanks
		367	* following the termination of the page-flipping sequence
		368	* and indeed performing the enable as a co-routine and not
		369	* waiting synchronously upon the vblank.
		370	*/
		371	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
		372	}
		373
		374	void intel_disable_fbc(struct drm_device *dev)
		375	{
		376	struct drm_i915_private *dev_priv = dev->dev_private;
		377
3482	Serge	378	intel_cancel_fbc_work(dev_priv);
3031	serge	379
3482	Serge	380	if (!dev_priv->display.disable_fbc)
		381	return;
3031	serge	382
3482	Serge	383	dev_priv->display.disable_fbc(dev);
3031	serge	384	dev_priv->cfb_plane = -1;
		385	}
		386
		387	/**
		388	* intel_update_fbc - enable/disable FBC as needed
		389	* @dev: the drm_device
		390	*
		391	* Set up the framebuffer compression hardware at mode set time. We
		392	* enable it if possible:
		393	* - plane A only (on pre-965)
		394	* - no pixel mulitply/line duplication
		395	* - no alpha buffer discard
		396	* - no dual wide
		397	* - framebuffer <= 2048 in width, 1536 in height
		398	*
		399	* We can't assume that any compression will take place (worst case),
		400	* so the compressed buffer has to be the same size as the uncompressed
		401	* one. It also must reside (along with the line length buffer) in
		402	* stolen memory.
		403	*
		404	* We need to enable/disable FBC on a global basis.
		405	*/
		406	void intel_update_fbc(struct drm_device *dev)
		407	{
		408	struct drm_i915_private *dev_priv = dev->dev_private;
		409	struct drm_crtc crtc = NULL, tmp_crtc;
		410	struct intel_crtc *intel_crtc;
		411	struct drm_framebuffer *fb;
		412	struct intel_framebuffer *intel_fb;
		413	struct drm_i915_gem_object *obj;
		414	int enable_fbc;
		415
3482	Serge	416	ENTER();
		417
3031	serge	418	if (!i915_powersave)
		419	return;
		420
		421	if (!I915_HAS_FBC(dev))
		422	return;
		423
		424	/*
		425	* If FBC is already on, we just have to verify that we can
		426	* keep it that way...
		427	* Need to disable if:
		428	* - more than one pipe is active
		429	* - changing FBC params (stride, fence, mode)
		430	* - new fb is too large to fit in compressed buffer
		431	* - going to an unsupported config (interlace, pixel multiply, etc.)
		432	*/
		433	list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	434	if (intel_crtc_active(tmp_crtc) &&
		435	!to_intel_crtc(tmp_crtc)->primary_disabled) {
3031	serge	436	if (crtc) {
		437	DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
		438	dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
		439	goto out_disable;
		440	}
		441	crtc = tmp_crtc;
		442	}
		443	}
		444
		445	if (!crtc \|\| crtc->fb == NULL) {
		446	DRM_DEBUG_KMS("no output, disabling\n");
		447	dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
		448	goto out_disable;
		449	}
		450
		451	intel_crtc = to_intel_crtc(crtc);
		452	fb = crtc->fb;
		453	intel_fb = to_intel_framebuffer(fb);
		454	obj = intel_fb->obj;
		455
		456	enable_fbc = i915_enable_fbc;
		457	if (enable_fbc < 0) {
		458	DRM_DEBUG_KMS("fbc set to per-chip default\n");
		459	enable_fbc = 1;
		460	if (INTEL_INFO(dev)->gen <= 6)
		461	enable_fbc = 0;
		462	}
		463	if (!enable_fbc) {
		464	DRM_DEBUG_KMS("fbc disabled per module param\n");
		465	dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
		466	goto out_disable;
		467	}
		468	if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) \|\|
		469	(crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
		470	DRM_DEBUG_KMS("mode incompatible with compression, "
		471	"disabling\n");
		472	dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
		473	goto out_disable;
		474	}
		475	if ((crtc->mode.hdisplay > 2048) \|\|
		476	(crtc->mode.vdisplay > 1536)) {
		477	DRM_DEBUG_KMS("mode too large for compression, disabling\n");
		478	dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
		479	goto out_disable;
		480	}
		481	if ((IS_I915GM(dev) \|\| IS_I945GM(dev)) && intel_crtc->plane != 0) {
		482	DRM_DEBUG_KMS("plane not 0, disabling compression\n");
		483	dev_priv->no_fbc_reason = FBC_BAD_PLANE;
		484	goto out_disable;
		485	}
		486
		487	/* The use of a CPU fence is mandatory in order to detect writes
		488	* by the CPU to the scanout and trigger updates to the FBC.
		489	*/
		490	if (obj->tiling_mode != I915_TILING_X \|\|
		491	obj->fence_reg == I915_FENCE_REG_NONE) {
		492	DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
		493	dev_priv->no_fbc_reason = FBC_NOT_TILED;
		494	goto out_disable;
		495	}
		496
		497	/* If the kernel debugger is active, always disable compression */
		498	if (in_dbg_master())
		499	goto out_disable;
		500
3480	Serge	501	if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
		502	DRM_INFO("not enough stolen space for compressed buffer (need %zd bytes), disabling\n", intel_fb->obj->base.size);
		503	DRM_INFO("hint: you may be able to increase stolen memory size in the BIOS to avoid this\n");
		504	DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
		505	dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
		506	goto out_disable;
		507	}
		508
3031	serge	509	/* If the scanout has not changed, don't modify the FBC settings.
		510	* Note that we make the fundamental assumption that the fb->obj
		511	* cannot be unpinned (and have its GTT offset and fence revoked)
		512	* without first being decoupled from the scanout and FBC disabled.
		513	*/
		514	if (dev_priv->cfb_plane == intel_crtc->plane &&
		515	dev_priv->cfb_fb == fb->base.id &&
		516	dev_priv->cfb_y == crtc->y)
		517	return;
		518
		519	if (intel_fbc_enabled(dev)) {
		520	/* We update FBC along two paths, after changing fb/crtc
		521	* configuration (modeswitching) and after page-flipping
		522	* finishes. For the latter, we know that not only did
		523	* we disable the FBC at the start of the page-flip
		524	* sequence, but also more than one vblank has passed.
		525	*
		526	* For the former case of modeswitching, it is possible
		527	* to switch between two FBC valid configurations
		528	* instantaneously so we do need to disable the FBC
		529	* before we can modify its control registers. We also
		530	* have to wait for the next vblank for that to take
		531	* effect. However, since we delay enabling FBC we can
		532	* assume that a vblank has passed since disabling and
		533	* that we can safely alter the registers in the deferred
		534	* callback.
		535	*
		536	* In the scenario that we go from a valid to invalid
		537	* and then back to valid FBC configuration we have
		538	* no strict enforcement that a vblank occurred since
		539	* disabling the FBC. However, along all current pipe
		540	* disabling paths we do need to wait for a vblank at
		541	* some point. And we wait before enabling FBC anyway.
		542	*/
		543	DRM_DEBUG_KMS("disabling active FBC for update\n");
		544	intel_disable_fbc(dev);
		545	}
		546
		547	intel_enable_fbc(crtc, 500);
3482	Serge	548	LEAVE();
		549
3031	serge	550	return;
		551
		552	out_disable:
		553	/* Multiple disables should be harmless */
		554	if (intel_fbc_enabled(dev)) {
		555	DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
		556	intel_disable_fbc(dev);
		557	}
3480	Serge	558	i915_gem_stolen_cleanup_compression(dev);
3482	Serge	559	LEAVE();
3031	serge	560	}
		561
		562	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		563	{
		564	drm_i915_private_t *dev_priv = dev->dev_private;
		565	u32 tmp;
		566
		567	tmp = I915_READ(CLKCFG);
		568
		569	switch (tmp & CLKCFG_FSB_MASK) {
		570	case CLKCFG_FSB_533:
		571	dev_priv->fsb_freq = 533; /* 1334 /
		572	break;
		573	case CLKCFG_FSB_800:
		574	dev_priv->fsb_freq = 800; /* 2004 /
		575	break;
		576	case CLKCFG_FSB_667:
		577	dev_priv->fsb_freq = 667; /* 1674 /
		578	break;
		579	case CLKCFG_FSB_400:
		580	dev_priv->fsb_freq = 400; /* 1004 /
		581	break;
		582	}
		583
		584	switch (tmp & CLKCFG_MEM_MASK) {
		585	case CLKCFG_MEM_533:
		586	dev_priv->mem_freq = 533;
		587	break;
		588	case CLKCFG_MEM_667:
		589	dev_priv->mem_freq = 667;
		590	break;
		591	case CLKCFG_MEM_800:
		592	dev_priv->mem_freq = 800;
		593	break;
		594	}
		595
		596	/* detect pineview DDR3 setting */
		597	tmp = I915_READ(CSHRDDR3CTL);
		598	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		599	}
		600
		601	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		602	{
		603	drm_i915_private_t *dev_priv = dev->dev_private;
		604	u16 ddrpll, csipll;
		605
		606	ddrpll = I915_READ16(DDRMPLL1);
		607	csipll = I915_READ16(CSIPLL0);
		608
		609	switch (ddrpll & 0xff) {
		610	case 0xc:
		611	dev_priv->mem_freq = 800;
		612	break;
		613	case 0x10:
		614	dev_priv->mem_freq = 1066;
		615	break;
		616	case 0x14:
		617	dev_priv->mem_freq = 1333;
		618	break;
		619	case 0x18:
		620	dev_priv->mem_freq = 1600;
		621	break;
		622	default:
		623	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		624	ddrpll & 0xff);
		625	dev_priv->mem_freq = 0;
		626	break;
		627	}
		628
		629	dev_priv->ips.r_t = dev_priv->mem_freq;
		630
		631	switch (csipll & 0x3ff) {
		632	case 0x00c:
		633	dev_priv->fsb_freq = 3200;
		634	break;
		635	case 0x00e:
		636	dev_priv->fsb_freq = 3733;
		637	break;
		638	case 0x010:
		639	dev_priv->fsb_freq = 4266;
		640	break;
		641	case 0x012:
		642	dev_priv->fsb_freq = 4800;
		643	break;
		644	case 0x014:
		645	dev_priv->fsb_freq = 5333;
		646	break;
		647	case 0x016:
		648	dev_priv->fsb_freq = 5866;
		649	break;
		650	case 0x018:
		651	dev_priv->fsb_freq = 6400;
		652	break;
		653	default:
		654	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		655	csipll & 0x3ff);
		656	dev_priv->fsb_freq = 0;
		657	break;
		658	}
		659
		660	if (dev_priv->fsb_freq == 3200) {
		661	dev_priv->ips.c_m = 0;
		662	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		663	dev_priv->ips.c_m = 1;
		664	} else {
		665	dev_priv->ips.c_m = 2;
		666	}
		667	}
		668
		669	static const struct cxsr_latency cxsr_latency_table[] = {
		670	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		671	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		672	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		673	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		674	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		675
		676	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		677	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		678	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		679	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		680	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		681
		682	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		683	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		684	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		685	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		686	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		687
		688	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		689	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		690	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		691	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		692	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		693
		694	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		695	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		696	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		697	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		698	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		699
		700	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		701	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		702	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		703	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		704	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		705	};
		706
		707	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		708	int is_ddr3,
		709	int fsb,
		710	int mem)
		711	{
		712	const struct cxsr_latency *latency;
		713	int i;
		714
		715	if (fsb == 0 \|\| mem == 0)
		716	return NULL;
		717
		718	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		719	latency = &cxsr_latency_table[i];
		720	if (is_desktop == latency->is_desktop &&
		721	is_ddr3 == latency->is_ddr3 &&
		722	fsb == latency->fsb_freq && mem == latency->mem_freq)
		723	return latency;
		724	}
		725
		726	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		727
		728	return NULL;
		729	}
		730
		731	static void pineview_disable_cxsr(struct drm_device *dev)
		732	{
		733	struct drm_i915_private *dev_priv = dev->dev_private;
		734
		735	/* deactivate cxsr */
		736	I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
		737	}
		738
		739	/*
		740	* Latency for FIFO fetches is dependent on several factors:
		741	* - memory configuration (speed, channels)
		742	* - chipset
		743	* - current MCH state
		744	* It can be fairly high in some situations, so here we assume a fairly
		745	* pessimal value. It's a tradeoff between extra memory fetches (if we
		746	* set this value too high, the FIFO will fetch frequently to stay full)
		747	* and power consumption (set it too low to save power and we might see
		748	* FIFO underruns and display "flicker").
		749	*
		750	* A value of 5us seems to be a good balance; safe for very low end
		751	* platforms but not overly aggressive on lower latency configs.
		752	*/
		753	static const int latency_ns = 5000;
		754
		755	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		756	{
		757	struct drm_i915_private *dev_priv = dev->dev_private;
		758	uint32_t dsparb = I915_READ(DSPARB);
		759	int size;
		760
		761	size = dsparb & 0x7f;
		762	if (plane)
		763	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		764
		765	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		766	plane ? "B" : "A", size);
		767
		768	return size;
		769	}
		770
		771	static int i85x_get_fifo_size(struct drm_device *dev, int plane)
		772	{
		773	struct drm_i915_private *dev_priv = dev->dev_private;
		774	uint32_t dsparb = I915_READ(DSPARB);
		775	int size;
		776
		777	size = dsparb & 0x1ff;
		778	if (plane)
		779	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		780	size >>= 1; /* Convert to cachelines */
		781
		782	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		783	plane ? "B" : "A", size);
		784
		785	return size;
		786	}
		787
		788	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		789	{
		790	struct drm_i915_private *dev_priv = dev->dev_private;
		791	uint32_t dsparb = I915_READ(DSPARB);
		792	int size;
		793
		794	size = dsparb & 0x7f;
		795	size >>= 2; /* Convert to cachelines */
		796
		797	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		798	plane ? "B" : "A",
		799	size);
		800
		801	return size;
		802	}
		803
		804	static int i830_get_fifo_size(struct drm_device *dev, int plane)
		805	{
		806	struct drm_i915_private *dev_priv = dev->dev_private;
		807	uint32_t dsparb = I915_READ(DSPARB);
		808	int size;
		809
		810	size = dsparb & 0x7f;
		811	size >>= 1; /* Convert to cachelines */
		812
		813	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		814	plane ? "B" : "A", size);
		815
		816	return size;
		817	}
		818
		819	/* Pineview has different values for various configs */
		820	static const struct intel_watermark_params pineview_display_wm = {
		821	PINEVIEW_DISPLAY_FIFO,
		822	PINEVIEW_MAX_WM,
		823	PINEVIEW_DFT_WM,
		824	PINEVIEW_GUARD_WM,
		825	PINEVIEW_FIFO_LINE_SIZE
		826	};
		827	static const struct intel_watermark_params pineview_display_hplloff_wm = {
		828	PINEVIEW_DISPLAY_FIFO,
		829	PINEVIEW_MAX_WM,
		830	PINEVIEW_DFT_HPLLOFF_WM,
		831	PINEVIEW_GUARD_WM,
		832	PINEVIEW_FIFO_LINE_SIZE
		833	};
		834	static const struct intel_watermark_params pineview_cursor_wm = {
		835	PINEVIEW_CURSOR_FIFO,
		836	PINEVIEW_CURSOR_MAX_WM,
		837	PINEVIEW_CURSOR_DFT_WM,
		838	PINEVIEW_CURSOR_GUARD_WM,
		839	PINEVIEW_FIFO_LINE_SIZE,
		840	};
		841	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
		842	PINEVIEW_CURSOR_FIFO,
		843	PINEVIEW_CURSOR_MAX_WM,
		844	PINEVIEW_CURSOR_DFT_WM,
		845	PINEVIEW_CURSOR_GUARD_WM,
		846	PINEVIEW_FIFO_LINE_SIZE
		847	};
		848	static const struct intel_watermark_params g4x_wm_info = {
		849	G4X_FIFO_SIZE,
		850	G4X_MAX_WM,
		851	G4X_MAX_WM,
		852	2,
		853	G4X_FIFO_LINE_SIZE,
		854	};
		855	static const struct intel_watermark_params g4x_cursor_wm_info = {
		856	I965_CURSOR_FIFO,
		857	I965_CURSOR_MAX_WM,
		858	I965_CURSOR_DFT_WM,
		859	2,
		860	G4X_FIFO_LINE_SIZE,
		861	};
		862	static const struct intel_watermark_params valleyview_wm_info = {
		863	VALLEYVIEW_FIFO_SIZE,
		864	VALLEYVIEW_MAX_WM,
		865	VALLEYVIEW_MAX_WM,
		866	2,
		867	G4X_FIFO_LINE_SIZE,
		868	};
		869	static const struct intel_watermark_params valleyview_cursor_wm_info = {
		870	I965_CURSOR_FIFO,
		871	VALLEYVIEW_CURSOR_MAX_WM,
		872	I965_CURSOR_DFT_WM,
		873	2,
		874	G4X_FIFO_LINE_SIZE,
		875	};
		876	static const struct intel_watermark_params i965_cursor_wm_info = {
		877	I965_CURSOR_FIFO,
		878	I965_CURSOR_MAX_WM,
		879	I965_CURSOR_DFT_WM,
		880	2,
		881	I915_FIFO_LINE_SIZE,
		882	};
		883	static const struct intel_watermark_params i945_wm_info = {
		884	I945_FIFO_SIZE,
		885	I915_MAX_WM,
		886	1,
		887	2,
		888	I915_FIFO_LINE_SIZE
		889	};
		890	static const struct intel_watermark_params i915_wm_info = {
		891	I915_FIFO_SIZE,
		892	I915_MAX_WM,
		893	1,
		894	2,
		895	I915_FIFO_LINE_SIZE
		896	};
		897	static const struct intel_watermark_params i855_wm_info = {
		898	I855GM_FIFO_SIZE,
		899	I915_MAX_WM,
		900	1,
		901	2,
		902	I830_FIFO_LINE_SIZE
		903	};
		904	static const struct intel_watermark_params i830_wm_info = {
		905	I830_FIFO_SIZE,
		906	I915_MAX_WM,
		907	1,
		908	2,
		909	I830_FIFO_LINE_SIZE
		910	};
		911
		912	static const struct intel_watermark_params ironlake_display_wm_info = {
		913	ILK_DISPLAY_FIFO,
		914	ILK_DISPLAY_MAXWM,
		915	ILK_DISPLAY_DFTWM,
		916	2,
		917	ILK_FIFO_LINE_SIZE
		918	};
		919	static const struct intel_watermark_params ironlake_cursor_wm_info = {
		920	ILK_CURSOR_FIFO,
		921	ILK_CURSOR_MAXWM,
		922	ILK_CURSOR_DFTWM,
		923	2,
		924	ILK_FIFO_LINE_SIZE
		925	};
		926	static const struct intel_watermark_params ironlake_display_srwm_info = {
		927	ILK_DISPLAY_SR_FIFO,
		928	ILK_DISPLAY_MAX_SRWM,
		929	ILK_DISPLAY_DFT_SRWM,
		930	2,
		931	ILK_FIFO_LINE_SIZE
		932	};
		933	static const struct intel_watermark_params ironlake_cursor_srwm_info = {
		934	ILK_CURSOR_SR_FIFO,
		935	ILK_CURSOR_MAX_SRWM,
		936	ILK_CURSOR_DFT_SRWM,
		937	2,
		938	ILK_FIFO_LINE_SIZE
		939	};
		940
		941	static const struct intel_watermark_params sandybridge_display_wm_info = {
		942	SNB_DISPLAY_FIFO,
		943	SNB_DISPLAY_MAXWM,
		944	SNB_DISPLAY_DFTWM,
		945	2,
		946	SNB_FIFO_LINE_SIZE
		947	};
		948	static const struct intel_watermark_params sandybridge_cursor_wm_info = {
		949	SNB_CURSOR_FIFO,
		950	SNB_CURSOR_MAXWM,
		951	SNB_CURSOR_DFTWM,
		952	2,
		953	SNB_FIFO_LINE_SIZE
		954	};
		955	static const struct intel_watermark_params sandybridge_display_srwm_info = {
		956	SNB_DISPLAY_SR_FIFO,
		957	SNB_DISPLAY_MAX_SRWM,
		958	SNB_DISPLAY_DFT_SRWM,
		959	2,
		960	SNB_FIFO_LINE_SIZE
		961	};
		962	static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
		963	SNB_CURSOR_SR_FIFO,
		964	SNB_CURSOR_MAX_SRWM,
		965	SNB_CURSOR_DFT_SRWM,
		966	2,
		967	SNB_FIFO_LINE_SIZE
		968	};
		969
		970
		971	/**
		972	* intel_calculate_wm - calculate watermark level
		973	* @clock_in_khz: pixel clock
		974	* @wm: chip FIFO params
		975	* @pixel_size: display pixel size
		976	* @latency_ns: memory latency for the platform
		977	*
		978	* Calculate the watermark level (the level at which the display plane will
		979	* start fetching from memory again). Each chip has a different display
		980	* FIFO size and allocation, so the caller needs to figure that out and pass
		981	* in the correct intel_watermark_params structure.
		982	*
		983	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		984	* on the pixel size. When it reaches the watermark level, it'll start
		985	* fetching FIFO line sized based chunks from memory until the FIFO fills
		986	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		987	* will occur, and a display engine hang could result.
		988	*/
		989	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		990	const struct intel_watermark_params *wm,
		991	int fifo_size,
		992	int pixel_size,
		993	unsigned long latency_ns)
		994	{
		995	long entries_required, wm_size;
		996
		997	/*
		998	* Note: we need to make sure we don't overflow for various clock &
		999	* latency values.
		1000	* clocks go from a few thousand to several hundred thousand.
		1001	* latency is usually a few thousand
		1002	*/
		1003	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		1004	1000;
		1005	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		1006
		1007	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		1008
		1009	wm_size = fifo_size - (entries_required + wm->guard_size);
		1010
		1011	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		1012
		1013	/* Don't promote wm_size to unsigned... */
		1014	if (wm_size > (long)wm->max_wm)
		1015	wm_size = wm->max_wm;
		1016	if (wm_size <= 0)
		1017	wm_size = wm->default_wm;
		1018	return wm_size;
		1019	}
		1020
		1021	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		1022	{
		1023	struct drm_crtc crtc, enabled = NULL;
		1024
		1025	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	1026	if (intel_crtc_active(crtc)) {
3031	serge	1027	if (enabled)
		1028	return NULL;
		1029	enabled = crtc;
		1030	}
		1031	}
		1032
		1033	return enabled;
		1034	}
		1035
		1036	static void pineview_update_wm(struct drm_device *dev)
		1037	{
		1038	struct drm_i915_private *dev_priv = dev->dev_private;
		1039	struct drm_crtc *crtc;
		1040	const struct cxsr_latency *latency;
		1041	u32 reg;
		1042	unsigned long wm;
		1043
		1044	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		1045	dev_priv->fsb_freq, dev_priv->mem_freq);
		1046	if (!latency) {
		1047	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		1048	pineview_disable_cxsr(dev);
		1049	return;
		1050	}
		1051
		1052	crtc = single_enabled_crtc(dev);
		1053	if (crtc) {
		1054	int clock = crtc->mode.clock;
		1055	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1056
		1057	/* Display SR */
		1058	wm = intel_calculate_wm(clock, &pineview_display_wm,
		1059	pineview_display_wm.fifo_size,
		1060	pixel_size, latency->display_sr);
		1061	reg = I915_READ(DSPFW1);
		1062	reg &= ~DSPFW_SR_MASK;
		1063	reg \|= wm << DSPFW_SR_SHIFT;
		1064	I915_WRITE(DSPFW1, reg);
		1065	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		1066
		1067	/* cursor SR */
		1068	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		1069	pineview_display_wm.fifo_size,
		1070	pixel_size, latency->cursor_sr);
		1071	reg = I915_READ(DSPFW3);
		1072	reg &= ~DSPFW_CURSOR_SR_MASK;
		1073	reg \|= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
		1074	I915_WRITE(DSPFW3, reg);
		1075
		1076	/* Display HPLL off SR */
		1077	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		1078	pineview_display_hplloff_wm.fifo_size,
		1079	pixel_size, latency->display_hpll_disable);
		1080	reg = I915_READ(DSPFW3);
		1081	reg &= ~DSPFW_HPLL_SR_MASK;
		1082	reg \|= wm & DSPFW_HPLL_SR_MASK;
		1083	I915_WRITE(DSPFW3, reg);
		1084
		1085	/* cursor HPLL off SR */
		1086	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		1087	pineview_display_hplloff_wm.fifo_size,
		1088	pixel_size, latency->cursor_hpll_disable);
		1089	reg = I915_READ(DSPFW3);
		1090	reg &= ~DSPFW_HPLL_CURSOR_MASK;
		1091	reg \|= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
		1092	I915_WRITE(DSPFW3, reg);
		1093	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		1094
		1095	/* activate cxsr */
		1096	I915_WRITE(DSPFW3,
		1097	I915_READ(DSPFW3) \| PINEVIEW_SELF_REFRESH_EN);
		1098	DRM_DEBUG_KMS("Self-refresh is enabled\n");
		1099	} else {
		1100	pineview_disable_cxsr(dev);
		1101	DRM_DEBUG_KMS("Self-refresh is disabled\n");
		1102	}
		1103	}
		1104
		1105	static bool g4x_compute_wm0(struct drm_device *dev,
		1106	int plane,
		1107	const struct intel_watermark_params *display,
		1108	int display_latency_ns,
		1109	const struct intel_watermark_params *cursor,
		1110	int cursor_latency_ns,
		1111	int *plane_wm,
		1112	int *cursor_wm)
		1113	{
		1114	struct drm_crtc *crtc;
		1115	int htotal, hdisplay, clock, pixel_size;
		1116	int line_time_us, line_count;
		1117	int entries, tlb_miss;
		1118
		1119	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1120	if (!intel_crtc_active(crtc)) {
3031	serge	1121	*cursor_wm = cursor->guard_size;
		1122	*plane_wm = display->guard_size;
		1123	return false;
		1124	}
		1125
		1126	htotal = crtc->mode.htotal;
		1127	hdisplay = crtc->mode.hdisplay;
		1128	clock = crtc->mode.clock;
		1129	pixel_size = crtc->fb->bits_per_pixel / 8;
		1130
		1131	/* Use the small buffer method to calculate plane watermark */
		1132	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1133	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		1134	if (tlb_miss > 0)
		1135	entries += tlb_miss;
		1136	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1137	*plane_wm = entries + display->guard_size;
		1138	if (*plane_wm > (int)display->max_wm)
		1139	*plane_wm = display->max_wm;
		1140
		1141	/* Use the large buffer method to calculate cursor watermark */
		1142	line_time_us = ((htotal * 1000) / clock);
		1143	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
		1144	entries = line_count * 64 * pixel_size;
		1145	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		1146	if (tlb_miss > 0)
		1147	entries += tlb_miss;
		1148	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1149	*cursor_wm = entries + cursor->guard_size;
		1150	if (*cursor_wm > (int)cursor->max_wm)
		1151	*cursor_wm = (int)cursor->max_wm;
		1152
		1153	return true;
		1154	}
		1155
		1156	/*
		1157	* Check the wm result.
		1158	*
		1159	* If any calculated watermark values is larger than the maximum value that
		1160	* can be programmed into the associated watermark register, that watermark
		1161	* must be disabled.
		1162	*/
		1163	static bool g4x_check_srwm(struct drm_device *dev,
		1164	int display_wm, int cursor_wm,
		1165	const struct intel_watermark_params *display,
		1166	const struct intel_watermark_params *cursor)
		1167	{
		1168	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		1169	display_wm, cursor_wm);
		1170
		1171	if (display_wm > display->max_wm) {
		1172	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		1173	display_wm, display->max_wm);
		1174	return false;
		1175	}
		1176
		1177	if (cursor_wm > cursor->max_wm) {
		1178	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		1179	cursor_wm, cursor->max_wm);
		1180	return false;
		1181	}
		1182
		1183	if (!(display_wm \|\| cursor_wm)) {
		1184	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		1185	return false;
		1186	}
		1187
		1188	return true;
		1189	}
		1190
		1191	static bool g4x_compute_srwm(struct drm_device *dev,
		1192	int plane,
		1193	int latency_ns,
		1194	const struct intel_watermark_params *display,
		1195	const struct intel_watermark_params *cursor,
		1196	int display_wm, int cursor_wm)
		1197	{
		1198	struct drm_crtc *crtc;
		1199	int hdisplay, htotal, pixel_size, clock;
		1200	unsigned long line_time_us;
		1201	int line_count, line_size;
		1202	int small, large;
		1203	int entries;
		1204
		1205	if (!latency_ns) {
		1206	display_wm = cursor_wm = 0;
		1207	return false;
		1208	}
		1209
		1210	crtc = intel_get_crtc_for_plane(dev, plane);
		1211	hdisplay = crtc->mode.hdisplay;
		1212	htotal = crtc->mode.htotal;
		1213	clock = crtc->mode.clock;
		1214	pixel_size = crtc->fb->bits_per_pixel / 8;
		1215
		1216	line_time_us = (htotal * 1000) / clock;
		1217	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1218	line_size = hdisplay * pixel_size;
		1219
		1220	/* Use the minimum of the small and large buffer method for primary */
		1221	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1222	large = line_count * line_size;
		1223
		1224	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1225	*display_wm = entries + display->guard_size;
		1226
		1227	/* calculate the self-refresh watermark for display cursor */
		1228	entries = line_count * pixel_size * 64;
		1229	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1230	*cursor_wm = entries + cursor->guard_size;
		1231
		1232	return g4x_check_srwm(dev,
		1233	display_wm, cursor_wm,
		1234	display, cursor);
		1235	}
		1236
		1237	static bool vlv_compute_drain_latency(struct drm_device *dev,
		1238	int plane,
		1239	int *plane_prec_mult,
		1240	int *plane_dl,
		1241	int *cursor_prec_mult,
		1242	int *cursor_dl)
		1243	{
		1244	struct drm_crtc *crtc;
		1245	int clock, pixel_size;
		1246	int entries;
		1247
		1248	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1249	if (!intel_crtc_active(crtc))
3031	serge	1250	return false;
		1251
		1252	clock = crtc->mode.clock; /* VESA DOT Clock */
		1253	pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */
		1254
		1255	entries = (clock / 1000) * pixel_size;
		1256	*plane_prec_mult = (entries > 256) ?
		1257	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1258	plane_dl = (64 (plane_prec_mult) 4) / ((clock / 1000) *
		1259	pixel_size);
		1260
		1261	entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
		1262	*cursor_prec_mult = (entries > 256) ?
		1263	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1264	cursor_dl = (64 (cursor_prec_mult) 4) / ((clock / 1000) * 4);
		1265
		1266	return true;
		1267	}
		1268
		1269	/*
		1270	* Update drain latency registers of memory arbiter
		1271	*
		1272	* Valleyview SoC has a new memory arbiter and needs drain latency registers
		1273	* to be programmed. Each plane has a drain latency multiplier and a drain
		1274	* latency value.
		1275	*/
		1276
		1277	static void vlv_update_drain_latency(struct drm_device *dev)
		1278	{
		1279	struct drm_i915_private *dev_priv = dev->dev_private;
		1280	int planea_prec, planea_dl, planeb_prec, planeb_dl;
		1281	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
		1282	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
		1283	either 16 or 32 */
		1284
		1285	/* For plane A, Cursor A */
		1286	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
		1287	&cursor_prec_mult, &cursora_dl)) {
		1288	cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1289	DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
		1290	planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1291	DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
		1292
		1293	I915_WRITE(VLV_DDL1, cursora_prec \|
		1294	(cursora_dl << DDL_CURSORA_SHIFT) \|
		1295	planea_prec \| planea_dl);
		1296	}
		1297
		1298	/* For plane B, Cursor B */
		1299	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
		1300	&cursor_prec_mult, &cursorb_dl)) {
		1301	cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1302	DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
		1303	planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1304	DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
		1305
		1306	I915_WRITE(VLV_DDL2, cursorb_prec \|
		1307	(cursorb_dl << DDL_CURSORB_SHIFT) \|
		1308	planeb_prec \| planeb_dl);
		1309	}
		1310	}
		1311
		1312	#define single_plane_enabled(mask) is_power_of_2(mask)
		1313
		1314	static void valleyview_update_wm(struct drm_device *dev)
		1315	{
		1316	static const int sr_latency_ns = 12000;
		1317	struct drm_i915_private *dev_priv = dev->dev_private;
		1318	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1319	int plane_sr, cursor_sr;
3243	Serge	1320	int ignore_plane_sr, ignore_cursor_sr;
3031	serge	1321	unsigned int enabled = 0;
		1322
		1323	vlv_update_drain_latency(dev);
		1324
3746	Serge	1325	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1326	&valleyview_wm_info, latency_ns,
		1327	&valleyview_cursor_wm_info, latency_ns,
		1328	&planea_wm, &cursora_wm))
3746	Serge	1329	enabled \|= 1 << PIPE_A;
3031	serge	1330
3746	Serge	1331	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1332	&valleyview_wm_info, latency_ns,
		1333	&valleyview_cursor_wm_info, latency_ns,
		1334	&planeb_wm, &cursorb_wm))
3746	Serge	1335	enabled \|= 1 << PIPE_B;
3031	serge	1336
		1337	if (single_plane_enabled(enabled) &&
		1338	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1339	sr_latency_ns,
		1340	&valleyview_wm_info,
		1341	&valleyview_cursor_wm_info,
3243	Serge	1342	&plane_sr, &ignore_cursor_sr) &&
		1343	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1344	2*sr_latency_ns,
		1345	&valleyview_wm_info,
		1346	&valleyview_cursor_wm_info,
		1347	&ignore_plane_sr, &cursor_sr)) {
3031	serge	1348	I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
3243	Serge	1349	} else {
3031	serge	1350	I915_WRITE(FW_BLC_SELF_VLV,
		1351	I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
3243	Serge	1352	plane_sr = cursor_sr = 0;
		1353	}
3031	serge	1354
		1355	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1356	planea_wm, cursora_wm,
		1357	planeb_wm, cursorb_wm,
		1358	plane_sr, cursor_sr);
		1359
		1360	I915_WRITE(DSPFW1,
		1361	(plane_sr << DSPFW_SR_SHIFT) \|
		1362	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1363	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1364	planea_wm);
		1365	I915_WRITE(DSPFW2,
3243	Serge	1366	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1367	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1368	I915_WRITE(DSPFW3,
3243	Serge	1369	(I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) \|
		1370	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
3031	serge	1371	}
		1372
		1373	static void g4x_update_wm(struct drm_device *dev)
		1374	{
		1375	static const int sr_latency_ns = 12000;
		1376	struct drm_i915_private *dev_priv = dev->dev_private;
		1377	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1378	int plane_sr, cursor_sr;
		1379	unsigned int enabled = 0;
		1380
3746	Serge	1381	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1382	&g4x_wm_info, latency_ns,
		1383	&g4x_cursor_wm_info, latency_ns,
		1384	&planea_wm, &cursora_wm))
3746	Serge	1385	enabled \|= 1 << PIPE_A;
3031	serge	1386
3746	Serge	1387	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1388	&g4x_wm_info, latency_ns,
		1389	&g4x_cursor_wm_info, latency_ns,
		1390	&planeb_wm, &cursorb_wm))
3746	Serge	1391	enabled \|= 1 << PIPE_B;
3031	serge	1392
		1393	if (single_plane_enabled(enabled) &&
		1394	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1395	sr_latency_ns,
		1396	&g4x_wm_info,
		1397	&g4x_cursor_wm_info,
3243	Serge	1398	&plane_sr, &cursor_sr)) {
3031	serge	1399	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
3243	Serge	1400	} else {
3031	serge	1401	I915_WRITE(FW_BLC_SELF,
		1402	I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
3243	Serge	1403	plane_sr = cursor_sr = 0;
		1404	}
3031	serge	1405
		1406	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1407	planea_wm, cursora_wm,
		1408	planeb_wm, cursorb_wm,
		1409	plane_sr, cursor_sr);
		1410
		1411	I915_WRITE(DSPFW1,
		1412	(plane_sr << DSPFW_SR_SHIFT) \|
		1413	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1414	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1415	planea_wm);
		1416	I915_WRITE(DSPFW2,
3243	Serge	1417	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1418	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1419	/* HPLL off in SR has some issues on G4x... disable it */
		1420	I915_WRITE(DSPFW3,
3243	Serge	1421	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
3031	serge	1422	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1423	}
		1424
		1425	static void i965_update_wm(struct drm_device *dev)
		1426	{
		1427	struct drm_i915_private *dev_priv = dev->dev_private;
		1428	struct drm_crtc *crtc;
		1429	int srwm = 1;
		1430	int cursor_sr = 16;
		1431
		1432	/* Calc sr entries for one plane configs */
		1433	crtc = single_enabled_crtc(dev);
		1434	if (crtc) {
		1435	/* self-refresh has much higher latency */
		1436	static const int sr_latency_ns = 12000;
		1437	int clock = crtc->mode.clock;
		1438	int htotal = crtc->mode.htotal;
		1439	int hdisplay = crtc->mode.hdisplay;
		1440	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1441	unsigned long line_time_us;
		1442	int entries;
		1443
		1444	line_time_us = ((htotal * 1000) / clock);
		1445
		1446	/* Use ns/us then divide to preserve precision */
		1447	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1448	pixel_size * hdisplay;
		1449	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1450	srwm = I965_FIFO_SIZE - entries;
		1451	if (srwm < 0)
		1452	srwm = 1;
		1453	srwm &= 0x1ff;
		1454	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1455	entries, srwm);
		1456
		1457	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1458	pixel_size * 64;
		1459	entries = DIV_ROUND_UP(entries,
		1460	i965_cursor_wm_info.cacheline_size);
		1461	cursor_sr = i965_cursor_wm_info.fifo_size -
		1462	(entries + i965_cursor_wm_info.guard_size);
		1463
		1464	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1465	cursor_sr = i965_cursor_wm_info.max_wm;
		1466
		1467	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1468	"cursor %d\n", srwm, cursor_sr);
		1469
		1470	if (IS_CRESTLINE(dev))
		1471	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
		1472	} else {
		1473	/* Turn off self refresh if both pipes are enabled */
		1474	if (IS_CRESTLINE(dev))
		1475	I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
		1476	& ~FW_BLC_SELF_EN);
		1477	}
		1478
		1479	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1480	srwm);
		1481
		1482	/* 965 has limitations... */
		1483	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) \|
		1484	(8 << 16) \| (8 << 8) \| (8 << 0));
		1485	I915_WRITE(DSPFW2, (8 << 8) \| (8 << 0));
		1486	/* update cursor SR watermark */
		1487	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1488	}
		1489
		1490	static void i9xx_update_wm(struct drm_device *dev)
		1491	{
		1492	struct drm_i915_private *dev_priv = dev->dev_private;
		1493	const struct intel_watermark_params *wm_info;
		1494	uint32_t fwater_lo;
		1495	uint32_t fwater_hi;
		1496	int cwm, srwm = 1;
		1497	int fifo_size;
		1498	int planea_wm, planeb_wm;
		1499	struct drm_crtc crtc, enabled = NULL;
		1500
		1501	if (IS_I945GM(dev))
		1502	wm_info = &i945_wm_info;
		1503	else if (!IS_GEN2(dev))
		1504	wm_info = &i915_wm_info;
		1505	else
		1506	wm_info = &i855_wm_info;
		1507
		1508	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1509	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1510	if (intel_crtc_active(crtc)) {
		1511	int cpp = crtc->fb->bits_per_pixel / 8;
		1512	if (IS_GEN2(dev))
		1513	cpp = 4;
		1514
3031	serge	1515	planea_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1516	wm_info, fifo_size, cpp,
3031	serge	1517	latency_ns);
		1518	enabled = crtc;
		1519	} else
		1520	planea_wm = fifo_size - wm_info->guard_size;
		1521
		1522	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1523	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1524	if (intel_crtc_active(crtc)) {
		1525	int cpp = crtc->fb->bits_per_pixel / 8;
		1526	if (IS_GEN2(dev))
		1527	cpp = 4;
		1528
3031	serge	1529	planeb_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1530	wm_info, fifo_size, cpp,
3031	serge	1531	latency_ns);
		1532	if (enabled == NULL)
		1533	enabled = crtc;
		1534	else
		1535	enabled = NULL;
		1536	} else
		1537	planeb_wm = fifo_size - wm_info->guard_size;
		1538
		1539	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1540
		1541	/*
		1542	* Overlay gets an aggressive default since video jitter is bad.
		1543	*/
		1544	cwm = 2;
		1545
		1546	/* Play safe and disable self-refresh before adjusting watermarks. */
		1547	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1548	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK \| 0);
		1549	else if (IS_I915GM(dev))
		1550	I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
		1551
		1552	/* Calc sr entries for one plane configs */
		1553	if (HAS_FW_BLC(dev) && enabled) {
		1554	/* self-refresh has much higher latency */
		1555	static const int sr_latency_ns = 6000;
		1556	int clock = enabled->mode.clock;
		1557	int htotal = enabled->mode.htotal;
		1558	int hdisplay = enabled->mode.hdisplay;
		1559	int pixel_size = enabled->fb->bits_per_pixel / 8;
		1560	unsigned long line_time_us;
		1561	int entries;
		1562
		1563	line_time_us = (htotal * 1000) / clock;
		1564
		1565	/* Use ns/us then divide to preserve precision */
		1566	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1567	pixel_size * hdisplay;
		1568	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1569	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1570	srwm = wm_info->fifo_size - entries;
		1571	if (srwm < 0)
		1572	srwm = 1;
		1573
		1574	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1575	I915_WRITE(FW_BLC_SELF,
		1576	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1577	else if (IS_I915GM(dev))
		1578	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1579	}
		1580
		1581	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1582	planea_wm, planeb_wm, cwm, srwm);
		1583
		1584	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1585	fwater_hi = (cwm & 0x1f);
		1586
		1587	/* Set request length to 8 cachelines per fetch */
		1588	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1589	fwater_hi = fwater_hi \| (1 << 8);
		1590
		1591	I915_WRITE(FW_BLC, fwater_lo);
		1592	I915_WRITE(FW_BLC2, fwater_hi);
		1593
		1594	if (HAS_FW_BLC(dev)) {
		1595	if (enabled) {
		1596	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1597	I915_WRITE(FW_BLC_SELF,
		1598	FW_BLC_SELF_EN_MASK \| FW_BLC_SELF_EN);
		1599	else if (IS_I915GM(dev))
		1600	I915_WRITE(INSTPM, I915_READ(INSTPM) \| INSTPM_SELF_EN);
		1601	DRM_DEBUG_KMS("memory self refresh enabled\n");
		1602	} else
		1603	DRM_DEBUG_KMS("memory self refresh disabled\n");
		1604	}
		1605	}
		1606
		1607	static void i830_update_wm(struct drm_device *dev)
		1608	{
		1609	struct drm_i915_private *dev_priv = dev->dev_private;
		1610	struct drm_crtc *crtc;
		1611	uint32_t fwater_lo;
		1612	int planea_wm;
		1613
		1614	crtc = single_enabled_crtc(dev);
		1615	if (crtc == NULL)
		1616	return;
		1617
		1618	planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
		1619	dev_priv->display.get_fifo_size(dev, 0),
3243	Serge	1620	4, latency_ns);
3031	serge	1621	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1622	fwater_lo \|= (3<<8) \| planea_wm;
		1623
		1624	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1625
		1626	I915_WRITE(FW_BLC, fwater_lo);
		1627	}
		1628
		1629	#define ILK_LP0_PLANE_LATENCY 700
		1630	#define ILK_LP0_CURSOR_LATENCY 1300
		1631
		1632	/*
		1633	* Check the wm result.
		1634	*
		1635	* If any calculated watermark values is larger than the maximum value that
		1636	* can be programmed into the associated watermark register, that watermark
		1637	* must be disabled.
		1638	*/
		1639	static bool ironlake_check_srwm(struct drm_device *dev, int level,
		1640	int fbc_wm, int display_wm, int cursor_wm,
		1641	const struct intel_watermark_params *display,
		1642	const struct intel_watermark_params *cursor)
		1643	{
		1644	struct drm_i915_private *dev_priv = dev->dev_private;
		1645
		1646	DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
		1647	" cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
		1648
		1649	if (fbc_wm > SNB_FBC_MAX_SRWM) {
		1650	DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
		1651	fbc_wm, SNB_FBC_MAX_SRWM, level);
		1652
		1653	/* fbc has it's own way to disable FBC WM */
		1654	I915_WRITE(DISP_ARB_CTL,
		1655	I915_READ(DISP_ARB_CTL) \| DISP_FBC_WM_DIS);
		1656	return false;
		1657	}
		1658
		1659	if (display_wm > display->max_wm) {
		1660	DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
		1661	display_wm, SNB_DISPLAY_MAX_SRWM, level);
		1662	return false;
		1663	}
		1664
		1665	if (cursor_wm > cursor->max_wm) {
		1666	DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
		1667	cursor_wm, SNB_CURSOR_MAX_SRWM, level);
		1668	return false;
		1669	}
		1670
		1671	if (!(fbc_wm \|\| display_wm \|\| cursor_wm)) {
		1672	DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
		1673	return false;
		1674	}
		1675
		1676	return true;
		1677	}
		1678
		1679	/*
		1680	* Compute watermark values of WM[1-3],
		1681	*/
		1682	static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
		1683	int latency_ns,
		1684	const struct intel_watermark_params *display,
		1685	const struct intel_watermark_params *cursor,
		1686	int fbc_wm, int display_wm, int *cursor_wm)
		1687	{
		1688	struct drm_crtc *crtc;
		1689	unsigned long line_time_us;
		1690	int hdisplay, htotal, pixel_size, clock;
		1691	int line_count, line_size;
		1692	int small, large;
		1693	int entries;
		1694
		1695	if (!latency_ns) {
		1696	fbc_wm = display_wm = *cursor_wm = 0;
		1697	return false;
		1698	}
		1699
		1700	crtc = intel_get_crtc_for_plane(dev, plane);
		1701	hdisplay = crtc->mode.hdisplay;
		1702	htotal = crtc->mode.htotal;
		1703	clock = crtc->mode.clock;
		1704	pixel_size = crtc->fb->bits_per_pixel / 8;
		1705
		1706	line_time_us = (htotal * 1000) / clock;
		1707	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1708	line_size = hdisplay * pixel_size;
		1709
		1710	/* Use the minimum of the small and large buffer method for primary */
		1711	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1712	large = line_count * line_size;
		1713
		1714	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1715	*display_wm = entries + display->guard_size;
		1716
		1717	/*
		1718	* Spec says:
		1719	* FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
		1720	*/
		1721	fbc_wm = DIV_ROUND_UP(display_wm * 64, line_size) + 2;
		1722
		1723	/* calculate the self-refresh watermark for display cursor */
		1724	entries = line_count * pixel_size * 64;
		1725	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1726	*cursor_wm = entries + cursor->guard_size;
		1727
		1728	return ironlake_check_srwm(dev, level,
		1729	fbc_wm, display_wm, *cursor_wm,
		1730	display, cursor);
		1731	}
		1732
		1733	static void ironlake_update_wm(struct drm_device *dev)
		1734	{
		1735	struct drm_i915_private *dev_priv = dev->dev_private;
		1736	int fbc_wm, plane_wm, cursor_wm;
		1737	unsigned int enabled;
		1738
		1739	enabled = 0;
3746	Serge	1740	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1741	&ironlake_display_wm_info,
		1742	ILK_LP0_PLANE_LATENCY,
		1743	&ironlake_cursor_wm_info,
		1744	ILK_LP0_CURSOR_LATENCY,
		1745	&plane_wm, &cursor_wm)) {
		1746	I915_WRITE(WM0_PIPEA_ILK,
		1747	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1748	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1749	" plane %d, " "cursor: %d\n",
		1750	plane_wm, cursor_wm);
3746	Serge	1751	enabled \|= 1 << PIPE_A;
3031	serge	1752	}
		1753
3746	Serge	1754	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1755	&ironlake_display_wm_info,
		1756	ILK_LP0_PLANE_LATENCY,
		1757	&ironlake_cursor_wm_info,
		1758	ILK_LP0_CURSOR_LATENCY,
		1759	&plane_wm, &cursor_wm)) {
		1760	I915_WRITE(WM0_PIPEB_ILK,
		1761	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1762	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1763	" plane %d, cursor: %d\n",
		1764	plane_wm, cursor_wm);
3746	Serge	1765	enabled \|= 1 << PIPE_B;
3031	serge	1766	}
		1767
		1768	/*
		1769	* Calculate and update the self-refresh watermark only when one
		1770	* display plane is used.
		1771	*/
		1772	I915_WRITE(WM3_LP_ILK, 0);
		1773	I915_WRITE(WM2_LP_ILK, 0);
		1774	I915_WRITE(WM1_LP_ILK, 0);
		1775
		1776	if (!single_plane_enabled(enabled))
		1777	return;
		1778	enabled = ffs(enabled) - 1;
		1779
		1780	/* WM1 */
		1781	if (!ironlake_compute_srwm(dev, 1, enabled,
		1782	ILK_READ_WM1_LATENCY() * 500,
		1783	&ironlake_display_srwm_info,
		1784	&ironlake_cursor_srwm_info,
		1785	&fbc_wm, &plane_wm, &cursor_wm))
		1786	return;
		1787
		1788	I915_WRITE(WM1_LP_ILK,
		1789	WM1_LP_SR_EN \|
		1790	(ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1791	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1792	(plane_wm << WM1_LP_SR_SHIFT) \|
		1793	cursor_wm);
		1794
		1795	/* WM2 */
		1796	if (!ironlake_compute_srwm(dev, 2, enabled,
		1797	ILK_READ_WM2_LATENCY() * 500,
		1798	&ironlake_display_srwm_info,
		1799	&ironlake_cursor_srwm_info,
		1800	&fbc_wm, &plane_wm, &cursor_wm))
		1801	return;
		1802
		1803	I915_WRITE(WM2_LP_ILK,
		1804	WM2_LP_EN \|
		1805	(ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1806	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1807	(plane_wm << WM1_LP_SR_SHIFT) \|
		1808	cursor_wm);
		1809
		1810	/*
		1811	* WM3 is unsupported on ILK, probably because we don't have latency
		1812	* data for that power state
		1813	*/
		1814	}
		1815
		1816	static void sandybridge_update_wm(struct drm_device *dev)
		1817	{
		1818	struct drm_i915_private *dev_priv = dev->dev_private;
		1819	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		1820	u32 val;
		1821	int fbc_wm, plane_wm, cursor_wm;
		1822	unsigned int enabled;
		1823
		1824	enabled = 0;
3746	Serge	1825	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1826	&sandybridge_display_wm_info, latency,
		1827	&sandybridge_cursor_wm_info, latency,
		1828	&plane_wm, &cursor_wm)) {
		1829	val = I915_READ(WM0_PIPEA_ILK);
		1830	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1831	I915_WRITE(WM0_PIPEA_ILK, val \|
		1832	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1833	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1834	" plane %d, " "cursor: %d\n",
		1835	plane_wm, cursor_wm);
3746	Serge	1836	enabled \|= 1 << PIPE_A;
3031	serge	1837	}
		1838
3746	Serge	1839	if (g4x_compute_wm0(dev, PIPE_B,
3031	serge	1840	&sandybridge_display_wm_info, latency,
		1841	&sandybridge_cursor_wm_info, latency,
		1842	&plane_wm, &cursor_wm)) {
		1843	val = I915_READ(WM0_PIPEB_ILK);
		1844	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1845	I915_WRITE(WM0_PIPEB_ILK, val \|
		1846	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1847	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1848	" plane %d, cursor: %d\n",
		1849	plane_wm, cursor_wm);
3746	Serge	1850	enabled \|= 1 << PIPE_B;
3031	serge	1851	}
		1852
3243	Serge	1853	/*
		1854	* Calculate and update the self-refresh watermark only when one
		1855	* display plane is used.
		1856	*
		1857	* SNB support 3 levels of watermark.
		1858	*
		1859	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1860	* and disabled in the descending order
		1861	*
		1862	*/
		1863	I915_WRITE(WM3_LP_ILK, 0);
		1864	I915_WRITE(WM2_LP_ILK, 0);
		1865	I915_WRITE(WM1_LP_ILK, 0);
		1866
		1867	if (!single_plane_enabled(enabled) \|\|
		1868	dev_priv->sprite_scaling_enabled)
		1869	return;
		1870	enabled = ffs(enabled) - 1;
		1871
		1872	/* WM1 */
		1873	if (!ironlake_compute_srwm(dev, 1, enabled,
		1874	SNB_READ_WM1_LATENCY() * 500,
		1875	&sandybridge_display_srwm_info,
		1876	&sandybridge_cursor_srwm_info,
		1877	&fbc_wm, &plane_wm, &cursor_wm))
		1878	return;
		1879
		1880	I915_WRITE(WM1_LP_ILK,
		1881	WM1_LP_SR_EN \|
		1882	(SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1883	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1884	(plane_wm << WM1_LP_SR_SHIFT) \|
		1885	cursor_wm);
		1886
		1887	/* WM2 */
		1888	if (!ironlake_compute_srwm(dev, 2, enabled,
		1889	SNB_READ_WM2_LATENCY() * 500,
		1890	&sandybridge_display_srwm_info,
		1891	&sandybridge_cursor_srwm_info,
		1892	&fbc_wm, &plane_wm, &cursor_wm))
		1893	return;
		1894
		1895	I915_WRITE(WM2_LP_ILK,
		1896	WM2_LP_EN \|
		1897	(SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1898	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1899	(plane_wm << WM1_LP_SR_SHIFT) \|
		1900	cursor_wm);
		1901
		1902	/* WM3 */
		1903	if (!ironlake_compute_srwm(dev, 3, enabled,
		1904	SNB_READ_WM3_LATENCY() * 500,
		1905	&sandybridge_display_srwm_info,
		1906	&sandybridge_cursor_srwm_info,
		1907	&fbc_wm, &plane_wm, &cursor_wm))
		1908	return;
		1909
		1910	I915_WRITE(WM3_LP_ILK,
		1911	WM3_LP_EN \|
		1912	(SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1913	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1914	(plane_wm << WM1_LP_SR_SHIFT) \|
		1915	cursor_wm);
		1916	}
		1917
		1918	static void ivybridge_update_wm(struct drm_device *dev)
		1919	{
		1920	struct drm_i915_private *dev_priv = dev->dev_private;
		1921	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		1922	u32 val;
		1923	int fbc_wm, plane_wm, cursor_wm;
		1924	int ignore_fbc_wm, ignore_plane_wm, ignore_cursor_wm;
		1925	unsigned int enabled;
		1926
		1927	enabled = 0;
3746	Serge	1928	if (g4x_compute_wm0(dev, PIPE_A,
3031	serge	1929	&sandybridge_display_wm_info, latency,
		1930	&sandybridge_cursor_wm_info, latency,
		1931	&plane_wm, &cursor_wm)) {
3243	Serge	1932	val = I915_READ(WM0_PIPEA_ILK);
		1933	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1934	I915_WRITE(WM0_PIPEA_ILK, val \|
		1935	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1936	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1937	" plane %d, " "cursor: %d\n",
		1938	plane_wm, cursor_wm);
3746	Serge	1939	enabled \|= 1 << PIPE_A;
3243	Serge	1940	}
		1941
3746	Serge	1942	if (g4x_compute_wm0(dev, PIPE_B,
3243	Serge	1943	&sandybridge_display_wm_info, latency,
		1944	&sandybridge_cursor_wm_info, latency,
		1945	&plane_wm, &cursor_wm)) {
		1946	val = I915_READ(WM0_PIPEB_ILK);
		1947	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1948	I915_WRITE(WM0_PIPEB_ILK, val \|
		1949	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1950	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1951	" plane %d, cursor: %d\n",
		1952	plane_wm, cursor_wm);
3746	Serge	1953	enabled \|= 1 << PIPE_B;
3243	Serge	1954	}
		1955
3746	Serge	1956	if (g4x_compute_wm0(dev, PIPE_C,
3243	Serge	1957	&sandybridge_display_wm_info, latency,
		1958	&sandybridge_cursor_wm_info, latency,
		1959	&plane_wm, &cursor_wm)) {
3031	serge	1960	val = I915_READ(WM0_PIPEC_IVB);
		1961	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1962	I915_WRITE(WM0_PIPEC_IVB, val \|
		1963	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1964	DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
		1965	" plane %d, cursor: %d\n",
		1966	plane_wm, cursor_wm);
3746	Serge	1967	enabled \|= 1 << PIPE_C;
3031	serge	1968	}
		1969
		1970	/*
		1971	* Calculate and update the self-refresh watermark only when one
		1972	* display plane is used.
		1973	*
		1974	* SNB support 3 levels of watermark.
		1975	*
		1976	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1977	* and disabled in the descending order
		1978	*
		1979	*/
		1980	I915_WRITE(WM3_LP_ILK, 0);
		1981	I915_WRITE(WM2_LP_ILK, 0);
		1982	I915_WRITE(WM1_LP_ILK, 0);
		1983
		1984	if (!single_plane_enabled(enabled) \|\|
		1985	dev_priv->sprite_scaling_enabled)
		1986	return;
		1987	enabled = ffs(enabled) - 1;
		1988
		1989	/* WM1 */
		1990	if (!ironlake_compute_srwm(dev, 1, enabled,
		1991	SNB_READ_WM1_LATENCY() * 500,
		1992	&sandybridge_display_srwm_info,
		1993	&sandybridge_cursor_srwm_info,
		1994	&fbc_wm, &plane_wm, &cursor_wm))
		1995	return;
		1996
		1997	I915_WRITE(WM1_LP_ILK,
		1998	WM1_LP_SR_EN \|
		1999	(SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2000	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2001	(plane_wm << WM1_LP_SR_SHIFT) \|
		2002	cursor_wm);
		2003
		2004	/* WM2 */
		2005	if (!ironlake_compute_srwm(dev, 2, enabled,
		2006	SNB_READ_WM2_LATENCY() * 500,
		2007	&sandybridge_display_srwm_info,
		2008	&sandybridge_cursor_srwm_info,
		2009	&fbc_wm, &plane_wm, &cursor_wm))
		2010	return;
		2011
		2012	I915_WRITE(WM2_LP_ILK,
		2013	WM2_LP_EN \|
		2014	(SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2015	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2016	(plane_wm << WM1_LP_SR_SHIFT) \|
		2017	cursor_wm);
		2018
3243	Serge	2019	/* WM3, note we have to correct the cursor latency */
3031	serge	2020	if (!ironlake_compute_srwm(dev, 3, enabled,
		2021	SNB_READ_WM3_LATENCY() * 500,
		2022	&sandybridge_display_srwm_info,
		2023	&sandybridge_cursor_srwm_info,
3243	Serge	2024	&fbc_wm, &plane_wm, &ignore_cursor_wm) \|\|
		2025	!ironlake_compute_srwm(dev, 3, enabled,
		2026	2 * SNB_READ_WM3_LATENCY() * 500,
		2027	&sandybridge_display_srwm_info,
		2028	&sandybridge_cursor_srwm_info,
		2029	&ignore_fbc_wm, &ignore_plane_wm, &cursor_wm))
3031	serge	2030	return;
		2031
		2032	I915_WRITE(WM3_LP_ILK,
		2033	WM3_LP_EN \|
		2034	(SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2035	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2036	(plane_wm << WM1_LP_SR_SHIFT) \|
		2037	cursor_wm);
		2038	}
		2039
		2040	static void
		2041	haswell_update_linetime_wm(struct drm_device *dev, int pipe,
		2042	struct drm_display_mode *mode)
		2043	{
		2044	struct drm_i915_private *dev_priv = dev->dev_private;
		2045	u32 temp;
		2046
		2047	temp = I915_READ(PIPE_WM_LINETIME(pipe));
		2048	temp &= ~PIPE_WM_LINETIME_MASK;
		2049
		2050	/* The WM are computed with base on how long it takes to fill a single
		2051	* row at the given clock rate, multiplied by 8.
		2052	* */
		2053	temp \|= PIPE_WM_LINETIME_TIME(
		2054	((mode->crtc_hdisplay * 1000) / mode->clock) * 8);
		2055
		2056	/* IPS watermarks are only used by pipe A, and are ignored by
		2057	* pipes B and C. They are calculated similarly to the common
		2058	* linetime values, except that we are using CD clock frequency
		2059	* in MHz instead of pixel rate for the division.
		2060	*
		2061	* This is a placeholder for the IPS watermark calculation code.
		2062	*/
		2063
		2064	I915_WRITE(PIPE_WM_LINETIME(pipe), temp);
		2065	}
		2066
		2067	static bool
		2068	sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
		2069	uint32_t sprite_width, int pixel_size,
		2070	const struct intel_watermark_params *display,
		2071	int display_latency_ns, int *sprite_wm)
		2072	{
		2073	struct drm_crtc *crtc;
		2074	int clock;
		2075	int entries, tlb_miss;
		2076
		2077	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	2078	if (!intel_crtc_active(crtc)) {
3031	serge	2079	*sprite_wm = display->guard_size;
		2080	return false;
		2081	}
		2082
		2083	clock = crtc->mode.clock;
		2084
		2085	/* Use the small buffer method to calculate the sprite watermark */
		2086	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		2087	tlb_miss = display->fifo_size*display->cacheline_size -
		2088	sprite_width * 8;
		2089	if (tlb_miss > 0)
		2090	entries += tlb_miss;
		2091	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		2092	*sprite_wm = entries + display->guard_size;
		2093	if (*sprite_wm > (int)display->max_wm)
		2094	*sprite_wm = display->max_wm;
		2095
		2096	return true;
		2097	}
		2098
		2099	static bool
		2100	sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
		2101	uint32_t sprite_width, int pixel_size,
		2102	const struct intel_watermark_params *display,
		2103	int latency_ns, int *sprite_wm)
		2104	{
		2105	struct drm_crtc *crtc;
		2106	unsigned long line_time_us;
		2107	int clock;
		2108	int line_count, line_size;
		2109	int small, large;
		2110	int entries;
		2111
		2112	if (!latency_ns) {
		2113	*sprite_wm = 0;
		2114	return false;
		2115	}
		2116
		2117	crtc = intel_get_crtc_for_plane(dev, plane);
		2118	clock = crtc->mode.clock;
		2119	if (!clock) {
		2120	*sprite_wm = 0;
		2121	return false;
		2122	}
		2123
		2124	line_time_us = (sprite_width * 1000) / clock;
		2125	if (!line_time_us) {
		2126	*sprite_wm = 0;
		2127	return false;
		2128	}
		2129
		2130	line_count = (latency_ns / line_time_us + 1000) / 1000;
		2131	line_size = sprite_width * pixel_size;
		2132
		2133	/* Use the minimum of the small and large buffer method for primary */
		2134	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		2135	large = line_count * line_size;
		2136
		2137	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		2138	*sprite_wm = entries + display->guard_size;
		2139
		2140	return *sprite_wm > 0x3ff ? false : true;
		2141	}
		2142
		2143	static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
		2144	uint32_t sprite_width, int pixel_size)
		2145	{
		2146	struct drm_i915_private *dev_priv = dev->dev_private;
		2147	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		2148	u32 val;
		2149	int sprite_wm, reg;
		2150	int ret;
		2151
		2152	switch (pipe) {
		2153	case 0:
		2154	reg = WM0_PIPEA_ILK;
		2155	break;
		2156	case 1:
		2157	reg = WM0_PIPEB_ILK;
		2158	break;
		2159	case 2:
		2160	reg = WM0_PIPEC_IVB;
		2161	break;
		2162	default:
		2163	return; /* bad pipe */
		2164	}
		2165
		2166	ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
		2167	&sandybridge_display_wm_info,
		2168	latency, &sprite_wm);
		2169	if (!ret) {
		2170	DRM_DEBUG_KMS("failed to compute sprite wm for pipe %d\n",
		2171	pipe);
		2172	return;
		2173	}
		2174
		2175	val = I915_READ(reg);
		2176	val &= ~WM0_PIPE_SPRITE_MASK;
		2177	I915_WRITE(reg, val \| (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
		2178	DRM_DEBUG_KMS("sprite watermarks For pipe %d - %d\n", pipe, sprite_wm);
		2179
		2180
		2181	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2182	pixel_size,
		2183	&sandybridge_display_srwm_info,
		2184	SNB_READ_WM1_LATENCY() * 500,
		2185	&sprite_wm);
		2186	if (!ret) {
		2187	DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %d\n",
		2188	pipe);
		2189	return;
		2190	}
		2191	I915_WRITE(WM1S_LP_ILK, sprite_wm);
		2192
		2193	/* Only IVB has two more LP watermarks for sprite */
		2194	if (!IS_IVYBRIDGE(dev))
		2195	return;
		2196
		2197	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2198	pixel_size,
		2199	&sandybridge_display_srwm_info,
		2200	SNB_READ_WM2_LATENCY() * 500,
		2201	&sprite_wm);
		2202	if (!ret) {
		2203	DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %d\n",
		2204	pipe);
		2205	return;
		2206	}
		2207	I915_WRITE(WM2S_LP_IVB, sprite_wm);
		2208
		2209	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2210	pixel_size,
		2211	&sandybridge_display_srwm_info,
		2212	SNB_READ_WM3_LATENCY() * 500,
		2213	&sprite_wm);
		2214	if (!ret) {
		2215	DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %d\n",
		2216	pipe);
		2217	return;
		2218	}
		2219	I915_WRITE(WM3S_LP_IVB, sprite_wm);
		2220	}
		2221
		2222	/**
		2223	* intel_update_watermarks - update FIFO watermark values based on current modes
		2224	*
		2225	* Calculate watermark values for the various WM regs based on current mode
		2226	* and plane configuration.
		2227	*
		2228	* There are several cases to deal with here:
		2229	* - normal (i.e. non-self-refresh)
		2230	* - self-refresh (SR) mode
		2231	* - lines are large relative to FIFO size (buffer can hold up to 2)
		2232	* - lines are small relative to FIFO size (buffer can hold more than 2
		2233	* lines), so need to account for TLB latency
		2234	*
		2235	* The normal calculation is:
		2236	* watermark = dotclock * bytes per pixel * latency
		2237	* where latency is platform & configuration dependent (we assume pessimal
		2238	* values here).
		2239	*
		2240	* The SR calculation is:
		2241	* watermark = (trunc(latency/line time)+1) * surface width *
		2242	* bytes per pixel
		2243	* where
		2244	* line time = htotal / dotclock
		2245	* surface width = hdisplay for normal plane and 64 for cursor
		2246	* and latency is assumed to be high, as above.
		2247	*
		2248	* The final value programmed to the register should always be rounded up,
		2249	* and include an extra 2 entries to account for clock crossings.
		2250	*
		2251	* We don't use the sprite, so we can ignore that. And on Crestline we have
		2252	* to set the non-SR watermarks to 8.
		2253	*/
		2254	void intel_update_watermarks(struct drm_device *dev)
		2255	{
		2256	struct drm_i915_private *dev_priv = dev->dev_private;
		2257
		2258	if (dev_priv->display.update_wm)
		2259	dev_priv->display.update_wm(dev);
		2260	}
		2261
		2262	void intel_update_linetime_watermarks(struct drm_device *dev,
		2263	int pipe, struct drm_display_mode *mode)
		2264	{
		2265	struct drm_i915_private *dev_priv = dev->dev_private;
		2266
		2267	if (dev_priv->display.update_linetime_wm)
		2268	dev_priv->display.update_linetime_wm(dev, pipe, mode);
		2269	}
		2270
		2271	void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
		2272	uint32_t sprite_width, int pixel_size)
		2273	{
		2274	struct drm_i915_private *dev_priv = dev->dev_private;
		2275
		2276	if (dev_priv->display.update_sprite_wm)
		2277	dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
		2278	pixel_size);
		2279	}
		2280
		2281	static struct drm_i915_gem_object *
		2282	intel_alloc_context_page(struct drm_device *dev)
		2283	{
		2284	struct drm_i915_gem_object *ctx;
		2285	int ret;
		2286
		2287	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2288
		2289	ctx = i915_gem_alloc_object(dev, 4096);
		2290	if (!ctx) {
		2291	DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
		2292	return NULL;
		2293	}
		2294
		2295	ret = i915_gem_object_pin(ctx, 4096, true, false);
		2296	if (ret) {
		2297	DRM_ERROR("failed to pin power context: %d\n", ret);
		2298	goto err_unref;
		2299	}
		2300
		2301	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
		2302	if (ret) {
		2303	DRM_ERROR("failed to set-domain on power context: %d\n", ret);
		2304	goto err_unpin;
		2305	}
		2306
		2307	return ctx;
		2308
		2309	err_unpin:
		2310	i915_gem_object_unpin(ctx);
		2311	err_unref:
		2312	drm_gem_object_unreference(&ctx->base);
		2313	return NULL;
		2314	}
		2315
		2316	/**
		2317	* Lock protecting IPS related data structures
		2318	*/
		2319	DEFINE_SPINLOCK(mchdev_lock);
		2320
		2321	/* Global for IPS driver to get at the current i915 device. Protected by
		2322	* mchdev_lock. */
		2323	static struct drm_i915_private *i915_mch_dev;
		2324
		2325	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		2326	{
		2327	struct drm_i915_private *dev_priv = dev->dev_private;
		2328	u16 rgvswctl;
		2329
		2330	assert_spin_locked(&mchdev_lock);
		2331
		2332	rgvswctl = I915_READ16(MEMSWCTL);
		2333	if (rgvswctl & MEMCTL_CMD_STS) {
		2334	DRM_DEBUG("gpu busy, RCS change rejected\n");
		2335	return false; /* still busy with another command */
		2336	}
		2337
		2338	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		2339	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		2340	I915_WRITE16(MEMSWCTL, rgvswctl);
		2341	POSTING_READ16(MEMSWCTL);
		2342
		2343	rgvswctl \|= MEMCTL_CMD_STS;
		2344	I915_WRITE16(MEMSWCTL, rgvswctl);
		2345
		2346	return true;
		2347	}
		2348
		2349	static void ironlake_enable_drps(struct drm_device *dev)
		2350	{
		2351	struct drm_i915_private *dev_priv = dev->dev_private;
		2352	u32 rgvmodectl = I915_READ(MEMMODECTL);
		2353	u8 fmax, fmin, fstart, vstart;
		2354
		2355	spin_lock_irq(&mchdev_lock);
		2356
		2357	/* Enable temp reporting */
		2358	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		2359	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		2360
		2361	/* 100ms RC evaluation intervals */
		2362	I915_WRITE(RCUPEI, 100000);
		2363	I915_WRITE(RCDNEI, 100000);
		2364
		2365	/* Set max/min thresholds to 90ms and 80ms respectively */
		2366	I915_WRITE(RCBMAXAVG, 90000);
		2367	I915_WRITE(RCBMINAVG, 80000);
		2368
		2369	I915_WRITE(MEMIHYST, 1);
		2370
		2371	/* Set up min, max, and cur for interrupt handling */
		2372	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		2373	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		2374	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		2375	MEMMODE_FSTART_SHIFT;
		2376
		2377	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
		2378	PXVFREQ_PX_SHIFT;
		2379
		2380	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		2381	dev_priv->ips.fstart = fstart;
		2382
		2383	dev_priv->ips.max_delay = fstart;
		2384	dev_priv->ips.min_delay = fmin;
		2385	dev_priv->ips.cur_delay = fstart;
		2386
		2387	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		2388	fmax, fmin, fstart);
		2389
		2390	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		2391
		2392	/*
		2393	* Interrupts will be enabled in ironlake_irq_postinstall
		2394	*/
		2395
		2396	I915_WRITE(VIDSTART, vstart);
		2397	POSTING_READ(VIDSTART);
		2398
		2399	rgvmodectl \|= MEMMODE_SWMODE_EN;
		2400	I915_WRITE(MEMMODECTL, rgvmodectl);
		2401
		2402	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		2403	DRM_ERROR("stuck trying to change perf mode\n");
		2404	mdelay(1);
		2405
		2406	ironlake_set_drps(dev, fstart);
		2407
		2408	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
		2409	I915_READ(0x112e0);
		2410	dev_priv->ips.last_time1 = jiffies_to_msecs(GetTimerTicks());
		2411	dev_priv->ips.last_count2 = I915_READ(0x112f4);
3482	Serge	2412	getrawmonotonic(&dev_priv->ips.last_time2);
3031	serge	2413
		2414	spin_unlock_irq(&mchdev_lock);
		2415	}
		2416
		2417	static void ironlake_disable_drps(struct drm_device *dev)
		2418	{
		2419	struct drm_i915_private *dev_priv = dev->dev_private;
		2420	u16 rgvswctl;
		2421
		2422	spin_lock_irq(&mchdev_lock);
		2423
		2424	rgvswctl = I915_READ16(MEMSWCTL);
		2425
		2426	/* Ack interrupts, disable EFC interrupt */
		2427	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		2428	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		2429	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		2430	I915_WRITE(DEIIR, DE_PCU_EVENT);
		2431	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		2432
		2433	/* Go back to the starting frequency */
		2434	ironlake_set_drps(dev, dev_priv->ips.fstart);
		2435	mdelay(1);
		2436	rgvswctl \|= MEMCTL_CMD_STS;
		2437	I915_WRITE(MEMSWCTL, rgvswctl);
		2438	mdelay(1);
		2439
		2440	spin_unlock_irq(&mchdev_lock);
		2441	}
		2442
		2443	/* There's a funny hw issue where the hw returns all 0 when reading from
		2444	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		2445	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		2446	* all limits and the gpu stuck at whatever frequency it is at atm).
		2447	*/
		2448	static u32 gen6_rps_limits(struct drm_i915_private dev_priv, u8 val)
		2449	{
		2450	u32 limits;
		2451
		2452	limits = 0;
		2453
		2454	if (*val >= dev_priv->rps.max_delay)
		2455	*val = dev_priv->rps.max_delay;
		2456	limits \|= dev_priv->rps.max_delay << 24;
		2457
		2458	/* Only set the down limit when we've reached the lowest level to avoid
		2459	* getting more interrupts, otherwise leave this clear. This prevents a
		2460	* race in the hw when coming out of rc6: There's a tiny window where
		2461	* the hw runs at the minimal clock before selecting the desired
		2462	* frequency, if the down threshold expires in that window we will not
		2463	* receive a down interrupt. */
		2464	if (*val <= dev_priv->rps.min_delay) {
		2465	*val = dev_priv->rps.min_delay;
		2466	limits \|= dev_priv->rps.min_delay << 16;
		2467	}
		2468
		2469	return limits;
		2470	}
		2471
		2472	void gen6_set_rps(struct drm_device *dev, u8 val)
		2473	{
		2474	struct drm_i915_private *dev_priv = dev->dev_private;
		2475	u32 limits = gen6_rps_limits(dev_priv, &val);
		2476
3243	Serge	2477	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2478	WARN_ON(val > dev_priv->rps.max_delay);
		2479	WARN_ON(val < dev_priv->rps.min_delay);
		2480
		2481	if (val == dev_priv->rps.cur_delay)
		2482	return;
		2483
3746	Serge	2484	if (IS_HASWELL(dev))
		2485	I915_WRITE(GEN6_RPNSWREQ,
		2486	HSW_FREQUENCY(val));
		2487	else
3031	serge	2488	I915_WRITE(GEN6_RPNSWREQ,
		2489	GEN6_FREQUENCY(val) \|
		2490	GEN6_OFFSET(0) \|
		2491	GEN6_AGGRESSIVE_TURBO);
		2492
		2493	/* Make sure we continue to get interrupts
		2494	* until we hit the minimum or maximum frequencies.
		2495	*/
		2496	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
		2497
		2498	POSTING_READ(GEN6_RPNSWREQ);
		2499
		2500	dev_priv->rps.cur_delay = val;
		2501
		2502	trace_intel_gpu_freq_change(val * 50);
		2503	}
		2504
		2505	static void gen6_disable_rps(struct drm_device *dev)
		2506	{
		2507	struct drm_i915_private *dev_priv = dev->dev_private;
		2508
		2509	I915_WRITE(GEN6_RC_CONTROL, 0);
		2510	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		2511	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
		2512	I915_WRITE(GEN6_PMIER, 0);
		2513	/* Complete PM interrupt masking here doesn't race with the rps work
		2514	* item again unmasking PM interrupts because that is using a different
		2515	* register (PMIMR) to mask PM interrupts. The only risk is in leaving
		2516	* stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
		2517
		2518	spin_lock_irq(&dev_priv->rps.lock);
		2519	dev_priv->rps.pm_iir = 0;
		2520	spin_unlock_irq(&dev_priv->rps.lock);
		2521
		2522	I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
		2523	}
		2524
		2525	int intel_enable_rc6(const struct drm_device *dev)
		2526	{
		2527	/* Respect the kernel parameter if it is set */
		2528	if (i915_enable_rc6 >= 0)
		2529	return i915_enable_rc6;
		2530
3120	serge	2531	/* Disable RC6 on Ironlake */
		2532	if (INTEL_INFO(dev)->gen == 5)
		2533	return 0;
3031	serge	2534
		2535	if (IS_HASWELL(dev)) {
		2536	DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
		2537	return INTEL_RC6_ENABLE;
		2538	}
		2539
		2540	/* snb/ivb have more than one rc6 state. */
		2541	if (INTEL_INFO(dev)->gen == 6) {
		2542	DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
		2543	return INTEL_RC6_ENABLE;
		2544	}
		2545
		2546	DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
		2547	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
		2548	}
		2549
		2550	static void gen6_enable_rps(struct drm_device *dev)
		2551	{
		2552	struct drm_i915_private *dev_priv = dev->dev_private;
		2553	struct intel_ring_buffer *ring;
		2554	u32 rp_state_cap;
		2555	u32 gt_perf_status;
3243	Serge	2556	u32 rc6vids, pcu_mbox, rc6_mask = 0;
3031	serge	2557	u32 gtfifodbg;
		2558	int rc6_mode;
3243	Serge	2559	int i, ret;
3031	serge	2560
3243	Serge	2561	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2562
		2563	/* Here begins a magic sequence of register writes to enable
		2564	* auto-downclocking.
		2565	*
		2566	* Perhaps there might be some value in exposing these to
		2567	* userspace...
		2568	*/
		2569	I915_WRITE(GEN6_RC_STATE, 0);
		2570
		2571	/* Clear the DBG now so we don't confuse earlier errors */
		2572	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		2573	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		2574	I915_WRITE(GTFIFODBG, gtfifodbg);
		2575	}
		2576
		2577	gen6_gt_force_wake_get(dev_priv);
		2578
		2579	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		2580	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
		2581
3746	Serge	2582	/* In units of 50MHz */
		2583	dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff;
3031	serge	2584	dev_priv->rps.min_delay = (rp_state_cap & 0xff0000) >> 16;
		2585	dev_priv->rps.cur_delay = 0;
		2586
		2587	/* disable the counters and set deterministic thresholds */
		2588	I915_WRITE(GEN6_RC_CONTROL, 0);
		2589
		2590	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		2591	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		2592	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		2593	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		2594	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		2595
		2596	for_each_ring(ring, dev_priv, i)
		2597	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		2598
		2599	I915_WRITE(GEN6_RC_SLEEP, 0);
		2600	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
		2601	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	2602	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	2603	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		2604
		2605	/* Check if we are enabling RC6 */
		2606	rc6_mode = intel_enable_rc6(dev_priv->dev);
		2607	if (rc6_mode & INTEL_RC6_ENABLE)
		2608	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		2609
		2610	/* We don't use those on Haswell */
		2611	if (!IS_HASWELL(dev)) {
		2612	if (rc6_mode & INTEL_RC6p_ENABLE)
		2613	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		2614
		2615	if (rc6_mode & INTEL_RC6pp_ENABLE)
		2616	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		2617	}
		2618
		2619	DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
		2620	(rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		2621	(rc6_mask & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		2622	(rc6_mask & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
		2623
		2624	I915_WRITE(GEN6_RC_CONTROL,
		2625	rc6_mask \|
		2626	GEN6_RC_CTL_EI_MODE(1) \|
		2627	GEN6_RC_CTL_HW_ENABLE);
		2628
3746	Serge	2629	if (IS_HASWELL(dev)) {
		2630	I915_WRITE(GEN6_RPNSWREQ,
		2631	HSW_FREQUENCY(10));
		2632	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		2633	HSW_FREQUENCY(12));
		2634	} else {
3031	serge	2635	I915_WRITE(GEN6_RPNSWREQ,
		2636	GEN6_FREQUENCY(10) \|
		2637	GEN6_OFFSET(0) \|
		2638	GEN6_AGGRESSIVE_TURBO);
		2639	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		2640	GEN6_FREQUENCY(12));
3746	Serge	2641	}
3031	serge	2642
		2643	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
		2644	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
		2645	dev_priv->rps.max_delay << 24 \|
		2646	dev_priv->rps.min_delay << 16);
		2647
		2648	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		2649	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		2650	I915_WRITE(GEN6_RP_UP_EI, 66000);
		2651	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		2652
		2653	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		2654	I915_WRITE(GEN6_RP_CONTROL,
		2655	GEN6_RP_MEDIA_TURBO \|
		2656	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		2657	GEN6_RP_MEDIA_IS_GFX \|
		2658	GEN6_RP_ENABLE \|
		2659	GEN6_RP_UP_BUSY_AVG \|
		2660	(IS_HASWELL(dev) ? GEN7_RP_DOWN_IDLE_AVG : GEN6_RP_DOWN_IDLE_CONT));
		2661
3243	Serge	2662	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
		2663	if (!ret) {
		2664	pcu_mbox = 0;
		2665	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
3746	Serge	2666	if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
		2667	DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
		2668	(dev_priv->rps.max_delay & 0xff) * 50,
		2669	(pcu_mbox & 0xff) * 50);
		2670	dev_priv->rps.hw_max = pcu_mbox & 0xff;
3031	serge	2671	}
3243	Serge	2672	} else {
		2673	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		2674	}
3031	serge	2675
		2676	gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
		2677
		2678	/* requires MSI enabled */
		2679	I915_WRITE(GEN6_PMIER, GEN6_PM_DEFERRED_EVENTS);
		2680	spin_lock_irq(&dev_priv->rps.lock);
		2681	WARN_ON(dev_priv->rps.pm_iir != 0);
		2682	I915_WRITE(GEN6_PMIMR, 0);
		2683	spin_unlock_irq(&dev_priv->rps.lock);
		2684	/* enable all PM interrupts */
		2685	I915_WRITE(GEN6_PMINTRMSK, 0);
		2686
3243	Serge	2687	rc6vids = 0;
		2688	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		2689	if (IS_GEN6(dev) && ret) {
		2690	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		2691	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		2692	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		2693	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		2694	rc6vids &= 0xffff00;
		2695	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		2696	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		2697	if (ret)
		2698	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		2699	}
		2700
3031	serge	2701	gen6_gt_force_wake_put(dev_priv);
		2702	}
		2703
		2704	static void gen6_update_ring_freq(struct drm_device *dev)
		2705	{
		2706	struct drm_i915_private *dev_priv = dev->dev_private;
		2707	int min_freq = 15;
3746	Serge	2708	unsigned int gpu_freq;
		2709	unsigned int max_ia_freq, min_ring_freq;
3031	serge	2710	int scaling_factor = 180;
		2711
3243	Serge	2712	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2713
		2714	max_ia_freq = cpufreq_quick_get_max(0);
		2715	/*
		2716	* Default to measured freq if none found, PCU will ensure we don't go
		2717	* over
		2718	*/
		2719	if (!max_ia_freq)
		2720	max_ia_freq = tsc_khz;
		2721
		2722	/* Convert from kHz to MHz */
		2723	max_ia_freq /= 1000;
		2724
3746	Serge	2725	min_ring_freq = I915_READ(MCHBAR_MIRROR_BASE_SNB + DCLK);
		2726	/* convert DDR frequency from units of 133.3MHz to bandwidth */
		2727	min_ring_freq = (2 * 4 * min_ring_freq + 2) / 3;
		2728
3031	serge	2729	/*
		2730	* For each potential GPU frequency, load a ring frequency we'd like
		2731	* to use for memory access. We do this by specifying the IA frequency
		2732	* the PCU should use as a reference to determine the ring frequency.
		2733	*/
		2734	for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
		2735	gpu_freq--) {
		2736	int diff = dev_priv->rps.max_delay - gpu_freq;
3746	Serge	2737	unsigned int ia_freq = 0, ring_freq = 0;
3031	serge	2738
3746	Serge	2739	if (IS_HASWELL(dev)) {
		2740	ring_freq = (gpu_freq * 5 + 3) / 4;
		2741	ring_freq = max(min_ring_freq, ring_freq);
		2742	/* leave ia_freq as the default, chosen by cpufreq */
		2743	} else {
		2744	/* On older processors, there is no separate ring
		2745	* clock domain, so in order to boost the bandwidth
		2746	* of the ring, we need to upclock the CPU (ia_freq).
		2747	*
		2748	* For GPU frequencies less than 750MHz,
		2749	* just use the lowest ring freq.
3031	serge	2750	*/
		2751	if (gpu_freq < min_freq)
		2752	ia_freq = 800;
		2753	else
		2754	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		2755	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3746	Serge	2756	}
3031	serge	2757
3243	Serge	2758	sandybridge_pcode_write(dev_priv,
		2759	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
3746	Serge	2760	ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT \|
		2761	ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT \|
		2762	gpu_freq);
3031	serge	2763	}
		2764	}
		2765
		2766	void ironlake_teardown_rc6(struct drm_device *dev)
		2767	{
		2768	struct drm_i915_private *dev_priv = dev->dev_private;
		2769
3243	Serge	2770	if (dev_priv->ips.renderctx) {
		2771	i915_gem_object_unpin(dev_priv->ips.renderctx);
		2772	drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
		2773	dev_priv->ips.renderctx = NULL;
3031	serge	2774	}
		2775
3243	Serge	2776	if (dev_priv->ips.pwrctx) {
		2777	i915_gem_object_unpin(dev_priv->ips.pwrctx);
		2778	drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
		2779	dev_priv->ips.pwrctx = NULL;
3031	serge	2780	}
		2781	}
		2782
		2783	static void ironlake_disable_rc6(struct drm_device *dev)
		2784	{
		2785	struct drm_i915_private *dev_priv = dev->dev_private;
		2786
		2787	if (I915_READ(PWRCTXA)) {
		2788	/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
		2789	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) \| RCX_SW_EXIT);
		2790	wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
		2791	50);
		2792
		2793	I915_WRITE(PWRCTXA, 0);
		2794	POSTING_READ(PWRCTXA);
		2795
		2796	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		2797	POSTING_READ(RSTDBYCTL);
		2798	}
		2799	}
		2800
		2801	static int ironlake_setup_rc6(struct drm_device *dev)
		2802	{
		2803	struct drm_i915_private *dev_priv = dev->dev_private;
		2804
3243	Serge	2805	if (dev_priv->ips.renderctx == NULL)
		2806	dev_priv->ips.renderctx = intel_alloc_context_page(dev);
		2807	if (!dev_priv->ips.renderctx)
3031	serge	2808	return -ENOMEM;
		2809
3243	Serge	2810	if (dev_priv->ips.pwrctx == NULL)
		2811	dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
		2812	if (!dev_priv->ips.pwrctx) {
3031	serge	2813	ironlake_teardown_rc6(dev);
		2814	return -ENOMEM;
		2815	}
		2816
		2817	return 0;
		2818	}
		2819
		2820	static void ironlake_enable_rc6(struct drm_device *dev)
		2821	{
		2822	struct drm_i915_private *dev_priv = dev->dev_private;
		2823	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
3243	Serge	2824	bool was_interruptible;
3031	serge	2825	int ret;
		2826
		2827	/* rc6 disabled by default due to repeated reports of hanging during
		2828	* boot and resume.
		2829	*/
		2830	if (!intel_enable_rc6(dev))
		2831	return;
		2832
		2833	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2834
		2835	ret = ironlake_setup_rc6(dev);
		2836	if (ret)
		2837	return;
		2838
3243	Serge	2839	was_interruptible = dev_priv->mm.interruptible;
		2840	dev_priv->mm.interruptible = false;
		2841
3031	serge	2842	/*
		2843	* GPU can automatically power down the render unit if given a page
		2844	* to save state.
		2845	*/
		2846	ret = intel_ring_begin(ring, 6);
		2847	if (ret) {
		2848	ironlake_teardown_rc6(dev);
3243	Serge	2849	dev_priv->mm.interruptible = was_interruptible;
3031	serge	2850	return;
		2851	}
		2852
		2853	intel_ring_emit(ring, MI_SUSPEND_FLUSH \| MI_SUSPEND_FLUSH_EN);
		2854	intel_ring_emit(ring, MI_SET_CONTEXT);
3243	Serge	2855	intel_ring_emit(ring, dev_priv->ips.renderctx->gtt_offset \|
3031	serge	2856	MI_MM_SPACE_GTT \|
		2857	MI_SAVE_EXT_STATE_EN \|
		2858	MI_RESTORE_EXT_STATE_EN \|
		2859	MI_RESTORE_INHIBIT);
		2860	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
		2861	intel_ring_emit(ring, MI_NOOP);
		2862	intel_ring_emit(ring, MI_FLUSH);
		2863	intel_ring_advance(ring);
		2864
		2865	/*
		2866	* Wait for the command parser to advance past MI_SET_CONTEXT. The HW
		2867	* does an implicit flush, combined with MI_FLUSH above, it should be
		2868	* safe to assume that renderctx is valid
		2869	*/
3243	Serge	2870	ret = intel_ring_idle(ring);
		2871	dev_priv->mm.interruptible = was_interruptible;
3031	serge	2872	if (ret) {
3746	Serge	2873	DRM_ERROR("failed to enable ironlake power savings\n");
3031	serge	2874	ironlake_teardown_rc6(dev);
		2875	return;
		2876	}
		2877
3243	Serge	2878	I915_WRITE(PWRCTXA, dev_priv->ips.pwrctx->gtt_offset \| PWRCTX_EN);
3031	serge	2879	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		2880	}
		2881
		2882	static unsigned long intel_pxfreq(u32 vidfreq)
		2883	{
		2884	unsigned long freq;
		2885	int div = (vidfreq & 0x3f0000) >> 16;
		2886	int post = (vidfreq & 0x3000) >> 12;
		2887	int pre = (vidfreq & 0x7);
		2888
		2889	if (!pre)
		2890	return 0;
		2891
		2892	freq = ((div * 133333) / ((1<
		2893
		2894	return freq;
		2895	}
		2896
		2897	static const struct cparams {
		2898	u16 i;
		2899	u16 t;
		2900	u16 m;
		2901	u16 c;
		2902	} cparams[] = {
		2903	{ 1, 1333, 301, 28664 },
		2904	{ 1, 1066, 294, 24460 },
		2905	{ 1, 800, 294, 25192 },
		2906	{ 0, 1333, 276, 27605 },
		2907	{ 0, 1066, 276, 27605 },
		2908	{ 0, 800, 231, 23784 },
		2909	};
		2910
		2911	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		2912	{
		2913	u64 total_count, diff, ret;
		2914	u32 count1, count2, count3, m = 0, c = 0;
		2915	unsigned long now = jiffies_to_msecs(GetTimerTicks()), diff1;
		2916	int i;
		2917
		2918	assert_spin_locked(&mchdev_lock);
		2919
		2920	diff1 = now - dev_priv->ips.last_time1;
		2921
		2922	/* Prevent division-by-zero if we are asking too fast.
		2923	* Also, we don't get interesting results if we are polling
		2924	* faster than once in 10ms, so just return the saved value
		2925	* in such cases.
		2926	*/
		2927	if (diff1 <= 10)
		2928	return dev_priv->ips.chipset_power;
		2929
		2930	count1 = I915_READ(DMIEC);
		2931	count2 = I915_READ(DDREC);
		2932	count3 = I915_READ(CSIEC);
		2933
		2934	total_count = count1 + count2 + count3;
		2935
		2936	/* FIXME: handle per-counter overflow */
		2937	if (total_count < dev_priv->ips.last_count1) {
		2938	diff = ~0UL - dev_priv->ips.last_count1;
		2939	diff += total_count;
		2940	} else {
		2941	diff = total_count - dev_priv->ips.last_count1;
		2942	}
		2943
		2944	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		2945	if (cparams[i].i == dev_priv->ips.c_m &&
		2946	cparams[i].t == dev_priv->ips.r_t) {
		2947	m = cparams[i].m;
		2948	c = cparams[i].c;
		2949	break;
		2950	}
		2951	}
		2952
		2953	diff = div_u64(diff, diff1);
		2954	ret = ((m * diff) + c);
		2955	ret = div_u64(ret, 10);
		2956
		2957	dev_priv->ips.last_count1 = total_count;
		2958	dev_priv->ips.last_time1 = now;
		2959
		2960	dev_priv->ips.chipset_power = ret;
		2961
		2962	return ret;
		2963	}
		2964
		2965	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		2966	{
		2967	unsigned long val;
		2968
		2969	if (dev_priv->info->gen != 5)
		2970	return 0;
		2971
		2972	spin_lock_irq(&mchdev_lock);
		2973
		2974	val = __i915_chipset_val(dev_priv);
		2975
		2976	spin_unlock_irq(&mchdev_lock);
		2977
		2978	return val;
		2979	}
		2980
		2981	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		2982	{
		2983	unsigned long m, x, b;
		2984	u32 tsfs;
		2985
		2986	tsfs = I915_READ(TSFS);
		2987
		2988	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		2989	x = I915_READ8(TR1);
		2990
		2991	b = tsfs & TSFS_INTR_MASK;
		2992
		2993	return ((m * x) / 127) - b;
		2994	}
		2995
		2996	static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		2997	{
		2998	static const struct v_table {
		2999	u16 vd; /* in .1 mil */
		3000	u16 vm; /* in .1 mil */
		3001	} v_table[] = {
		3002	{ 0, 0, },
		3003	{ 375, 0, },
		3004	{ 500, 0, },
		3005	{ 625, 0, },
		3006	{ 750, 0, },
		3007	{ 875, 0, },
		3008	{ 1000, 0, },
		3009	{ 1125, 0, },
		3010	{ 4125, 3000, },
		3011	{ 4125, 3000, },
		3012	{ 4125, 3000, },
		3013	{ 4125, 3000, },
		3014	{ 4125, 3000, },
		3015	{ 4125, 3000, },
		3016	{ 4125, 3000, },
		3017	{ 4125, 3000, },
		3018	{ 4125, 3000, },
		3019	{ 4125, 3000, },
		3020	{ 4125, 3000, },
		3021	{ 4125, 3000, },
		3022	{ 4125, 3000, },
		3023	{ 4125, 3000, },
		3024	{ 4125, 3000, },
		3025	{ 4125, 3000, },
		3026	{ 4125, 3000, },
		3027	{ 4125, 3000, },
		3028	{ 4125, 3000, },
		3029	{ 4125, 3000, },
		3030	{ 4125, 3000, },
		3031	{ 4125, 3000, },
		3032	{ 4125, 3000, },
		3033	{ 4125, 3000, },
		3034	{ 4250, 3125, },
		3035	{ 4375, 3250, },
		3036	{ 4500, 3375, },
		3037	{ 4625, 3500, },
		3038	{ 4750, 3625, },
		3039	{ 4875, 3750, },
		3040	{ 5000, 3875, },
		3041	{ 5125, 4000, },
		3042	{ 5250, 4125, },
		3043	{ 5375, 4250, },
		3044	{ 5500, 4375, },
		3045	{ 5625, 4500, },
		3046	{ 5750, 4625, },
		3047	{ 5875, 4750, },
		3048	{ 6000, 4875, },
		3049	{ 6125, 5000, },
		3050	{ 6250, 5125, },
		3051	{ 6375, 5250, },
		3052	{ 6500, 5375, },
		3053	{ 6625, 5500, },
		3054	{ 6750, 5625, },
		3055	{ 6875, 5750, },
		3056	{ 7000, 5875, },
		3057	{ 7125, 6000, },
		3058	{ 7250, 6125, },
		3059	{ 7375, 6250, },
		3060	{ 7500, 6375, },
		3061	{ 7625, 6500, },
		3062	{ 7750, 6625, },
		3063	{ 7875, 6750, },
		3064	{ 8000, 6875, },
		3065	{ 8125, 7000, },
		3066	{ 8250, 7125, },
		3067	{ 8375, 7250, },
		3068	{ 8500, 7375, },
		3069	{ 8625, 7500, },
		3070	{ 8750, 7625, },
		3071	{ 8875, 7750, },
		3072	{ 9000, 7875, },
		3073	{ 9125, 8000, },
		3074	{ 9250, 8125, },
		3075	{ 9375, 8250, },
		3076	{ 9500, 8375, },
		3077	{ 9625, 8500, },
		3078	{ 9750, 8625, },
		3079	{ 9875, 8750, },
		3080	{ 10000, 8875, },
		3081	{ 10125, 9000, },
		3082	{ 10250, 9125, },
		3083	{ 10375, 9250, },
		3084	{ 10500, 9375, },
		3085	{ 10625, 9500, },
		3086	{ 10750, 9625, },
		3087	{ 10875, 9750, },
		3088	{ 11000, 9875, },
		3089	{ 11125, 10000, },
		3090	{ 11250, 10125, },
		3091	{ 11375, 10250, },
		3092	{ 11500, 10375, },
		3093	{ 11625, 10500, },
		3094	{ 11750, 10625, },
		3095	{ 11875, 10750, },
		3096	{ 12000, 10875, },
		3097	{ 12125, 11000, },
		3098	{ 12250, 11125, },
		3099	{ 12375, 11250, },
		3100	{ 12500, 11375, },
		3101	{ 12625, 11500, },
		3102	{ 12750, 11625, },
		3103	{ 12875, 11750, },
		3104	{ 13000, 11875, },
		3105	{ 13125, 12000, },
		3106	{ 13250, 12125, },
		3107	{ 13375, 12250, },
		3108	{ 13500, 12375, },
		3109	{ 13625, 12500, },
		3110	{ 13750, 12625, },
		3111	{ 13875, 12750, },
		3112	{ 14000, 12875, },
		3113	{ 14125, 13000, },
		3114	{ 14250, 13125, },
		3115	{ 14375, 13250, },
		3116	{ 14500, 13375, },
		3117	{ 14625, 13500, },
		3118	{ 14750, 13625, },
		3119	{ 14875, 13750, },
		3120	{ 15000, 13875, },
		3121	{ 15125, 14000, },
		3122	{ 15250, 14125, },
		3123	{ 15375, 14250, },
		3124	{ 15500, 14375, },
		3125	{ 15625, 14500, },
		3126	{ 15750, 14625, },
		3127	{ 15875, 14750, },
		3128	{ 16000, 14875, },
		3129	{ 16125, 15000, },
		3130	};
		3131	if (dev_priv->info->is_mobile)
		3132	return v_table[pxvid].vm;
		3133	else
		3134	return v_table[pxvid].vd;
		3135	}
		3136
		3137	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		3138	{
		3139	struct timespec now, diff1;
		3140	u64 diff;
		3141	unsigned long diffms;
		3142	u32 count;
		3143
		3144	assert_spin_locked(&mchdev_lock);
		3145
		3146	getrawmonotonic(&now);
		3147	diff1 = timespec_sub(now, dev_priv->ips.last_time2);
		3148
		3149	/* Don't divide by 0 */
		3150	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
		3151	if (!diffms)
		3152	return;
		3153
		3154	count = I915_READ(GFXEC);
		3155
		3156	if (count < dev_priv->ips.last_count2) {
		3157	diff = ~0UL - dev_priv->ips.last_count2;
		3158	diff += count;
		3159	} else {
		3160	diff = count - dev_priv->ips.last_count2;
		3161	}
		3162
		3163	dev_priv->ips.last_count2 = count;
		3164	dev_priv->ips.last_time2 = now;
		3165
		3166	/* More magic constants... */
		3167	diff = diff * 1181;
		3168	diff = div_u64(diff, diffms * 10);
		3169	dev_priv->ips.gfx_power = diff;
		3170	}
		3171
		3172	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		3173	{
		3174	if (dev_priv->info->gen != 5)
		3175	return;
		3176
		3177	spin_lock_irq(&mchdev_lock);
		3178
		3179	__i915_update_gfx_val(dev_priv);
		3180
		3181	spin_unlock_irq(&mchdev_lock);
		3182	}
		3183
		3184	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		3185	{
		3186	unsigned long t, corr, state1, corr2, state2;
		3187	u32 pxvid, ext_v;
		3188
		3189	assert_spin_locked(&mchdev_lock);
		3190
		3191	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
		3192	pxvid = (pxvid >> 24) & 0x7f;
		3193	ext_v = pvid_to_extvid(dev_priv, pxvid);
		3194
		3195	state1 = ext_v;
		3196
		3197	t = i915_mch_val(dev_priv);
		3198
		3199	/* Revel in the empirically derived constants */
		3200
		3201	/* Correction factor in 1/100000 units */
		3202	if (t > 80)
		3203	corr = ((t * 2349) + 135940);
		3204	else if (t >= 50)
		3205	corr = ((t * 964) + 29317);
		3206	else /* < 50 */
		3207	corr = ((t * 301) + 1004);
		3208
		3209	corr = corr * ((150142 * state1) / 10000 - 78642);
		3210	corr /= 100000;
		3211	corr2 = (corr * dev_priv->ips.corr);
		3212
		3213	state2 = (corr2 * state1) / 10000;
		3214	state2 /= 100; /* convert to mW */
		3215
		3216	__i915_update_gfx_val(dev_priv);
		3217
		3218	return dev_priv->ips.gfx_power + state2;
		3219	}
		3220
		3221	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		3222	{
		3223	unsigned long val;
		3224
		3225	if (dev_priv->info->gen != 5)
		3226	return 0;
		3227
		3228	spin_lock_irq(&mchdev_lock);
		3229
		3230	val = __i915_gfx_val(dev_priv);
		3231
		3232	spin_unlock_irq(&mchdev_lock);
		3233
		3234	return val;
		3235	}
		3236
		3237	/**
		3238	* i915_read_mch_val - return value for IPS use
		3239	*
		3240	* Calculate and return a value for the IPS driver to use when deciding whether
		3241	* we have thermal and power headroom to increase CPU or GPU power budget.
		3242	*/
		3243	unsigned long i915_read_mch_val(void)
		3244	{
		3245	struct drm_i915_private *dev_priv;
		3246	unsigned long chipset_val, graphics_val, ret = 0;
		3247
		3248	spin_lock_irq(&mchdev_lock);
		3249	if (!i915_mch_dev)
		3250	goto out_unlock;
		3251	dev_priv = i915_mch_dev;
		3252
		3253	chipset_val = __i915_chipset_val(dev_priv);
		3254	graphics_val = __i915_gfx_val(dev_priv);
		3255
		3256	ret = chipset_val + graphics_val;
		3257
		3258	out_unlock:
		3259	spin_unlock_irq(&mchdev_lock);
		3260
		3261	return ret;
		3262	}
		3263	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		3264
		3265	/**
		3266	* i915_gpu_raise - raise GPU frequency limit
		3267	*
		3268	* Raise the limit; IPS indicates we have thermal headroom.
		3269	*/
		3270	bool i915_gpu_raise(void)
		3271	{
		3272	struct drm_i915_private *dev_priv;
		3273	bool ret = true;
		3274
		3275	spin_lock_irq(&mchdev_lock);
		3276	if (!i915_mch_dev) {
		3277	ret = false;
		3278	goto out_unlock;
		3279	}
		3280	dev_priv = i915_mch_dev;
		3281
		3282	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		3283	dev_priv->ips.max_delay--;
		3284
		3285	out_unlock:
		3286	spin_unlock_irq(&mchdev_lock);
		3287
		3288	return ret;
		3289	}
		3290	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		3291
		3292	/**
		3293	* i915_gpu_lower - lower GPU frequency limit
		3294	*
		3295	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		3296	* frequency maximum.
		3297	*/
		3298	bool i915_gpu_lower(void)
		3299	{
		3300	struct drm_i915_private *dev_priv;
		3301	bool ret = true;
		3302
		3303	spin_lock_irq(&mchdev_lock);
		3304	if (!i915_mch_dev) {
		3305	ret = false;
		3306	goto out_unlock;
		3307	}
		3308	dev_priv = i915_mch_dev;
		3309
		3310	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		3311	dev_priv->ips.max_delay++;
		3312
		3313	out_unlock:
		3314	spin_unlock_irq(&mchdev_lock);
		3315
		3316	return ret;
		3317	}
		3318	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		3319
		3320	/**
		3321	* i915_gpu_busy - indicate GPU business to IPS
		3322	*
		3323	* Tell the IPS driver whether or not the GPU is busy.
		3324	*/
		3325	bool i915_gpu_busy(void)
		3326	{
		3327	struct drm_i915_private *dev_priv;
		3328	struct intel_ring_buffer *ring;
		3329	bool ret = false;
		3330	int i;
		3331
		3332	spin_lock_irq(&mchdev_lock);
		3333	if (!i915_mch_dev)
		3334	goto out_unlock;
		3335	dev_priv = i915_mch_dev;
		3336
		3337	for_each_ring(ring, dev_priv, i)
		3338	ret \|= !list_empty(&ring->request_list);
		3339
		3340	out_unlock:
		3341	spin_unlock_irq(&mchdev_lock);
		3342
		3343	return ret;
		3344	}
		3345	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		3346
		3347	/**
		3348	* i915_gpu_turbo_disable - disable graphics turbo
		3349	*
		3350	* Disable graphics turbo by resetting the max frequency and setting the
		3351	* current frequency to the default.
		3352	*/
		3353	bool i915_gpu_turbo_disable(void)
		3354	{
		3355	struct drm_i915_private *dev_priv;
		3356	bool ret = true;
		3357
		3358	spin_lock_irq(&mchdev_lock);
		3359	if (!i915_mch_dev) {
		3360	ret = false;
		3361	goto out_unlock;
		3362	}
		3363	dev_priv = i915_mch_dev;
		3364
		3365	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		3366
		3367	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		3368	ret = false;
		3369
		3370	out_unlock:
		3371	spin_unlock_irq(&mchdev_lock);
		3372
		3373	return ret;
		3374	}
		3375	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		3376
		3377	/**
		3378	* Tells the intel_ips driver that the i915 driver is now loaded, if
		3379	* IPS got loaded first.
		3380	*
		3381	* This awkward dance is so that neither module has to depend on the
		3382	* other in order for IPS to do the appropriate communication of
		3383	* GPU turbo limits to i915.
		3384	*/
		3385	static void
		3386	ips_ping_for_i915_load(void)
		3387	{
		3388	void (*link)(void);
		3389
		3390	// link = symbol_get(ips_link_to_i915_driver);
		3391	// if (link) {
		3392	// link();
		3393	// symbol_put(ips_link_to_i915_driver);
		3394	// }
		3395	}
		3396
		3397	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		3398	{
		3399	/* We only register the i915 ips part with intel-ips once everything is
		3400	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		3401	spin_lock_irq(&mchdev_lock);
		3402	i915_mch_dev = dev_priv;
		3403	spin_unlock_irq(&mchdev_lock);
		3404
		3405	ips_ping_for_i915_load();
		3406	}
		3407
		3408	void intel_gpu_ips_teardown(void)
		3409	{
		3410	spin_lock_irq(&mchdev_lock);
		3411	i915_mch_dev = NULL;
		3412	spin_unlock_irq(&mchdev_lock);
		3413	}
		3414	static void intel_init_emon(struct drm_device *dev)
		3415	{
		3416	struct drm_i915_private *dev_priv = dev->dev_private;
		3417	u32 lcfuse;
		3418	u8 pxw[16];
		3419	int i;
		3420
		3421	/* Disable to program */
		3422	I915_WRITE(ECR, 0);
		3423	POSTING_READ(ECR);
		3424
		3425	/* Program energy weights for various events */
		3426	I915_WRITE(SDEW, 0x15040d00);
		3427	I915_WRITE(CSIEW0, 0x007f0000);
		3428	I915_WRITE(CSIEW1, 0x1e220004);
		3429	I915_WRITE(CSIEW2, 0x04000004);
		3430
		3431	for (i = 0; i < 5; i++)
		3432	I915_WRITE(PEW + (i * 4), 0);
		3433	for (i = 0; i < 3; i++)
		3434	I915_WRITE(DEW + (i * 4), 0);
		3435
		3436	/* Program P-state weights to account for frequency power adjustment */
		3437	for (i = 0; i < 16; i++) {
		3438	u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
		3439	unsigned long freq = intel_pxfreq(pxvidfreq);
		3440	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		3441	PXVFREQ_PX_SHIFT;
		3442	unsigned long val;
		3443
		3444	val = vid * vid;
		3445	val *= (freq / 1000);
		3446	val *= 255;
		3447	val /= (127127900);
		3448	if (val > 0xff)
		3449	DRM_ERROR("bad pxval: %ld\n", val);
		3450	pxw[i] = val;
		3451	}
		3452	/* Render standby states get 0 weight */
		3453	pxw[14] = 0;
		3454	pxw[15] = 0;
		3455
		3456	for (i = 0; i < 4; i++) {
		3457	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		3458	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
		3459	I915_WRITE(PXW + (i * 4), val);
		3460	}
		3461
		3462	/* Adjust magic regs to magic values (more experimental results) */
		3463	I915_WRITE(OGW0, 0);
		3464	I915_WRITE(OGW1, 0);
		3465	I915_WRITE(EG0, 0x00007f00);
		3466	I915_WRITE(EG1, 0x0000000e);
		3467	I915_WRITE(EG2, 0x000e0000);
		3468	I915_WRITE(EG3, 0x68000300);
		3469	I915_WRITE(EG4, 0x42000000);
		3470	I915_WRITE(EG5, 0x00140031);
		3471	I915_WRITE(EG6, 0);
		3472	I915_WRITE(EG7, 0);
		3473
		3474	for (i = 0; i < 8; i++)
		3475	I915_WRITE(PXWL + (i * 4), 0);
		3476
		3477	/* Enable PMON + select events */
		3478	I915_WRITE(ECR, 0x80000019);
		3479
		3480	lcfuse = I915_READ(LCFUSE02);
		3481
		3482	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		3483	}
		3484
		3485	void intel_disable_gt_powersave(struct drm_device *dev)
		3486	{
3243	Serge	3487	struct drm_i915_private *dev_priv = dev->dev_private;
		3488
3031	serge	3489	if (IS_IRONLAKE_M(dev)) {
		3490	ironlake_disable_drps(dev);
		3491	ironlake_disable_rc6(dev);
		3492	} else if (INTEL_INFO(dev)->gen >= 6 && !IS_VALLEYVIEW(dev)) {
3482	Serge	3493	// cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
		3494	mutex_lock(&dev_priv->rps.hw_lock);
3031	serge	3495	gen6_disable_rps(dev);
3480	Serge	3496	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	3497	}
		3498	}
		3499
3482	Serge	3500	static void intel_gen6_powersave_work(struct work_struct *work)
		3501	{
		3502	struct drm_i915_private *dev_priv =
		3503	container_of(work, struct drm_i915_private,
		3504	rps.delayed_resume_work.work);
		3505	struct drm_device *dev = dev_priv->dev;
		3506
		3507	ENTER();
		3508
		3509	mutex_lock(&dev_priv->rps.hw_lock);
		3510	gen6_enable_rps(dev);
		3511	gen6_update_ring_freq(dev);
		3512	mutex_unlock(&dev_priv->rps.hw_lock);
		3513
		3514	LEAVE();
		3515	}
		3516
3031	serge	3517	void intel_enable_gt_powersave(struct drm_device *dev)
		3518	{
3243	Serge	3519	struct drm_i915_private *dev_priv = dev->dev_private;
		3520
3031	serge	3521	if (IS_IRONLAKE_M(dev)) {
		3522	ironlake_enable_drps(dev);
		3523	ironlake_enable_rc6(dev);
		3524	intel_init_emon(dev);
		3525	} else if ((IS_GEN6(dev) \|\| IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) {
3243	Serge	3526	/*
		3527	* PCU communication is slow and this doesn't need to be
		3528	* done at any specific time, so do this out of our fast path
		3529	* to make resume and init faster.
		3530	*/
3482	Serge	3531	schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		3532	round_jiffies_up_relative(HZ));
3031	serge	3533	}
		3534	}
		3535
3243	Serge	3536	static void ibx_init_clock_gating(struct drm_device *dev)
		3537	{
		3538	struct drm_i915_private *dev_priv = dev->dev_private;
		3539
		3540	/*
		3541	* On Ibex Peak and Cougar Point, we need to disable clock
		3542	* gating for the panel power sequencer or it will fail to
		3543	* start up when no ports are active.
		3544	*/
		3545	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		3546	}
		3547
3031	serge	3548	static void ironlake_init_clock_gating(struct drm_device *dev)
		3549	{
		3550	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	3551	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	3552
		3553	/* Required for FBC */
3243	Serge	3554	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		3555	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		3556	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	3557
		3558	I915_WRITE(PCH_3DCGDIS0,
		3559	MARIUNIT_CLOCK_GATE_DISABLE \|
		3560	SVSMUNIT_CLOCK_GATE_DISABLE);
		3561	I915_WRITE(PCH_3DCGDIS1,
		3562	VFMUNIT_CLOCK_GATE_DISABLE);
		3563
		3564	/*
		3565	* According to the spec the following bits should be set in
		3566	* order to enable memory self-refresh
		3567	* The bit 22/21 of 0x42004
		3568	* The bit 5 of 0x42020
		3569	* The bit 15 of 0x45000
		3570	*/
		3571	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3572	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3573	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	3574	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	3575	I915_WRITE(DISP_ARB_CTL,
		3576	(I915_READ(DISP_ARB_CTL) \|
		3577	DISP_FBC_WM_DIS));
		3578	I915_WRITE(WM3_LP_ILK, 0);
		3579	I915_WRITE(WM2_LP_ILK, 0);
		3580	I915_WRITE(WM1_LP_ILK, 0);
		3581
		3582	/*
		3583	* Based on the document from hardware guys the following bits
		3584	* should be set unconditionally in order to enable FBC.
		3585	* The bit 22 of 0x42000
		3586	* The bit 22 of 0x42004
		3587	* The bit 7,8,9 of 0x42020.
		3588	*/
		3589	if (IS_IRONLAKE_M(dev)) {
		3590	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		3591	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		3592	ILK_FBCQ_DIS);
		3593	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3594	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3595	ILK_DPARB_GATE);
		3596	}
		3597
3243	Serge	3598	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		3599
3031	serge	3600	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3601	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3602	ILK_ELPIN_409_SELECT);
		3603	I915_WRITE(_3D_CHICKEN2,
		3604	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		3605	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	3606
		3607	/* WaDisableRenderCachePipelinedFlush */
		3608	I915_WRITE(CACHE_MODE_0,
		3609	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		3610
		3611	ibx_init_clock_gating(dev);
3031	serge	3612	}
		3613
3243	Serge	3614	static void cpt_init_clock_gating(struct drm_device *dev)
		3615	{
		3616	struct drm_i915_private *dev_priv = dev->dev_private;
		3617	int pipe;
3746	Serge	3618	uint32_t val;
3243	Serge	3619
		3620	/*
		3621	* On Ibex Peak and Cougar Point, we need to disable clock
		3622	* gating for the panel power sequencer or it will fail to
		3623	* start up when no ports are active.
		3624	*/
		3625	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		3626	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		3627	DPLS_EDP_PPS_FIX_DIS);
		3628	/* The below fixes the weird display corruption, a few pixels shifted
		3629	* downward, on (only) LVDS of some HP laptops with IVY.
		3630	*/
3746	Serge	3631	for_each_pipe(pipe) {
		3632	val = I915_READ(TRANS_CHICKEN2(pipe));
		3633	val \|= TRANS_CHICKEN2_TIMING_OVERRIDE;
		3634	val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		3635	if (dev_priv->fdi_rx_polarity_inverted)
		3636	val \|= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
		3637	val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
		3638	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
		3639	val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
		3640	I915_WRITE(TRANS_CHICKEN2(pipe), val);
		3641	}
3243	Serge	3642	/* WADP0ClockGatingDisable */
		3643	for_each_pipe(pipe) {
		3644	I915_WRITE(TRANS_CHICKEN1(pipe),
		3645	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		3646	}
		3647	}
		3648
3480	Serge	3649	static void gen6_check_mch_setup(struct drm_device *dev)
		3650	{
		3651	struct drm_i915_private *dev_priv = dev->dev_private;
		3652	uint32_t tmp;
		3653
		3654	tmp = I915_READ(MCH_SSKPD);
		3655	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) {
		3656	DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp);
		3657	DRM_INFO("This can cause pipe underruns and display issues.\n");
		3658	DRM_INFO("Please upgrade your BIOS to fix this.\n");
		3659	}
		3660	}
		3661
3031	serge	3662	static void gen6_init_clock_gating(struct drm_device *dev)
		3663	{
		3664	struct drm_i915_private *dev_priv = dev->dev_private;
		3665	int pipe;
3243	Serge	3666	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	3667
3243	Serge	3668	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	3669
		3670	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3671	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3672	ILK_ELPIN_409_SELECT);
		3673
3243	Serge	3674	/* WaDisableHiZPlanesWhenMSAAEnabled */
		3675	I915_WRITE(_3D_CHICKEN,
		3676	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		3677
		3678	/* WaSetupGtModeTdRowDispatch */
		3679	if (IS_SNB_GT1(dev))
		3680	I915_WRITE(GEN6_GT_MODE,
		3681	_MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
		3682
3031	serge	3683	I915_WRITE(WM3_LP_ILK, 0);
		3684	I915_WRITE(WM2_LP_ILK, 0);
		3685	I915_WRITE(WM1_LP_ILK, 0);
		3686
		3687	I915_WRITE(CACHE_MODE_0,
		3688	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		3689
		3690	I915_WRITE(GEN6_UCGCTL1,
		3691	I915_READ(GEN6_UCGCTL1) \|
		3692	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		3693	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		3694
		3695	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3696	* gating disable must be set. Failure to set it results in
		3697	* flickering pixels due to Z write ordering failures after
		3698	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3699	* Sanctuary and Tropics, and apparently anything else with
		3700	* alpha test or pixel discard.
		3701	*
		3702	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3703	* but we didn't debug actual testcases to find it out.
		3704	*
		3705	* Also apply WaDisableVDSUnitClockGating and
		3706	* WaDisableRCPBUnitClockGating.
		3707	*/
		3708	I915_WRITE(GEN6_UCGCTL2,
		3709	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		3710	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		3711	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3712
		3713	/* Bspec says we need to always set all mask bits. */
		3714	I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) \|
		3715	_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
		3716
		3717	/*
		3718	* According to the spec the following bits should be
		3719	* set in order to enable memory self-refresh and fbc:
		3720	* The bit21 and bit22 of 0x42000
		3721	* The bit21 and bit22 of 0x42004
		3722	* The bit5 and bit7 of 0x42020
		3723	* The bit14 of 0x70180
		3724	* The bit14 of 0x71180
		3725	*/
		3726	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		3727	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		3728	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		3729	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3730	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3731	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	3732	I915_WRITE(ILK_DSPCLK_GATE_D,
		3733	I915_READ(ILK_DSPCLK_GATE_D) \|
		3734	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		3735	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	3736
3243	Serge	3737	/* WaMbcDriverBootEnable */
3031	serge	3738	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3739	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3740
		3741	for_each_pipe(pipe) {
		3742	I915_WRITE(DSPCNTR(pipe),
		3743	I915_READ(DSPCNTR(pipe)) \|
		3744	DISPPLANE_TRICKLE_FEED_DISABLE);
		3745	intel_flush_display_plane(dev_priv, pipe);
		3746	}
		3747
		3748	/* The default value should be 0x200 according to docs, but the two
		3749	* platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
		3750	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
		3751	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
3243	Serge	3752
		3753	cpt_init_clock_gating(dev);
3480	Serge	3754
		3755	gen6_check_mch_setup(dev);
3031	serge	3756	}
		3757
		3758	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		3759	{
		3760	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		3761
		3762	reg &= ~GEN7_FF_SCHED_MASK;
		3763	reg \|= GEN7_FF_TS_SCHED_HW;
		3764	reg \|= GEN7_FF_VS_SCHED_HW;
		3765	reg \|= GEN7_FF_DS_SCHED_HW;
		3766
3480	Serge	3767	/* WaVSRefCountFullforceMissDisable */
		3768	if (IS_HASWELL(dev_priv->dev))
		3769	reg &= ~GEN7_FF_VS_REF_CNT_FFME;
		3770
3031	serge	3771	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		3772	}
		3773
3243	Serge	3774	static void lpt_init_clock_gating(struct drm_device *dev)
		3775	{
		3776	struct drm_i915_private *dev_priv = dev->dev_private;
		3777
		3778	/*
		3779	* TODO: this bit should only be enabled when really needed, then
		3780	* disabled when not needed anymore in order to save power.
		3781	*/
		3782	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
		3783	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		3784	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		3785	PCH_LP_PARTITION_LEVEL_DISABLE);
		3786	}
		3787
3031	serge	3788	static void haswell_init_clock_gating(struct drm_device *dev)
		3789	{
		3790	struct drm_i915_private *dev_priv = dev->dev_private;
		3791	int pipe;
		3792
		3793	I915_WRITE(WM3_LP_ILK, 0);
		3794	I915_WRITE(WM2_LP_ILK, 0);
		3795	I915_WRITE(WM1_LP_ILK, 0);
		3796
		3797	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3798	* This implements the WaDisableRCZUnitClockGating workaround.
		3799	*/
		3800	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
		3801
		3802	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3803	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3804	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3805
		3806	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3807	I915_WRITE(GEN7_L3CNTLREG1,
		3808	GEN7_WA_FOR_GEN7_L3_CONTROL);
		3809	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		3810	GEN7_WA_L3_CHICKEN_MODE);
		3811
		3812	/* This is required by WaCatErrorRejectionIssue */
		3813	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3814	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3815	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3816
		3817	for_each_pipe(pipe) {
		3818	I915_WRITE(DSPCNTR(pipe),
		3819	I915_READ(DSPCNTR(pipe)) \|
		3820	DISPPLANE_TRICKLE_FEED_DISABLE);
		3821	intel_flush_display_plane(dev_priv, pipe);
		3822	}
		3823
		3824	gen7_setup_fixed_func_scheduler(dev_priv);
		3825
		3826	/* WaDisable4x2SubspanOptimization */
		3827	I915_WRITE(CACHE_MODE_1,
		3828	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3829
3243	Serge	3830	/* WaMbcDriverBootEnable */
		3831	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3832	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3833
3746	Serge	3834	/* WaSwitchSolVfFArbitrationPriority */
		3835	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) \| HSW_ECOCHK_ARB_PRIO_SOL);
		3836
3031	serge	3837	/* XXX: This is a workaround for early silicon revisions and should be
		3838	* removed later.
		3839	*/
		3840	I915_WRITE(WM_DBG,
		3841	I915_READ(WM_DBG) \|
		3842	WM_DBG_DISALLOW_MULTIPLE_LP \|
		3843	WM_DBG_DISALLOW_SPRITE \|
		3844	WM_DBG_DISALLOW_MAXFIFO);
		3845
3243	Serge	3846	lpt_init_clock_gating(dev);
3031	serge	3847	}
		3848
		3849	static void ivybridge_init_clock_gating(struct drm_device *dev)
		3850	{
		3851	struct drm_i915_private *dev_priv = dev->dev_private;
		3852	int pipe;
		3853	uint32_t snpcr;
		3854
		3855	I915_WRITE(WM3_LP_ILK, 0);
		3856	I915_WRITE(WM2_LP_ILK, 0);
		3857	I915_WRITE(WM1_LP_ILK, 0);
		3858
3243	Serge	3859	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	3860
3243	Serge	3861	/* WaDisableEarlyCull */
		3862	I915_WRITE(_3D_CHICKEN3,
		3863	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		3864
		3865	/* WaDisableBackToBackFlipFix */
3031	serge	3866	I915_WRITE(IVB_CHICKEN3,
		3867	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3868	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3869
3243	Serge	3870	/* WaDisablePSDDualDispatchEnable */
		3871	if (IS_IVB_GT1(dev))
		3872	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		3873	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3874	else
		3875	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
		3876	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3877
3031	serge	3878	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3879	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3880	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3881
		3882	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3883	I915_WRITE(GEN7_L3CNTLREG1,
		3884	GEN7_WA_FOR_GEN7_L3_CONTROL);
		3885	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		3886	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	3887	if (IS_IVB_GT1(dev))
		3888	I915_WRITE(GEN7_ROW_CHICKEN2,
		3889	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		3890	else
		3891	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		3892	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3031	serge	3893
3243	Serge	3894
		3895	/* WaForceL3Serialization */
		3896	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3897	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3898
3031	serge	3899	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3900	* gating disable must be set. Failure to set it results in
		3901	* flickering pixels due to Z write ordering failures after
		3902	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3903	* Sanctuary and Tropics, and apparently anything else with
		3904	* alpha test or pixel discard.
		3905	*
		3906	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3907	* but we didn't debug actual testcases to find it out.
		3908	*
		3909	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3910	* This implements the WaDisableRCZUnitClockGating workaround.
		3911	*/
		3912	I915_WRITE(GEN6_UCGCTL2,
		3913	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		3914	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3915
		3916	/* This is required by WaCatErrorRejectionIssue */
		3917	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3918	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3919	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3920
		3921	for_each_pipe(pipe) {
		3922	I915_WRITE(DSPCNTR(pipe),
		3923	I915_READ(DSPCNTR(pipe)) \|
		3924	DISPPLANE_TRICKLE_FEED_DISABLE);
		3925	intel_flush_display_plane(dev_priv, pipe);
		3926	}
		3927
3243	Serge	3928	/* WaMbcDriverBootEnable */
3031	serge	3929	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3930	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3931
		3932	gen7_setup_fixed_func_scheduler(dev_priv);
		3933
		3934	/* WaDisable4x2SubspanOptimization */
		3935	I915_WRITE(CACHE_MODE_1,
		3936	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3937
		3938	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		3939	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		3940	snpcr \|= GEN6_MBC_SNPCR_MED;
		3941	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	3942
3746	Serge	3943	if (!HAS_PCH_NOP(dev))
3243	Serge	3944	cpt_init_clock_gating(dev);
3480	Serge	3945
		3946	gen6_check_mch_setup(dev);
3031	serge	3947	}
		3948
		3949	static void valleyview_init_clock_gating(struct drm_device *dev)
		3950	{
		3951	struct drm_i915_private *dev_priv = dev->dev_private;
		3952	int pipe;
		3953
		3954	I915_WRITE(WM3_LP_ILK, 0);
		3955	I915_WRITE(WM2_LP_ILK, 0);
		3956	I915_WRITE(WM1_LP_ILK, 0);
		3957
3243	Serge	3958	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	3959
3243	Serge	3960	/* WaDisableEarlyCull */
		3961	I915_WRITE(_3D_CHICKEN3,
		3962	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		3963
		3964	/* WaDisableBackToBackFlipFix */
3031	serge	3965	I915_WRITE(IVB_CHICKEN3,
		3966	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3967	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3968
3746	Serge	3969	/* WaDisablePSDDualDispatchEnable */
3243	Serge	3970	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
3746	Serge	3971	_MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP \|
		3972	GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
3243	Serge	3973
3031	serge	3974	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3975	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3976	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3977
		3978	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
3243	Serge	3979	I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) \| GEN7_L3AGDIS);
3031	serge	3980	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
		3981
3243	Serge	3982	/* WaForceL3Serialization */
		3983	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3984	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3985
		3986	/* WaDisableDopClockGating */
		3987	I915_WRITE(GEN7_ROW_CHICKEN2,
		3988	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		3989
		3990	/* WaForceL3Serialization */
		3991	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3992	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3993
3031	serge	3994	/* This is required by WaCatErrorRejectionIssue */
		3995	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3996	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3997	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3998
3243	Serge	3999	/* WaMbcDriverBootEnable */
3031	serge	4000	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		4001	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		4002
		4003
		4004	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		4005	* gating disable must be set. Failure to set it results in
		4006	* flickering pixels due to Z write ordering failures after
		4007	* some amount of runtime in the Mesa "fire" demo, and Unigine
		4008	* Sanctuary and Tropics, and apparently anything else with
		4009	* alpha test or pixel discard.
		4010	*
		4011	* According to the spec, bit 11 (RCCUNIT) must also be set,
		4012	* but we didn't debug actual testcases to find it out.
		4013	*
		4014	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		4015	* This implements the WaDisableRCZUnitClockGating workaround.
		4016	*
		4017	* Also apply WaDisableVDSUnitClockGating and
		4018	* WaDisableRCPBUnitClockGating.
		4019	*/
		4020	I915_WRITE(GEN6_UCGCTL2,
		4021	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		4022	GEN7_TDLUNIT_CLOCK_GATE_DISABLE \|
		4023	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		4024	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		4025	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		4026
		4027	I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
		4028
		4029	for_each_pipe(pipe) {
		4030	I915_WRITE(DSPCNTR(pipe),
		4031	I915_READ(DSPCNTR(pipe)) \|
		4032	DISPPLANE_TRICKLE_FEED_DISABLE);
		4033	intel_flush_display_plane(dev_priv, pipe);
		4034	}
		4035
		4036	I915_WRITE(CACHE_MODE_1,
		4037	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		4038
		4039	/*
3243	Serge	4040	* WaDisableVLVClockGating_VBIIssue
		4041	* Disable clock gating on th GCFG unit to prevent a delay
		4042	* in the reporting of vblank events.
		4043	*/
3746	Serge	4044	I915_WRITE(VLV_GUNIT_CLOCK_GATE, 0xffffffff);
		4045
		4046	/* Conservative clock gating settings for now */
		4047	I915_WRITE(0x9400, 0xffffffff);
		4048	I915_WRITE(0x9404, 0xffffffff);
		4049	I915_WRITE(0x9408, 0xffffffff);
		4050	I915_WRITE(0x940c, 0xffffffff);
		4051	I915_WRITE(0x9410, 0xffffffff);
		4052	I915_WRITE(0x9414, 0xffffffff);
		4053	I915_WRITE(0x9418, 0xffffffff);
3031	serge	4054	}
		4055
		4056	static void g4x_init_clock_gating(struct drm_device *dev)
		4057	{
		4058	struct drm_i915_private *dev_priv = dev->dev_private;
		4059	uint32_t dspclk_gate;
		4060
		4061	I915_WRITE(RENCLK_GATE_D1, 0);
		4062	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		4063	GS_UNIT_CLOCK_GATE_DISABLE \|
		4064	CL_UNIT_CLOCK_GATE_DISABLE);
		4065	I915_WRITE(RAMCLK_GATE_D, 0);
		4066	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		4067	OVRUNIT_CLOCK_GATE_DISABLE \|
		4068	OVCUNIT_CLOCK_GATE_DISABLE;
		4069	if (IS_GM45(dev))
		4070	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		4071	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	4072
		4073	/* WaDisableRenderCachePipelinedFlush */
		4074	I915_WRITE(CACHE_MODE_0,
		4075	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
3031	serge	4076	}
		4077
		4078	static void crestline_init_clock_gating(struct drm_device *dev)
		4079	{
		4080	struct drm_i915_private *dev_priv = dev->dev_private;
		4081
		4082	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		4083	I915_WRITE(RENCLK_GATE_D2, 0);
		4084	I915_WRITE(DSPCLK_GATE_D, 0);
		4085	I915_WRITE(RAMCLK_GATE_D, 0);
		4086	I915_WRITE16(DEUC, 0);
		4087	}
		4088
		4089	static void broadwater_init_clock_gating(struct drm_device *dev)
		4090	{
		4091	struct drm_i915_private *dev_priv = dev->dev_private;
		4092
		4093	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		4094	I965_RCC_CLOCK_GATE_DISABLE \|
		4095	I965_RCPB_CLOCK_GATE_DISABLE \|
		4096	I965_ISC_CLOCK_GATE_DISABLE \|
		4097	I965_FBC_CLOCK_GATE_DISABLE);
		4098	I915_WRITE(RENCLK_GATE_D2, 0);
		4099	}
		4100
		4101	static void gen3_init_clock_gating(struct drm_device *dev)
		4102	{
		4103	struct drm_i915_private *dev_priv = dev->dev_private;
		4104	u32 dstate = I915_READ(D_STATE);
		4105
		4106	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		4107	DSTATE_DOT_CLOCK_GATING;
		4108	I915_WRITE(D_STATE, dstate);
		4109
		4110	if (IS_PINEVIEW(dev))
		4111	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		4112
		4113	/* IIR "flip pending" means done if this bit is set */
		4114	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
		4115	}
		4116
		4117	static void i85x_init_clock_gating(struct drm_device *dev)
		4118	{
		4119	struct drm_i915_private *dev_priv = dev->dev_private;
		4120
		4121	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
		4122	}
		4123
		4124	static void i830_init_clock_gating(struct drm_device *dev)
		4125	{
		4126	struct drm_i915_private *dev_priv = dev->dev_private;
		4127
		4128	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
		4129	}
		4130
		4131	void intel_init_clock_gating(struct drm_device *dev)
		4132	{
		4133	struct drm_i915_private *dev_priv = dev->dev_private;
		4134
		4135	dev_priv->display.init_clock_gating(dev);
		4136	}
		4137
3746	Serge	4138	/**
		4139	* We should only use the power well if we explicitly asked the hardware to
		4140	* enable it, so check if it's enabled and also check if we've requested it to
		4141	* be enabled.
		4142	*/
		4143	bool intel_using_power_well(struct drm_device *dev)
		4144	{
		4145	struct drm_i915_private *dev_priv = dev->dev_private;
		4146
		4147	if (IS_HASWELL(dev))
		4148	return I915_READ(HSW_PWR_WELL_DRIVER) ==
		4149	(HSW_PWR_WELL_ENABLE \| HSW_PWR_WELL_STATE);
		4150	else
		4151	return true;
		4152	}
		4153
3480	Serge	4154	void intel_set_power_well(struct drm_device *dev, bool enable)
3031	serge	4155	{
		4156	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	4157	bool is_enabled, enable_requested;
		4158	uint32_t tmp;
3031	serge	4159
3746	Serge	4160	if (!HAS_POWER_WELL(dev))
3031	serge	4161	return;
		4162
3482	Serge	4163	if (!i915_disable_power_well && !enable)
		4164	return;
		4165
3480	Serge	4166	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
		4167	is_enabled = tmp & HSW_PWR_WELL_STATE;
		4168	enable_requested = tmp & HSW_PWR_WELL_ENABLE;
3031	serge	4169
3480	Serge	4170	if (enable) {
		4171	if (!enable_requested)
		4172	I915_WRITE(HSW_PWR_WELL_DRIVER, HSW_PWR_WELL_ENABLE);
3031	serge	4173
3480	Serge	4174	if (!is_enabled) {
		4175	DRM_DEBUG_KMS("Enabling power well\n");
		4176	if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
		4177	HSW_PWR_WELL_STATE), 20))
		4178	DRM_ERROR("Timeout enabling power well\n");
3031	serge	4179	}
3480	Serge	4180	} else {
		4181	if (enable_requested) {
		4182	I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
		4183	DRM_DEBUG_KMS("Requesting to disable the power well\n");
3031	serge	4184	}
3480	Serge	4185	}
		4186	}
3031	serge	4187
3480	Serge	4188	/*
		4189	* Starting with Haswell, we have a "Power Down Well" that can be turned off
		4190	* when not needed anymore. We have 4 registers that can request the power well
		4191	* to be enabled, and it will only be disabled if none of the registers is
		4192	* requesting it to be enabled.
		4193	*/
		4194	void intel_init_power_well(struct drm_device *dev)
		4195	{
		4196	struct drm_i915_private *dev_priv = dev->dev_private;
		4197
3746	Serge	4198	if (!HAS_POWER_WELL(dev))
3480	Serge	4199	return;
		4200
		4201	/* For now, we need the power well to be always enabled. */
		4202	intel_set_power_well(dev, true);
		4203
		4204	/* We're taking over the BIOS, so clear any requests made by it since
		4205	* the driver is in charge now. */
		4206	if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE)
		4207	I915_WRITE(HSW_PWR_WELL_BIOS, 0);
3031	serge	4208	}
		4209
		4210	/* Set up chip specific power management-related functions */
		4211	void intel_init_pm(struct drm_device *dev)
		4212	{
		4213	struct drm_i915_private *dev_priv = dev->dev_private;
		4214
		4215	if (I915_HAS_FBC(dev)) {
		4216	if (HAS_PCH_SPLIT(dev)) {
		4217	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
		4218	dev_priv->display.enable_fbc = ironlake_enable_fbc;
		4219	dev_priv->display.disable_fbc = ironlake_disable_fbc;
		4220	} else if (IS_GM45(dev)) {
		4221	dev_priv->display.fbc_enabled = g4x_fbc_enabled;
		4222	dev_priv->display.enable_fbc = g4x_enable_fbc;
		4223	dev_priv->display.disable_fbc = g4x_disable_fbc;
		4224	} else if (IS_CRESTLINE(dev)) {
		4225	dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
		4226	dev_priv->display.enable_fbc = i8xx_enable_fbc;
		4227	dev_priv->display.disable_fbc = i8xx_disable_fbc;
		4228	}
		4229	/* 855GM needs testing */
		4230	}
		4231
		4232	/* For cxsr */
		4233	if (IS_PINEVIEW(dev))
		4234	i915_pineview_get_mem_freq(dev);
		4235	else if (IS_GEN5(dev))
		4236	i915_ironlake_get_mem_freq(dev);
		4237
		4238	/* For FIFO watermark updates */
		4239	if (HAS_PCH_SPLIT(dev)) {
		4240	if (IS_GEN5(dev)) {
		4241	if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
		4242	dev_priv->display.update_wm = ironlake_update_wm;
		4243	else {
		4244	DRM_DEBUG_KMS("Failed to get proper latency. "
		4245	"Disable CxSR\n");
		4246	dev_priv->display.update_wm = NULL;
		4247	}
		4248	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		4249	} else if (IS_GEN6(dev)) {
		4250	if (SNB_READ_WM0_LATENCY()) {
		4251	dev_priv->display.update_wm = sandybridge_update_wm;
		4252	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4253	} else {
		4254	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4255	"Disable CxSR\n");
		4256	dev_priv->display.update_wm = NULL;
		4257	}
		4258	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
		4259	} else if (IS_IVYBRIDGE(dev)) {
		4260	if (SNB_READ_WM0_LATENCY()) {
3243	Serge	4261	dev_priv->display.update_wm = ivybridge_update_wm;
3031	serge	4262	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4263	} else {
		4264	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4265	"Disable CxSR\n");
		4266	dev_priv->display.update_wm = NULL;
		4267	}
		4268	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
		4269	} else if (IS_HASWELL(dev)) {
		4270	if (SNB_READ_WM0_LATENCY()) {
		4271	dev_priv->display.update_wm = sandybridge_update_wm;
		4272	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4273	dev_priv->display.update_linetime_wm = haswell_update_linetime_wm;
		4274	} else {
		4275	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4276	"Disable CxSR\n");
		4277	dev_priv->display.update_wm = NULL;
		4278	}
		4279	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
		4280	} else
		4281	dev_priv->display.update_wm = NULL;
		4282	} else if (IS_VALLEYVIEW(dev)) {
		4283	dev_priv->display.update_wm = valleyview_update_wm;
		4284	dev_priv->display.init_clock_gating =
		4285	valleyview_init_clock_gating;
		4286	} else if (IS_PINEVIEW(dev)) {
		4287	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		4288	dev_priv->is_ddr3,
		4289	dev_priv->fsb_freq,
		4290	dev_priv->mem_freq)) {
		4291	DRM_INFO("failed to find known CxSR latency "
		4292	"(found ddr%s fsb freq %d, mem freq %d), "
		4293	"disabling CxSR\n",
		4294	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		4295	dev_priv->fsb_freq, dev_priv->mem_freq);
		4296	/* Disable CxSR and never update its watermark again */
		4297	pineview_disable_cxsr(dev);
		4298	dev_priv->display.update_wm = NULL;
		4299	} else
		4300	dev_priv->display.update_wm = pineview_update_wm;
		4301	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		4302	} else if (IS_G4X(dev)) {
		4303	dev_priv->display.update_wm = g4x_update_wm;
		4304	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		4305	} else if (IS_GEN4(dev)) {
		4306	dev_priv->display.update_wm = i965_update_wm;
		4307	if (IS_CRESTLINE(dev))
		4308	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		4309	else if (IS_BROADWATER(dev))
		4310	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		4311	} else if (IS_GEN3(dev)) {
		4312	dev_priv->display.update_wm = i9xx_update_wm;
		4313	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		4314	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		4315	} else if (IS_I865G(dev)) {
		4316	dev_priv->display.update_wm = i830_update_wm;
		4317	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		4318	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		4319	} else if (IS_I85X(dev)) {
		4320	dev_priv->display.update_wm = i9xx_update_wm;
		4321	dev_priv->display.get_fifo_size = i85x_get_fifo_size;
		4322	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		4323	} else {
		4324	dev_priv->display.update_wm = i830_update_wm;
		4325	dev_priv->display.init_clock_gating = i830_init_clock_gating;
		4326	if (IS_845G(dev))
		4327	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		4328	else
		4329	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		4330	}
		4331	}
		4332
		4333	static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
		4334	{
		4335	u32 gt_thread_status_mask;
		4336
		4337	if (IS_HASWELL(dev_priv->dev))
		4338	gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
		4339	else
		4340	gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
		4341
		4342	/* w/a for a sporadic read returning 0 by waiting for the GT
		4343	* thread to wake up.
		4344	*/
		4345	if (wait_for_atomic_us((I915_READ_NOTRACE(GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
		4346	DRM_ERROR("GT thread status wait timed out\n");
		4347	}
		4348
3243	Serge	4349	static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
		4350	{
		4351	I915_WRITE_NOTRACE(FORCEWAKE, 0);
		4352	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
		4353	}
		4354
3031	serge	4355	static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
		4356	{
3746	Serge	4357	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK) & 1) == 0,
3031	serge	4358	FORCEWAKE_ACK_TIMEOUT_MS))
		4359	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4360
3746	Serge	4361	I915_WRITE_NOTRACE(FORCEWAKE, 1);
3031	serge	4362	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
		4363
3746	Serge	4364	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK) & 1),
3031	serge	4365	FORCEWAKE_ACK_TIMEOUT_MS))
		4366	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4367
		4368	__gen6_gt_wait_for_thread_c0(dev_priv);
		4369	}
		4370
3243	Serge	4371	static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
		4372	{
		4373	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
		4374	/* something from same cacheline, but !FORCEWAKE_MT */
		4375	POSTING_READ(ECOBUS);
		4376	}
		4377
3031	serge	4378	static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
		4379	{
		4380	u32 forcewake_ack;
		4381
		4382	if (IS_HASWELL(dev_priv->dev))
		4383	forcewake_ack = FORCEWAKE_ACK_HSW;
		4384	else
		4385	forcewake_ack = FORCEWAKE_MT_ACK;
		4386
3746	Serge	4387	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & FORCEWAKE_KERNEL) == 0,
3031	serge	4388	FORCEWAKE_ACK_TIMEOUT_MS))
		4389	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4390
3243	Serge	4391	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
		4392	/* something from same cacheline, but !FORCEWAKE_MT */
		4393	POSTING_READ(ECOBUS);
3031	serge	4394
3746	Serge	4395	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & FORCEWAKE_KERNEL),
3031	serge	4396	FORCEWAKE_ACK_TIMEOUT_MS))
		4397	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4398
		4399	__gen6_gt_wait_for_thread_c0(dev_priv);
		4400	}
		4401
		4402	/*
		4403	* Generally this is called implicitly by the register read function. However,
		4404	* if some sequence requires the GT to not power down then this function should
		4405	* be called at the beginning of the sequence followed by a call to
		4406	* gen6_gt_force_wake_put() at the end of the sequence.
		4407	*/
		4408	void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
		4409	{
		4410	unsigned long irqflags;
		4411
		4412	spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
		4413	if (dev_priv->forcewake_count++ == 0)
		4414	dev_priv->gt.force_wake_get(dev_priv);
		4415	spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
		4416	}
		4417
		4418	void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
		4419	{
		4420	u32 gtfifodbg;
		4421	gtfifodbg = I915_READ_NOTRACE(GTFIFODBG);
		4422	if (WARN(gtfifodbg & GT_FIFO_CPU_ERROR_MASK,
		4423	"MMIO read or write has been dropped %x\n", gtfifodbg))
		4424	I915_WRITE_NOTRACE(GTFIFODBG, GT_FIFO_CPU_ERROR_MASK);
		4425	}
		4426
		4427	static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
		4428	{
		4429	I915_WRITE_NOTRACE(FORCEWAKE, 0);
3243	Serge	4430	/* something from same cacheline, but !FORCEWAKE */
		4431	POSTING_READ(ECOBUS);
3031	serge	4432	gen6_gt_check_fifodbg(dev_priv);
		4433	}
		4434
		4435	static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
		4436	{
3243	Serge	4437	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
		4438	/* something from same cacheline, but !FORCEWAKE_MT */
		4439	POSTING_READ(ECOBUS);
3031	serge	4440	gen6_gt_check_fifodbg(dev_priv);
		4441	}
		4442
		4443	/*
		4444	* see gen6_gt_force_wake_get()
		4445	*/
		4446	void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
		4447	{
		4448	unsigned long irqflags;
		4449
		4450	spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
		4451	if (--dev_priv->forcewake_count == 0)
		4452	dev_priv->gt.force_wake_put(dev_priv);
		4453	spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
		4454	}
		4455
		4456	int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
		4457	{
		4458	int ret = 0;
		4459
		4460	if (dev_priv->gt_fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
		4461	int loop = 500;
		4462	u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
		4463	while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
		4464	udelay(10);
		4465	fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
		4466	}
		4467	if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
		4468	++ret;
		4469	dev_priv->gt_fifo_count = fifo;
		4470	}
		4471	dev_priv->gt_fifo_count--;
		4472
		4473	return ret;
		4474	}
		4475
3243	Serge	4476	static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
		4477	{
		4478	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(0xffff));
		4479	/* something from same cacheline, but !FORCEWAKE_VLV */
		4480	POSTING_READ(FORCEWAKE_ACK_VLV);
		4481	}
		4482
3031	serge	4483	static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
		4484	{
3746	Serge	4485	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL) == 0,
3031	serge	4486	FORCEWAKE_ACK_TIMEOUT_MS))
		4487	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4488
3243	Serge	4489	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
3746	Serge	4490	I915_WRITE_NOTRACE(FORCEWAKE_MEDIA_VLV,
		4491	_MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
3031	serge	4492
3746	Serge	4493	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & FORCEWAKE_KERNEL),
3031	serge	4494	FORCEWAKE_ACK_TIMEOUT_MS))
3746	Serge	4495	DRM_ERROR("Timed out waiting for GT to ack forcewake request.\n");
3031	serge	4496
3746	Serge	4497	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_MEDIA_VLV) &
		4498	FORCEWAKE_KERNEL),
		4499	FORCEWAKE_ACK_TIMEOUT_MS))
		4500	DRM_ERROR("Timed out waiting for media to ack forcewake request.\n");
		4501
3031	serge	4502	__gen6_gt_wait_for_thread_c0(dev_priv);
		4503	}
		4504
		4505	static void vlv_force_wake_put(struct drm_i915_private *dev_priv)
		4506	{
3243	Serge	4507	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
3746	Serge	4508	I915_WRITE_NOTRACE(FORCEWAKE_MEDIA_VLV,
		4509	_MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
		4510	/* The below doubles as a POSTING_READ */
3031	serge	4511	gen6_gt_check_fifodbg(dev_priv);
		4512	}
		4513
3243	Serge	4514	void intel_gt_reset(struct drm_device *dev)
		4515	{
		4516	struct drm_i915_private *dev_priv = dev->dev_private;
		4517
		4518	if (IS_VALLEYVIEW(dev)) {
		4519	vlv_force_wake_reset(dev_priv);
		4520	} else if (INTEL_INFO(dev)->gen >= 6) {
		4521	__gen6_gt_force_wake_reset(dev_priv);
		4522	if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev))
		4523	__gen6_gt_force_wake_mt_reset(dev_priv);
		4524	}
		4525	}
		4526
3031	serge	4527	void intel_gt_init(struct drm_device *dev)
		4528	{
		4529	struct drm_i915_private *dev_priv = dev->dev_private;
		4530
		4531	spin_lock_init(&dev_priv->gt_lock);
		4532
3243	Serge	4533	intel_gt_reset(dev);
		4534
3031	serge	4535	if (IS_VALLEYVIEW(dev)) {
		4536	dev_priv->gt.force_wake_get = vlv_force_wake_get;
		4537	dev_priv->gt.force_wake_put = vlv_force_wake_put;
3243	Serge	4538	} else if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev)) {
		4539	dev_priv->gt.force_wake_get = __gen6_gt_force_wake_mt_get;
		4540	dev_priv->gt.force_wake_put = __gen6_gt_force_wake_mt_put;
		4541	} else if (IS_GEN6(dev)) {
3031	serge	4542	dev_priv->gt.force_wake_get = __gen6_gt_force_wake_get;
		4543	dev_priv->gt.force_wake_put = __gen6_gt_force_wake_put;
3243	Serge	4544	}
3482	Serge	4545	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		4546	intel_gen6_powersave_work);
3243	Serge	4547	}
3031	serge	4548
3243	Serge	4549	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u8 mbox, u32 val)
		4550	{
		4551	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4552
3243	Serge	4553	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		4554	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		4555	return -EAGAIN;
		4556	}
3031	serge	4557
3243	Serge	4558	I915_WRITE(GEN6_PCODE_DATA, *val);
		4559	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		4560
		4561	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		4562	500)) {
		4563	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		4564	return -ETIMEDOUT;
3031	serge	4565	}
3243	Serge	4566
		4567	*val = I915_READ(GEN6_PCODE_DATA);
		4568	I915_WRITE(GEN6_PCODE_DATA, 0);
		4569
		4570	return 0;
		4571	}
		4572
		4573	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
		4574	{
		4575	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4576
		4577	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		4578	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		4579	return -EAGAIN;
3031	serge	4580	}
3243	Serge	4581
		4582	I915_WRITE(GEN6_PCODE_DATA, val);
		4583	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		4584
		4585	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		4586	500)) {
		4587	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		4588	return -ETIMEDOUT;
3031	serge	4589	}
3243	Serge	4590
		4591	I915_WRITE(GEN6_PCODE_DATA, 0);
		4592
		4593	return 0;
3031	serge	4594	}
3746	Serge	4595
		4596	static int vlv_punit_rw(struct drm_i915_private *dev_priv, u8 opcode,
		4597	u8 addr, u32 *val)
		4598	{
		4599	u32 cmd, devfn, port, be, bar;
		4600
		4601	bar = 0;
		4602	be = 0xf;
		4603	port = IOSF_PORT_PUNIT;
		4604	devfn = PCI_DEVFN(2, 0);
		4605
		4606	cmd = (devfn << IOSF_DEVFN_SHIFT) \| (opcode << IOSF_OPCODE_SHIFT) \|
		4607	(port << IOSF_PORT_SHIFT) \| (be << IOSF_BYTE_ENABLES_SHIFT) \|
		4608	(bar << IOSF_BAR_SHIFT);
		4609
		4610	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4611
		4612	if (I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) {
		4613	DRM_DEBUG_DRIVER("warning: pcode (%s) mailbox access failed\n",
		4614	opcode == PUNIT_OPCODE_REG_READ ?
		4615	"read" : "write");
		4616	return -EAGAIN;
		4617	}
		4618
		4619	I915_WRITE(VLV_IOSF_ADDR, addr);
		4620	if (opcode == PUNIT_OPCODE_REG_WRITE)
		4621	I915_WRITE(VLV_IOSF_DATA, *val);
		4622	I915_WRITE(VLV_IOSF_DOORBELL_REQ, cmd);
		4623
		4624	if (wait_for((I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) == 0,
		4625	500)) {
		4626	DRM_ERROR("timeout waiting for pcode %s (%d) to finish\n",
		4627	opcode == PUNIT_OPCODE_REG_READ ? "read" : "write",
		4628	addr);
		4629	return -ETIMEDOUT;
		4630	}
		4631
		4632	if (opcode == PUNIT_OPCODE_REG_READ)
		4633	*val = I915_READ(VLV_IOSF_DATA);
		4634	I915_WRITE(VLV_IOSF_DATA, 0);
		4635
		4636	return 0;
		4637	}
		4638
		4639	int valleyview_punit_read(struct drm_i915_private dev_priv, u8 addr, u32 val)
		4640	{
		4641	return vlv_punit_rw(dev_priv, PUNIT_OPCODE_REG_READ, addr, val);
		4642	}
		4643
		4644	int valleyview_punit_write(struct drm_i915_private *dev_priv, u8 addr, u32 val)
		4645	{
		4646	return vlv_punit_rw(dev_priv, PUNIT_OPCODE_REG_WRITE, addr, &val);
		4647	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 3746