WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	#define iowrite32(v, addr) writel((v), (addr))
		29	#define ioread32(addr) readl(addr)
		30
		31	//#include
		32	#include "i915_drv.h"
		33	#include "intel_drv.h"
		34	#include
		35	//#include "../../../platform/x86/intel_ips.h"
		36	#include
		37
		38	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		39
		40	#define assert_spin_locked(x)
		41
		42	void getrawmonotonic(struct timespec *ts);
		43	void set_normalized_timespec(struct timespec *ts, time_t sec, long nsec);
		44
		45	static inline struct timespec timespec_sub(struct timespec lhs,
		46	struct timespec rhs)
		47	{
		48	struct timespec ts_delta;
		49	set_normalized_timespec(&ts_delta, lhs.tv_sec - rhs.tv_sec,
		50	lhs.tv_nsec - rhs.tv_nsec);
		51	return ts_delta;
		52	}
		53
		54
		55	/* FBC, or Frame Buffer Compression, is a technique employed to compress the
		56	* framebuffer contents in-memory, aiming at reducing the required bandwidth
		57	* during in-memory transfers and, therefore, reduce the power packet.
		58	*
		59	* The benefits of FBC are mostly visible with solid backgrounds and
		60	* variation-less patterns.
		61	*
		62	* FBC-related functionality can be enabled by the means of the
		63	* i915.i915_enable_fbc parameter
		64	*/
		65
3243	Serge	66	static bool intel_crtc_active(struct drm_crtc *crtc)
		67	{
		68	/* Be paranoid as we can arrive here with only partial
		69	* state retrieved from the hardware during setup.
		70	*/
		71	return to_intel_crtc(crtc)->active && crtc->fb && crtc->mode.clock;
		72	}
		73
3031	serge	74	static void i8xx_disable_fbc(struct drm_device *dev)
		75	{
		76	struct drm_i915_private *dev_priv = dev->dev_private;
		77	u32 fbc_ctl;
		78
		79	/* Disable compression */
		80	fbc_ctl = I915_READ(FBC_CONTROL);
		81	if ((fbc_ctl & FBC_CTL_EN) == 0)
		82	return;
		83
		84	fbc_ctl &= ~FBC_CTL_EN;
		85	I915_WRITE(FBC_CONTROL, fbc_ctl);
		86
		87	/* Wait for compressing bit to clear */
		88	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
		89	DRM_DEBUG_KMS("FBC idle timed out\n");
		90	return;
		91	}
		92
		93	DRM_DEBUG_KMS("disabled FBC\n");
		94	}
		95
		96	static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		97	{
		98	struct drm_device *dev = crtc->dev;
		99	struct drm_i915_private *dev_priv = dev->dev_private;
		100	struct drm_framebuffer *fb = crtc->fb;
		101	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		102	struct drm_i915_gem_object *obj = intel_fb->obj;
		103	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		104	int cfb_pitch;
		105	int plane, i;
		106	u32 fbc_ctl, fbc_ctl2;
		107
		108	cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
		109	if (fb->pitches[0] < cfb_pitch)
		110	cfb_pitch = fb->pitches[0];
		111
		112	/* FBC_CTL wants 64B units */
		113	cfb_pitch = (cfb_pitch / 64) - 1;
		114	plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
		115
		116	/* Clear old tags */
		117	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
		118	I915_WRITE(FBC_TAG + (i * 4), 0);
		119
		120	/* Set it up... */
		121	fbc_ctl2 = FBC_CTL_FENCE_DBL \| FBC_CTL_IDLE_IMM \| FBC_CTL_CPU_FENCE;
		122	fbc_ctl2 \|= plane;
		123	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
		124	I915_WRITE(FBC_FENCE_OFF, crtc->y);
		125
		126	/* enable it... */
		127	fbc_ctl = FBC_CTL_EN \| FBC_CTL_PERIODIC;
		128	if (IS_I945GM(dev))
		129	fbc_ctl \|= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
		130	fbc_ctl \|= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
		131	fbc_ctl \|= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
		132	fbc_ctl \|= obj->fence_reg;
		133	I915_WRITE(FBC_CONTROL, fbc_ctl);
		134
		135	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
		136	cfb_pitch, crtc->y, intel_crtc->plane);
		137	}
		138
		139	static bool i8xx_fbc_enabled(struct drm_device *dev)
		140	{
		141	struct drm_i915_private *dev_priv = dev->dev_private;
		142
		143	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
		144	}
		145
		146	static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		147	{
		148	struct drm_device *dev = crtc->dev;
		149	struct drm_i915_private *dev_priv = dev->dev_private;
		150	struct drm_framebuffer *fb = crtc->fb;
		151	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		152	struct drm_i915_gem_object *obj = intel_fb->obj;
		153	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		154	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		155	unsigned long stall_watermark = 200;
		156	u32 dpfc_ctl;
		157
		158	dpfc_ctl = plane \| DPFC_SR_EN \| DPFC_CTL_LIMIT_1X;
		159	dpfc_ctl \|= DPFC_CTL_FENCE_EN \| obj->fence_reg;
		160	I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
		161
		162	I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		163	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		164	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		165	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
		166
		167	/* enable it... */
		168	I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) \| DPFC_CTL_EN);
		169
		170	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		171	}
		172
		173	static void g4x_disable_fbc(struct drm_device *dev)
		174	{
		175	struct drm_i915_private *dev_priv = dev->dev_private;
		176	u32 dpfc_ctl;
		177
		178	/* Disable compression */
		179	dpfc_ctl = I915_READ(DPFC_CONTROL);
		180	if (dpfc_ctl & DPFC_CTL_EN) {
		181	dpfc_ctl &= ~DPFC_CTL_EN;
		182	I915_WRITE(DPFC_CONTROL, dpfc_ctl);
		183
		184	DRM_DEBUG_KMS("disabled FBC\n");
		185	}
		186	}
		187
		188	static bool g4x_fbc_enabled(struct drm_device *dev)
		189	{
		190	struct drm_i915_private *dev_priv = dev->dev_private;
		191
		192	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
		193	}
		194
		195	static void sandybridge_blit_fbc_update(struct drm_device *dev)
		196	{
		197	struct drm_i915_private *dev_priv = dev->dev_private;
		198	u32 blt_ecoskpd;
		199
		200	/* Make sure blitter notifies FBC of writes */
		201	gen6_gt_force_wake_get(dev_priv);
		202	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
		203	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY <<
		204	GEN6_BLITTER_LOCK_SHIFT;
		205	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		206	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY;
		207	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		208	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
		209	GEN6_BLITTER_LOCK_SHIFT);
		210	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		211	POSTING_READ(GEN6_BLITTER_ECOSKPD);
		212	gen6_gt_force_wake_put(dev_priv);
		213	}
		214
		215	static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		216	{
		217	struct drm_device *dev = crtc->dev;
		218	struct drm_i915_private *dev_priv = dev->dev_private;
		219	struct drm_framebuffer *fb = crtc->fb;
		220	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		221	struct drm_i915_gem_object *obj = intel_fb->obj;
		222	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		223	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		224	unsigned long stall_watermark = 200;
		225	u32 dpfc_ctl;
		226
		227	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		228	dpfc_ctl &= DPFC_RESERVED;
		229	dpfc_ctl \|= (plane \| DPFC_CTL_LIMIT_1X);
		230	/* Set persistent mode for front-buffer rendering, ala X. */
		231	dpfc_ctl \|= DPFC_CTL_PERSISTENT_MODE;
		232	dpfc_ctl \|= (DPFC_CTL_FENCE_EN \| obj->fence_reg);
		233	I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
		234
		235	I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		236	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		237	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		238	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
		239	I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset \| ILK_FBC_RT_VALID);
		240	/* enable it... */
		241	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		242
		243	if (IS_GEN6(dev)) {
		244	I915_WRITE(SNB_DPFC_CTL_SA,
		245	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		246	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		247	sandybridge_blit_fbc_update(dev);
		248	}
		249
		250	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		251	}
		252
		253	static void ironlake_disable_fbc(struct drm_device *dev)
		254	{
		255	struct drm_i915_private *dev_priv = dev->dev_private;
		256	u32 dpfc_ctl;
		257
		258	/* Disable compression */
		259	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		260	if (dpfc_ctl & DPFC_CTL_EN) {
		261	dpfc_ctl &= ~DPFC_CTL_EN;
		262	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
		263
		264	DRM_DEBUG_KMS("disabled FBC\n");
		265	}
		266	}
		267
		268	static bool ironlake_fbc_enabled(struct drm_device *dev)
		269	{
		270	struct drm_i915_private *dev_priv = dev->dev_private;
		271
		272	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
		273	}
		274
		275	bool intel_fbc_enabled(struct drm_device *dev)
		276	{
		277	struct drm_i915_private *dev_priv = dev->dev_private;
		278
		279	if (!dev_priv->display.fbc_enabled)
		280	return false;
		281
		282	return dev_priv->display.fbc_enabled(dev);
		283	}
		284
		285	static void intel_fbc_work_fn(struct work_struct *__work)
		286	{
		287	struct intel_fbc_work *work =
		288	container_of(to_delayed_work(__work),
		289	struct intel_fbc_work, work);
		290	struct drm_device *dev = work->crtc->dev;
		291	struct drm_i915_private *dev_priv = dev->dev_private;
		292
		293	mutex_lock(&dev->struct_mutex);
		294	if (work == dev_priv->fbc_work) {
		295	/* Double check that we haven't switched fb without cancelling
		296	* the prior work.
		297	*/
		298	if (work->crtc->fb == work->fb) {
		299	dev_priv->display.enable_fbc(work->crtc,
		300	work->interval);
		301
		302	dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
		303	dev_priv->cfb_fb = work->crtc->fb->base.id;
		304	dev_priv->cfb_y = work->crtc->y;
		305	}
		306
		307	dev_priv->fbc_work = NULL;
		308	}
		309	mutex_unlock(&dev->struct_mutex);
		310
		311	kfree(work);
		312	}
		313
		314	static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
		315	{
		316	if (dev_priv->fbc_work == NULL)
		317	return;
		318
		319	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
		320
		321	/* Synchronisation is provided by struct_mutex and checking of
		322	* dev_priv->fbc_work, so we can perform the cancellation
		323	* entirely asynchronously.
		324	*/
3482	Serge	325	// if (cancel_delayed_work(&dev_priv->fbc_work->work))
3031	serge	326	/* tasklet was killed before being run, clean up */
3482	Serge	327	// kfree(dev_priv->fbc_work);
3031	serge	328
		329	/* Mark the work as no longer wanted so that if it does
		330	* wake-up (because the work was already running and waiting
		331	* for our mutex), it will discover that is no longer
		332	* necessary to run.
		333	*/
		334	dev_priv->fbc_work = NULL;
		335	}
		336
		337	void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		338	{
		339	struct intel_fbc_work *work;
		340	struct drm_device *dev = crtc->dev;
		341	struct drm_i915_private *dev_priv = dev->dev_private;
		342
3482	Serge	343	if (!dev_priv->display.enable_fbc)
3031	serge	344	return;
3482	Serge	345
3031	serge	346	intel_cancel_fbc_work(dev_priv);
		347
		348	work = kzalloc(sizeof *work, GFP_KERNEL);
		349	if (work == NULL) {
		350	dev_priv->display.enable_fbc(crtc, interval);
		351	return;
		352	}
		353
		354	work->crtc = crtc;
		355	work->fb = crtc->fb;
		356	work->interval = interval;
		357	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
		358
		359	dev_priv->fbc_work = work;
		360
		361	DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
		362
		363	/* Delay the actual enabling to let pageflipping cease and the
		364	* display to settle before starting the compression. Note that
		365	* this delay also serves a second purpose: it allows for a
		366	* vblank to pass after disabling the FBC before we attempt
		367	* to modify the control registers.
		368	*
		369	* A more complicated solution would involve tracking vblanks
		370	* following the termination of the page-flipping sequence
		371	* and indeed performing the enable as a co-routine and not
		372	* waiting synchronously upon the vblank.
		373	*/
		374	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
		375	}
		376
		377	void intel_disable_fbc(struct drm_device *dev)
		378	{
		379	struct drm_i915_private *dev_priv = dev->dev_private;
		380
3482	Serge	381	intel_cancel_fbc_work(dev_priv);
3031	serge	382
3482	Serge	383	if (!dev_priv->display.disable_fbc)
		384	return;
3031	serge	385
3482	Serge	386	dev_priv->display.disable_fbc(dev);
3031	serge	387	dev_priv->cfb_plane = -1;
		388	}
		389
		390	/**
		391	* intel_update_fbc - enable/disable FBC as needed
		392	* @dev: the drm_device
		393	*
		394	* Set up the framebuffer compression hardware at mode set time. We
		395	* enable it if possible:
		396	* - plane A only (on pre-965)
		397	* - no pixel mulitply/line duplication
		398	* - no alpha buffer discard
		399	* - no dual wide
		400	* - framebuffer <= 2048 in width, 1536 in height
		401	*
		402	* We can't assume that any compression will take place (worst case),
		403	* so the compressed buffer has to be the same size as the uncompressed
		404	* one. It also must reside (along with the line length buffer) in
		405	* stolen memory.
		406	*
		407	* We need to enable/disable FBC on a global basis.
		408	*/
		409	void intel_update_fbc(struct drm_device *dev)
		410	{
		411	struct drm_i915_private *dev_priv = dev->dev_private;
		412	struct drm_crtc crtc = NULL, tmp_crtc;
		413	struct intel_crtc *intel_crtc;
		414	struct drm_framebuffer *fb;
		415	struct intel_framebuffer *intel_fb;
		416	struct drm_i915_gem_object *obj;
		417	int enable_fbc;
		418
3482	Serge	419	ENTER();
		420
3031	serge	421	if (!i915_powersave)
		422	return;
		423
		424	if (!I915_HAS_FBC(dev))
		425	return;
		426
		427	/*
		428	* If FBC is already on, we just have to verify that we can
		429	* keep it that way...
		430	* Need to disable if:
		431	* - more than one pipe is active
		432	* - changing FBC params (stride, fence, mode)
		433	* - new fb is too large to fit in compressed buffer
		434	* - going to an unsupported config (interlace, pixel multiply, etc.)
		435	*/
		436	list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	437	if (intel_crtc_active(tmp_crtc) &&
		438	!to_intel_crtc(tmp_crtc)->primary_disabled) {
3031	serge	439	if (crtc) {
		440	DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
		441	dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
		442	goto out_disable;
		443	}
		444	crtc = tmp_crtc;
		445	}
		446	}
		447
		448	if (!crtc \|\| crtc->fb == NULL) {
		449	DRM_DEBUG_KMS("no output, disabling\n");
		450	dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
		451	goto out_disable;
		452	}
		453
		454	intel_crtc = to_intel_crtc(crtc);
		455	fb = crtc->fb;
		456	intel_fb = to_intel_framebuffer(fb);
		457	obj = intel_fb->obj;
		458
		459	enable_fbc = i915_enable_fbc;
		460	if (enable_fbc < 0) {
		461	DRM_DEBUG_KMS("fbc set to per-chip default\n");
		462	enable_fbc = 1;
		463	if (INTEL_INFO(dev)->gen <= 6)
		464	enable_fbc = 0;
		465	}
		466	if (!enable_fbc) {
		467	DRM_DEBUG_KMS("fbc disabled per module param\n");
		468	dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
		469	goto out_disable;
		470	}
		471	if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) \|\|
		472	(crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
		473	DRM_DEBUG_KMS("mode incompatible with compression, "
		474	"disabling\n");
		475	dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
		476	goto out_disable;
		477	}
		478	if ((crtc->mode.hdisplay > 2048) \|\|
		479	(crtc->mode.vdisplay > 1536)) {
		480	DRM_DEBUG_KMS("mode too large for compression, disabling\n");
		481	dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
		482	goto out_disable;
		483	}
		484	if ((IS_I915GM(dev) \|\| IS_I945GM(dev)) && intel_crtc->plane != 0) {
		485	DRM_DEBUG_KMS("plane not 0, disabling compression\n");
		486	dev_priv->no_fbc_reason = FBC_BAD_PLANE;
		487	goto out_disable;
		488	}
		489
		490	/* The use of a CPU fence is mandatory in order to detect writes
		491	* by the CPU to the scanout and trigger updates to the FBC.
		492	*/
		493	if (obj->tiling_mode != I915_TILING_X \|\|
		494	obj->fence_reg == I915_FENCE_REG_NONE) {
		495	DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
		496	dev_priv->no_fbc_reason = FBC_NOT_TILED;
		497	goto out_disable;
		498	}
		499
		500	/* If the kernel debugger is active, always disable compression */
		501	if (in_dbg_master())
		502	goto out_disable;
		503
3480	Serge	504	if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
		505	DRM_INFO("not enough stolen space for compressed buffer (need %zd bytes), disabling\n", intel_fb->obj->base.size);
		506	DRM_INFO("hint: you may be able to increase stolen memory size in the BIOS to avoid this\n");
		507	DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
		508	dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
		509	goto out_disable;
		510	}
		511
3031	serge	512	/* If the scanout has not changed, don't modify the FBC settings.
		513	* Note that we make the fundamental assumption that the fb->obj
		514	* cannot be unpinned (and have its GTT offset and fence revoked)
		515	* without first being decoupled from the scanout and FBC disabled.
		516	*/
		517	if (dev_priv->cfb_plane == intel_crtc->plane &&
		518	dev_priv->cfb_fb == fb->base.id &&
		519	dev_priv->cfb_y == crtc->y)
		520	return;
		521
		522	if (intel_fbc_enabled(dev)) {
		523	/* We update FBC along two paths, after changing fb/crtc
		524	* configuration (modeswitching) and after page-flipping
		525	* finishes. For the latter, we know that not only did
		526	* we disable the FBC at the start of the page-flip
		527	* sequence, but also more than one vblank has passed.
		528	*
		529	* For the former case of modeswitching, it is possible
		530	* to switch between two FBC valid configurations
		531	* instantaneously so we do need to disable the FBC
		532	* before we can modify its control registers. We also
		533	* have to wait for the next vblank for that to take
		534	* effect. However, since we delay enabling FBC we can
		535	* assume that a vblank has passed since disabling and
		536	* that we can safely alter the registers in the deferred
		537	* callback.
		538	*
		539	* In the scenario that we go from a valid to invalid
		540	* and then back to valid FBC configuration we have
		541	* no strict enforcement that a vblank occurred since
		542	* disabling the FBC. However, along all current pipe
		543	* disabling paths we do need to wait for a vblank at
		544	* some point. And we wait before enabling FBC anyway.
		545	*/
		546	DRM_DEBUG_KMS("disabling active FBC for update\n");
		547	intel_disable_fbc(dev);
		548	}
		549
		550	intel_enable_fbc(crtc, 500);
3482	Serge	551	LEAVE();
		552
3031	serge	553	return;
		554
		555	out_disable:
		556	/* Multiple disables should be harmless */
		557	if (intel_fbc_enabled(dev)) {
		558	DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
		559	intel_disable_fbc(dev);
		560	}
3480	Serge	561	i915_gem_stolen_cleanup_compression(dev);
3482	Serge	562	LEAVE();
3031	serge	563	}
		564
		565	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		566	{
		567	drm_i915_private_t *dev_priv = dev->dev_private;
		568	u32 tmp;
		569
		570	tmp = I915_READ(CLKCFG);
		571
		572	switch (tmp & CLKCFG_FSB_MASK) {
		573	case CLKCFG_FSB_533:
		574	dev_priv->fsb_freq = 533; /* 1334 /
		575	break;
		576	case CLKCFG_FSB_800:
		577	dev_priv->fsb_freq = 800; /* 2004 /
		578	break;
		579	case CLKCFG_FSB_667:
		580	dev_priv->fsb_freq = 667; /* 1674 /
		581	break;
		582	case CLKCFG_FSB_400:
		583	dev_priv->fsb_freq = 400; /* 1004 /
		584	break;
		585	}
		586
		587	switch (tmp & CLKCFG_MEM_MASK) {
		588	case CLKCFG_MEM_533:
		589	dev_priv->mem_freq = 533;
		590	break;
		591	case CLKCFG_MEM_667:
		592	dev_priv->mem_freq = 667;
		593	break;
		594	case CLKCFG_MEM_800:
		595	dev_priv->mem_freq = 800;
		596	break;
		597	}
		598
		599	/* detect pineview DDR3 setting */
		600	tmp = I915_READ(CSHRDDR3CTL);
		601	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		602	}
		603
		604	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		605	{
		606	drm_i915_private_t *dev_priv = dev->dev_private;
		607	u16 ddrpll, csipll;
		608
		609	ddrpll = I915_READ16(DDRMPLL1);
		610	csipll = I915_READ16(CSIPLL0);
		611
		612	switch (ddrpll & 0xff) {
		613	case 0xc:
		614	dev_priv->mem_freq = 800;
		615	break;
		616	case 0x10:
		617	dev_priv->mem_freq = 1066;
		618	break;
		619	case 0x14:
		620	dev_priv->mem_freq = 1333;
		621	break;
		622	case 0x18:
		623	dev_priv->mem_freq = 1600;
		624	break;
		625	default:
		626	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		627	ddrpll & 0xff);
		628	dev_priv->mem_freq = 0;
		629	break;
		630	}
		631
		632	dev_priv->ips.r_t = dev_priv->mem_freq;
		633
		634	switch (csipll & 0x3ff) {
		635	case 0x00c:
		636	dev_priv->fsb_freq = 3200;
		637	break;
		638	case 0x00e:
		639	dev_priv->fsb_freq = 3733;
		640	break;
		641	case 0x010:
		642	dev_priv->fsb_freq = 4266;
		643	break;
		644	case 0x012:
		645	dev_priv->fsb_freq = 4800;
		646	break;
		647	case 0x014:
		648	dev_priv->fsb_freq = 5333;
		649	break;
		650	case 0x016:
		651	dev_priv->fsb_freq = 5866;
		652	break;
		653	case 0x018:
		654	dev_priv->fsb_freq = 6400;
		655	break;
		656	default:
		657	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		658	csipll & 0x3ff);
		659	dev_priv->fsb_freq = 0;
		660	break;
		661	}
		662
		663	if (dev_priv->fsb_freq == 3200) {
		664	dev_priv->ips.c_m = 0;
		665	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		666	dev_priv->ips.c_m = 1;
		667	} else {
		668	dev_priv->ips.c_m = 2;
		669	}
		670	}
		671
		672	static const struct cxsr_latency cxsr_latency_table[] = {
		673	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		674	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		675	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		676	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		677	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		678
		679	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		680	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		681	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		682	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		683	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		684
		685	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		686	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		687	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		688	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		689	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		690
		691	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		692	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		693	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		694	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		695	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		696
		697	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		698	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		699	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		700	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		701	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		702
		703	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		704	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		705	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		706	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		707	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		708	};
		709
		710	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		711	int is_ddr3,
		712	int fsb,
		713	int mem)
		714	{
		715	const struct cxsr_latency *latency;
		716	int i;
		717
		718	if (fsb == 0 \|\| mem == 0)
		719	return NULL;
		720
		721	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		722	latency = &cxsr_latency_table[i];
		723	if (is_desktop == latency->is_desktop &&
		724	is_ddr3 == latency->is_ddr3 &&
		725	fsb == latency->fsb_freq && mem == latency->mem_freq)
		726	return latency;
		727	}
		728
		729	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		730
		731	return NULL;
		732	}
		733
		734	static void pineview_disable_cxsr(struct drm_device *dev)
		735	{
		736	struct drm_i915_private *dev_priv = dev->dev_private;
		737
		738	/* deactivate cxsr */
		739	I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
		740	}
		741
		742	/*
		743	* Latency for FIFO fetches is dependent on several factors:
		744	* - memory configuration (speed, channels)
		745	* - chipset
		746	* - current MCH state
		747	* It can be fairly high in some situations, so here we assume a fairly
		748	* pessimal value. It's a tradeoff between extra memory fetches (if we
		749	* set this value too high, the FIFO will fetch frequently to stay full)
		750	* and power consumption (set it too low to save power and we might see
		751	* FIFO underruns and display "flicker").
		752	*
		753	* A value of 5us seems to be a good balance; safe for very low end
		754	* platforms but not overly aggressive on lower latency configs.
		755	*/
		756	static const int latency_ns = 5000;
		757
		758	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		759	{
		760	struct drm_i915_private *dev_priv = dev->dev_private;
		761	uint32_t dsparb = I915_READ(DSPARB);
		762	int size;
		763
		764	size = dsparb & 0x7f;
		765	if (plane)
		766	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		767
		768	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		769	plane ? "B" : "A", size);
		770
		771	return size;
		772	}
		773
		774	static int i85x_get_fifo_size(struct drm_device *dev, int plane)
		775	{
		776	struct drm_i915_private *dev_priv = dev->dev_private;
		777	uint32_t dsparb = I915_READ(DSPARB);
		778	int size;
		779
		780	size = dsparb & 0x1ff;
		781	if (plane)
		782	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		783	size >>= 1; /* Convert to cachelines */
		784
		785	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		786	plane ? "B" : "A", size);
		787
		788	return size;
		789	}
		790
		791	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		792	{
		793	struct drm_i915_private *dev_priv = dev->dev_private;
		794	uint32_t dsparb = I915_READ(DSPARB);
		795	int size;
		796
		797	size = dsparb & 0x7f;
		798	size >>= 2; /* Convert to cachelines */
		799
		800	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		801	plane ? "B" : "A",
		802	size);
		803
		804	return size;
		805	}
		806
		807	static int i830_get_fifo_size(struct drm_device *dev, int plane)
		808	{
		809	struct drm_i915_private *dev_priv = dev->dev_private;
		810	uint32_t dsparb = I915_READ(DSPARB);
		811	int size;
		812
		813	size = dsparb & 0x7f;
		814	size >>= 1; /* Convert to cachelines */
		815
		816	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		817	plane ? "B" : "A", size);
		818
		819	return size;
		820	}
		821
		822	/* Pineview has different values for various configs */
		823	static const struct intel_watermark_params pineview_display_wm = {
		824	PINEVIEW_DISPLAY_FIFO,
		825	PINEVIEW_MAX_WM,
		826	PINEVIEW_DFT_WM,
		827	PINEVIEW_GUARD_WM,
		828	PINEVIEW_FIFO_LINE_SIZE
		829	};
		830	static const struct intel_watermark_params pineview_display_hplloff_wm = {
		831	PINEVIEW_DISPLAY_FIFO,
		832	PINEVIEW_MAX_WM,
		833	PINEVIEW_DFT_HPLLOFF_WM,
		834	PINEVIEW_GUARD_WM,
		835	PINEVIEW_FIFO_LINE_SIZE
		836	};
		837	static const struct intel_watermark_params pineview_cursor_wm = {
		838	PINEVIEW_CURSOR_FIFO,
		839	PINEVIEW_CURSOR_MAX_WM,
		840	PINEVIEW_CURSOR_DFT_WM,
		841	PINEVIEW_CURSOR_GUARD_WM,
		842	PINEVIEW_FIFO_LINE_SIZE,
		843	};
		844	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
		845	PINEVIEW_CURSOR_FIFO,
		846	PINEVIEW_CURSOR_MAX_WM,
		847	PINEVIEW_CURSOR_DFT_WM,
		848	PINEVIEW_CURSOR_GUARD_WM,
		849	PINEVIEW_FIFO_LINE_SIZE
		850	};
		851	static const struct intel_watermark_params g4x_wm_info = {
		852	G4X_FIFO_SIZE,
		853	G4X_MAX_WM,
		854	G4X_MAX_WM,
		855	2,
		856	G4X_FIFO_LINE_SIZE,
		857	};
		858	static const struct intel_watermark_params g4x_cursor_wm_info = {
		859	I965_CURSOR_FIFO,
		860	I965_CURSOR_MAX_WM,
		861	I965_CURSOR_DFT_WM,
		862	2,
		863	G4X_FIFO_LINE_SIZE,
		864	};
		865	static const struct intel_watermark_params valleyview_wm_info = {
		866	VALLEYVIEW_FIFO_SIZE,
		867	VALLEYVIEW_MAX_WM,
		868	VALLEYVIEW_MAX_WM,
		869	2,
		870	G4X_FIFO_LINE_SIZE,
		871	};
		872	static const struct intel_watermark_params valleyview_cursor_wm_info = {
		873	I965_CURSOR_FIFO,
		874	VALLEYVIEW_CURSOR_MAX_WM,
		875	I965_CURSOR_DFT_WM,
		876	2,
		877	G4X_FIFO_LINE_SIZE,
		878	};
		879	static const struct intel_watermark_params i965_cursor_wm_info = {
		880	I965_CURSOR_FIFO,
		881	I965_CURSOR_MAX_WM,
		882	I965_CURSOR_DFT_WM,
		883	2,
		884	I915_FIFO_LINE_SIZE,
		885	};
		886	static const struct intel_watermark_params i945_wm_info = {
		887	I945_FIFO_SIZE,
		888	I915_MAX_WM,
		889	1,
		890	2,
		891	I915_FIFO_LINE_SIZE
		892	};
		893	static const struct intel_watermark_params i915_wm_info = {
		894	I915_FIFO_SIZE,
		895	I915_MAX_WM,
		896	1,
		897	2,
		898	I915_FIFO_LINE_SIZE
		899	};
		900	static const struct intel_watermark_params i855_wm_info = {
		901	I855GM_FIFO_SIZE,
		902	I915_MAX_WM,
		903	1,
		904	2,
		905	I830_FIFO_LINE_SIZE
		906	};
		907	static const struct intel_watermark_params i830_wm_info = {
		908	I830_FIFO_SIZE,
		909	I915_MAX_WM,
		910	1,
		911	2,
		912	I830_FIFO_LINE_SIZE
		913	};
		914
		915	static const struct intel_watermark_params ironlake_display_wm_info = {
		916	ILK_DISPLAY_FIFO,
		917	ILK_DISPLAY_MAXWM,
		918	ILK_DISPLAY_DFTWM,
		919	2,
		920	ILK_FIFO_LINE_SIZE
		921	};
		922	static const struct intel_watermark_params ironlake_cursor_wm_info = {
		923	ILK_CURSOR_FIFO,
		924	ILK_CURSOR_MAXWM,
		925	ILK_CURSOR_DFTWM,
		926	2,
		927	ILK_FIFO_LINE_SIZE
		928	};
		929	static const struct intel_watermark_params ironlake_display_srwm_info = {
		930	ILK_DISPLAY_SR_FIFO,
		931	ILK_DISPLAY_MAX_SRWM,
		932	ILK_DISPLAY_DFT_SRWM,
		933	2,
		934	ILK_FIFO_LINE_SIZE
		935	};
		936	static const struct intel_watermark_params ironlake_cursor_srwm_info = {
		937	ILK_CURSOR_SR_FIFO,
		938	ILK_CURSOR_MAX_SRWM,
		939	ILK_CURSOR_DFT_SRWM,
		940	2,
		941	ILK_FIFO_LINE_SIZE
		942	};
		943
		944	static const struct intel_watermark_params sandybridge_display_wm_info = {
		945	SNB_DISPLAY_FIFO,
		946	SNB_DISPLAY_MAXWM,
		947	SNB_DISPLAY_DFTWM,
		948	2,
		949	SNB_FIFO_LINE_SIZE
		950	};
		951	static const struct intel_watermark_params sandybridge_cursor_wm_info = {
		952	SNB_CURSOR_FIFO,
		953	SNB_CURSOR_MAXWM,
		954	SNB_CURSOR_DFTWM,
		955	2,
		956	SNB_FIFO_LINE_SIZE
		957	};
		958	static const struct intel_watermark_params sandybridge_display_srwm_info = {
		959	SNB_DISPLAY_SR_FIFO,
		960	SNB_DISPLAY_MAX_SRWM,
		961	SNB_DISPLAY_DFT_SRWM,
		962	2,
		963	SNB_FIFO_LINE_SIZE
		964	};
		965	static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
		966	SNB_CURSOR_SR_FIFO,
		967	SNB_CURSOR_MAX_SRWM,
		968	SNB_CURSOR_DFT_SRWM,
		969	2,
		970	SNB_FIFO_LINE_SIZE
		971	};
		972
		973
		974	/**
		975	* intel_calculate_wm - calculate watermark level
		976	* @clock_in_khz: pixel clock
		977	* @wm: chip FIFO params
		978	* @pixel_size: display pixel size
		979	* @latency_ns: memory latency for the platform
		980	*
		981	* Calculate the watermark level (the level at which the display plane will
		982	* start fetching from memory again). Each chip has a different display
		983	* FIFO size and allocation, so the caller needs to figure that out and pass
		984	* in the correct intel_watermark_params structure.
		985	*
		986	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		987	* on the pixel size. When it reaches the watermark level, it'll start
		988	* fetching FIFO line sized based chunks from memory until the FIFO fills
		989	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		990	* will occur, and a display engine hang could result.
		991	*/
		992	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		993	const struct intel_watermark_params *wm,
		994	int fifo_size,
		995	int pixel_size,
		996	unsigned long latency_ns)
		997	{
		998	long entries_required, wm_size;
		999
		1000	/*
		1001	* Note: we need to make sure we don't overflow for various clock &
		1002	* latency values.
		1003	* clocks go from a few thousand to several hundred thousand.
		1004	* latency is usually a few thousand
		1005	*/
		1006	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		1007	1000;
		1008	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		1009
		1010	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		1011
		1012	wm_size = fifo_size - (entries_required + wm->guard_size);
		1013
		1014	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		1015
		1016	/* Don't promote wm_size to unsigned... */
		1017	if (wm_size > (long)wm->max_wm)
		1018	wm_size = wm->max_wm;
		1019	if (wm_size <= 0)
		1020	wm_size = wm->default_wm;
		1021	return wm_size;
		1022	}
		1023
		1024	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		1025	{
		1026	struct drm_crtc crtc, enabled = NULL;
		1027
		1028	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	1029	if (intel_crtc_active(crtc)) {
3031	serge	1030	if (enabled)
		1031	return NULL;
		1032	enabled = crtc;
		1033	}
		1034	}
		1035
		1036	return enabled;
		1037	}
		1038
		1039	static void pineview_update_wm(struct drm_device *dev)
		1040	{
		1041	struct drm_i915_private *dev_priv = dev->dev_private;
		1042	struct drm_crtc *crtc;
		1043	const struct cxsr_latency *latency;
		1044	u32 reg;
		1045	unsigned long wm;
		1046
		1047	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		1048	dev_priv->fsb_freq, dev_priv->mem_freq);
		1049	if (!latency) {
		1050	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		1051	pineview_disable_cxsr(dev);
		1052	return;
		1053	}
		1054
		1055	crtc = single_enabled_crtc(dev);
		1056	if (crtc) {
		1057	int clock = crtc->mode.clock;
		1058	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1059
		1060	/* Display SR */
		1061	wm = intel_calculate_wm(clock, &pineview_display_wm,
		1062	pineview_display_wm.fifo_size,
		1063	pixel_size, latency->display_sr);
		1064	reg = I915_READ(DSPFW1);
		1065	reg &= ~DSPFW_SR_MASK;
		1066	reg \|= wm << DSPFW_SR_SHIFT;
		1067	I915_WRITE(DSPFW1, reg);
		1068	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		1069
		1070	/* cursor SR */
		1071	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		1072	pineview_display_wm.fifo_size,
		1073	pixel_size, latency->cursor_sr);
		1074	reg = I915_READ(DSPFW3);
		1075	reg &= ~DSPFW_CURSOR_SR_MASK;
		1076	reg \|= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
		1077	I915_WRITE(DSPFW3, reg);
		1078
		1079	/* Display HPLL off SR */
		1080	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		1081	pineview_display_hplloff_wm.fifo_size,
		1082	pixel_size, latency->display_hpll_disable);
		1083	reg = I915_READ(DSPFW3);
		1084	reg &= ~DSPFW_HPLL_SR_MASK;
		1085	reg \|= wm & DSPFW_HPLL_SR_MASK;
		1086	I915_WRITE(DSPFW3, reg);
		1087
		1088	/* cursor HPLL off SR */
		1089	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		1090	pineview_display_hplloff_wm.fifo_size,
		1091	pixel_size, latency->cursor_hpll_disable);
		1092	reg = I915_READ(DSPFW3);
		1093	reg &= ~DSPFW_HPLL_CURSOR_MASK;
		1094	reg \|= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
		1095	I915_WRITE(DSPFW3, reg);
		1096	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		1097
		1098	/* activate cxsr */
		1099	I915_WRITE(DSPFW3,
		1100	I915_READ(DSPFW3) \| PINEVIEW_SELF_REFRESH_EN);
		1101	DRM_DEBUG_KMS("Self-refresh is enabled\n");
		1102	} else {
		1103	pineview_disable_cxsr(dev);
		1104	DRM_DEBUG_KMS("Self-refresh is disabled\n");
		1105	}
		1106	}
		1107
		1108	static bool g4x_compute_wm0(struct drm_device *dev,
		1109	int plane,
		1110	const struct intel_watermark_params *display,
		1111	int display_latency_ns,
		1112	const struct intel_watermark_params *cursor,
		1113	int cursor_latency_ns,
		1114	int *plane_wm,
		1115	int *cursor_wm)
		1116	{
		1117	struct drm_crtc *crtc;
		1118	int htotal, hdisplay, clock, pixel_size;
		1119	int line_time_us, line_count;
		1120	int entries, tlb_miss;
		1121
		1122	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1123	if (!intel_crtc_active(crtc)) {
3031	serge	1124	*cursor_wm = cursor->guard_size;
		1125	*plane_wm = display->guard_size;
		1126	return false;
		1127	}
		1128
		1129	htotal = crtc->mode.htotal;
		1130	hdisplay = crtc->mode.hdisplay;
		1131	clock = crtc->mode.clock;
		1132	pixel_size = crtc->fb->bits_per_pixel / 8;
		1133
		1134	/* Use the small buffer method to calculate plane watermark */
		1135	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1136	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		1137	if (tlb_miss > 0)
		1138	entries += tlb_miss;
		1139	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1140	*plane_wm = entries + display->guard_size;
		1141	if (*plane_wm > (int)display->max_wm)
		1142	*plane_wm = display->max_wm;
		1143
		1144	/* Use the large buffer method to calculate cursor watermark */
		1145	line_time_us = ((htotal * 1000) / clock);
		1146	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
		1147	entries = line_count * 64 * pixel_size;
		1148	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		1149	if (tlb_miss > 0)
		1150	entries += tlb_miss;
		1151	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1152	*cursor_wm = entries + cursor->guard_size;
		1153	if (*cursor_wm > (int)cursor->max_wm)
		1154	*cursor_wm = (int)cursor->max_wm;
		1155
		1156	return true;
		1157	}
		1158
		1159	/*
		1160	* Check the wm result.
		1161	*
		1162	* If any calculated watermark values is larger than the maximum value that
		1163	* can be programmed into the associated watermark register, that watermark
		1164	* must be disabled.
		1165	*/
		1166	static bool g4x_check_srwm(struct drm_device *dev,
		1167	int display_wm, int cursor_wm,
		1168	const struct intel_watermark_params *display,
		1169	const struct intel_watermark_params *cursor)
		1170	{
		1171	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		1172	display_wm, cursor_wm);
		1173
		1174	if (display_wm > display->max_wm) {
		1175	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		1176	display_wm, display->max_wm);
		1177	return false;
		1178	}
		1179
		1180	if (cursor_wm > cursor->max_wm) {
		1181	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		1182	cursor_wm, cursor->max_wm);
		1183	return false;
		1184	}
		1185
		1186	if (!(display_wm \|\| cursor_wm)) {
		1187	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		1188	return false;
		1189	}
		1190
		1191	return true;
		1192	}
		1193
		1194	static bool g4x_compute_srwm(struct drm_device *dev,
		1195	int plane,
		1196	int latency_ns,
		1197	const struct intel_watermark_params *display,
		1198	const struct intel_watermark_params *cursor,
		1199	int display_wm, int cursor_wm)
		1200	{
		1201	struct drm_crtc *crtc;
		1202	int hdisplay, htotal, pixel_size, clock;
		1203	unsigned long line_time_us;
		1204	int line_count, line_size;
		1205	int small, large;
		1206	int entries;
		1207
		1208	if (!latency_ns) {
		1209	display_wm = cursor_wm = 0;
		1210	return false;
		1211	}
		1212
		1213	crtc = intel_get_crtc_for_plane(dev, plane);
		1214	hdisplay = crtc->mode.hdisplay;
		1215	htotal = crtc->mode.htotal;
		1216	clock = crtc->mode.clock;
		1217	pixel_size = crtc->fb->bits_per_pixel / 8;
		1218
		1219	line_time_us = (htotal * 1000) / clock;
		1220	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1221	line_size = hdisplay * pixel_size;
		1222
		1223	/* Use the minimum of the small and large buffer method for primary */
		1224	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1225	large = line_count * line_size;
		1226
		1227	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1228	*display_wm = entries + display->guard_size;
		1229
		1230	/* calculate the self-refresh watermark for display cursor */
		1231	entries = line_count * pixel_size * 64;
		1232	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1233	*cursor_wm = entries + cursor->guard_size;
		1234
		1235	return g4x_check_srwm(dev,
		1236	display_wm, cursor_wm,
		1237	display, cursor);
		1238	}
		1239
		1240	static bool vlv_compute_drain_latency(struct drm_device *dev,
		1241	int plane,
		1242	int *plane_prec_mult,
		1243	int *plane_dl,
		1244	int *cursor_prec_mult,
		1245	int *cursor_dl)
		1246	{
		1247	struct drm_crtc *crtc;
		1248	int clock, pixel_size;
		1249	int entries;
		1250
		1251	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1252	if (!intel_crtc_active(crtc))
3031	serge	1253	return false;
		1254
		1255	clock = crtc->mode.clock; /* VESA DOT Clock */
		1256	pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */
		1257
		1258	entries = (clock / 1000) * pixel_size;
		1259	*plane_prec_mult = (entries > 256) ?
		1260	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1261	plane_dl = (64 (plane_prec_mult) 4) / ((clock / 1000) *
		1262	pixel_size);
		1263
		1264	entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
		1265	*cursor_prec_mult = (entries > 256) ?
		1266	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1267	cursor_dl = (64 (cursor_prec_mult) 4) / ((clock / 1000) * 4);
		1268
		1269	return true;
		1270	}
		1271
		1272	/*
		1273	* Update drain latency registers of memory arbiter
		1274	*
		1275	* Valleyview SoC has a new memory arbiter and needs drain latency registers
		1276	* to be programmed. Each plane has a drain latency multiplier and a drain
		1277	* latency value.
		1278	*/
		1279
		1280	static void vlv_update_drain_latency(struct drm_device *dev)
		1281	{
		1282	struct drm_i915_private *dev_priv = dev->dev_private;
		1283	int planea_prec, planea_dl, planeb_prec, planeb_dl;
		1284	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
		1285	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
		1286	either 16 or 32 */
		1287
		1288	/* For plane A, Cursor A */
		1289	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
		1290	&cursor_prec_mult, &cursora_dl)) {
		1291	cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1292	DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
		1293	planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1294	DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
		1295
		1296	I915_WRITE(VLV_DDL1, cursora_prec \|
		1297	(cursora_dl << DDL_CURSORA_SHIFT) \|
		1298	planea_prec \| planea_dl);
		1299	}
		1300
		1301	/* For plane B, Cursor B */
		1302	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
		1303	&cursor_prec_mult, &cursorb_dl)) {
		1304	cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1305	DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
		1306	planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1307	DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
		1308
		1309	I915_WRITE(VLV_DDL2, cursorb_prec \|
		1310	(cursorb_dl << DDL_CURSORB_SHIFT) \|
		1311	planeb_prec \| planeb_dl);
		1312	}
		1313	}
		1314
		1315	#define single_plane_enabled(mask) is_power_of_2(mask)
		1316
		1317	static void valleyview_update_wm(struct drm_device *dev)
		1318	{
		1319	static const int sr_latency_ns = 12000;
		1320	struct drm_i915_private *dev_priv = dev->dev_private;
		1321	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1322	int plane_sr, cursor_sr;
3243	Serge	1323	int ignore_plane_sr, ignore_cursor_sr;
3031	serge	1324	unsigned int enabled = 0;
		1325
		1326	vlv_update_drain_latency(dev);
		1327
		1328	if (g4x_compute_wm0(dev, 0,
		1329	&valleyview_wm_info, latency_ns,
		1330	&valleyview_cursor_wm_info, latency_ns,
		1331	&planea_wm, &cursora_wm))
		1332	enabled \|= 1;
		1333
		1334	if (g4x_compute_wm0(dev, 1,
		1335	&valleyview_wm_info, latency_ns,
		1336	&valleyview_cursor_wm_info, latency_ns,
		1337	&planeb_wm, &cursorb_wm))
		1338	enabled \|= 2;
		1339
		1340	if (single_plane_enabled(enabled) &&
		1341	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1342	sr_latency_ns,
		1343	&valleyview_wm_info,
		1344	&valleyview_cursor_wm_info,
3243	Serge	1345	&plane_sr, &ignore_cursor_sr) &&
		1346	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1347	2*sr_latency_ns,
		1348	&valleyview_wm_info,
		1349	&valleyview_cursor_wm_info,
		1350	&ignore_plane_sr, &cursor_sr)) {
3031	serge	1351	I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
3243	Serge	1352	} else {
3031	serge	1353	I915_WRITE(FW_BLC_SELF_VLV,
		1354	I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
3243	Serge	1355	plane_sr = cursor_sr = 0;
		1356	}
3031	serge	1357
		1358	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1359	planea_wm, cursora_wm,
		1360	planeb_wm, cursorb_wm,
		1361	plane_sr, cursor_sr);
		1362
		1363	I915_WRITE(DSPFW1,
		1364	(plane_sr << DSPFW_SR_SHIFT) \|
		1365	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1366	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1367	planea_wm);
		1368	I915_WRITE(DSPFW2,
3243	Serge	1369	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1370	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1371	I915_WRITE(DSPFW3,
3243	Serge	1372	(I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) \|
		1373	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
3031	serge	1374	}
		1375
		1376	static void g4x_update_wm(struct drm_device *dev)
		1377	{
		1378	static const int sr_latency_ns = 12000;
		1379	struct drm_i915_private *dev_priv = dev->dev_private;
		1380	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1381	int plane_sr, cursor_sr;
		1382	unsigned int enabled = 0;
		1383
		1384	if (g4x_compute_wm0(dev, 0,
		1385	&g4x_wm_info, latency_ns,
		1386	&g4x_cursor_wm_info, latency_ns,
		1387	&planea_wm, &cursora_wm))
		1388	enabled \|= 1;
		1389
		1390	if (g4x_compute_wm0(dev, 1,
		1391	&g4x_wm_info, latency_ns,
		1392	&g4x_cursor_wm_info, latency_ns,
		1393	&planeb_wm, &cursorb_wm))
		1394	enabled \|= 2;
		1395
		1396	if (single_plane_enabled(enabled) &&
		1397	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1398	sr_latency_ns,
		1399	&g4x_wm_info,
		1400	&g4x_cursor_wm_info,
3243	Serge	1401	&plane_sr, &cursor_sr)) {
3031	serge	1402	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
3243	Serge	1403	} else {
3031	serge	1404	I915_WRITE(FW_BLC_SELF,
		1405	I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
3243	Serge	1406	plane_sr = cursor_sr = 0;
		1407	}
3031	serge	1408
		1409	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1410	planea_wm, cursora_wm,
		1411	planeb_wm, cursorb_wm,
		1412	plane_sr, cursor_sr);
		1413
		1414	I915_WRITE(DSPFW1,
		1415	(plane_sr << DSPFW_SR_SHIFT) \|
		1416	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1417	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1418	planea_wm);
		1419	I915_WRITE(DSPFW2,
3243	Serge	1420	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1421	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1422	/* HPLL off in SR has some issues on G4x... disable it */
		1423	I915_WRITE(DSPFW3,
3243	Serge	1424	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
3031	serge	1425	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1426	}
		1427
		1428	static void i965_update_wm(struct drm_device *dev)
		1429	{
		1430	struct drm_i915_private *dev_priv = dev->dev_private;
		1431	struct drm_crtc *crtc;
		1432	int srwm = 1;
		1433	int cursor_sr = 16;
		1434
		1435	/* Calc sr entries for one plane configs */
		1436	crtc = single_enabled_crtc(dev);
		1437	if (crtc) {
		1438	/* self-refresh has much higher latency */
		1439	static const int sr_latency_ns = 12000;
		1440	int clock = crtc->mode.clock;
		1441	int htotal = crtc->mode.htotal;
		1442	int hdisplay = crtc->mode.hdisplay;
		1443	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1444	unsigned long line_time_us;
		1445	int entries;
		1446
		1447	line_time_us = ((htotal * 1000) / clock);
		1448
		1449	/* Use ns/us then divide to preserve precision */
		1450	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1451	pixel_size * hdisplay;
		1452	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1453	srwm = I965_FIFO_SIZE - entries;
		1454	if (srwm < 0)
		1455	srwm = 1;
		1456	srwm &= 0x1ff;
		1457	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1458	entries, srwm);
		1459
		1460	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1461	pixel_size * 64;
		1462	entries = DIV_ROUND_UP(entries,
		1463	i965_cursor_wm_info.cacheline_size);
		1464	cursor_sr = i965_cursor_wm_info.fifo_size -
		1465	(entries + i965_cursor_wm_info.guard_size);
		1466
		1467	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1468	cursor_sr = i965_cursor_wm_info.max_wm;
		1469
		1470	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1471	"cursor %d\n", srwm, cursor_sr);
		1472
		1473	if (IS_CRESTLINE(dev))
		1474	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
		1475	} else {
		1476	/* Turn off self refresh if both pipes are enabled */
		1477	if (IS_CRESTLINE(dev))
		1478	I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
		1479	& ~FW_BLC_SELF_EN);
		1480	}
		1481
		1482	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1483	srwm);
		1484
		1485	/* 965 has limitations... */
		1486	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) \|
		1487	(8 << 16) \| (8 << 8) \| (8 << 0));
		1488	I915_WRITE(DSPFW2, (8 << 8) \| (8 << 0));
		1489	/* update cursor SR watermark */
		1490	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1491	}
		1492
		1493	static void i9xx_update_wm(struct drm_device *dev)
		1494	{
		1495	struct drm_i915_private *dev_priv = dev->dev_private;
		1496	const struct intel_watermark_params *wm_info;
		1497	uint32_t fwater_lo;
		1498	uint32_t fwater_hi;
		1499	int cwm, srwm = 1;
		1500	int fifo_size;
		1501	int planea_wm, planeb_wm;
		1502	struct drm_crtc crtc, enabled = NULL;
		1503
		1504	if (IS_I945GM(dev))
		1505	wm_info = &i945_wm_info;
		1506	else if (!IS_GEN2(dev))
		1507	wm_info = &i915_wm_info;
		1508	else
		1509	wm_info = &i855_wm_info;
		1510
		1511	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1512	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1513	if (intel_crtc_active(crtc)) {
		1514	int cpp = crtc->fb->bits_per_pixel / 8;
		1515	if (IS_GEN2(dev))
		1516	cpp = 4;
		1517
3031	serge	1518	planea_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1519	wm_info, fifo_size, cpp,
3031	serge	1520	latency_ns);
		1521	enabled = crtc;
		1522	} else
		1523	planea_wm = fifo_size - wm_info->guard_size;
		1524
		1525	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1526	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1527	if (intel_crtc_active(crtc)) {
		1528	int cpp = crtc->fb->bits_per_pixel / 8;
		1529	if (IS_GEN2(dev))
		1530	cpp = 4;
		1531
3031	serge	1532	planeb_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1533	wm_info, fifo_size, cpp,
3031	serge	1534	latency_ns);
		1535	if (enabled == NULL)
		1536	enabled = crtc;
		1537	else
		1538	enabled = NULL;
		1539	} else
		1540	planeb_wm = fifo_size - wm_info->guard_size;
		1541
		1542	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1543
		1544	/*
		1545	* Overlay gets an aggressive default since video jitter is bad.
		1546	*/
		1547	cwm = 2;
		1548
		1549	/* Play safe and disable self-refresh before adjusting watermarks. */
		1550	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1551	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK \| 0);
		1552	else if (IS_I915GM(dev))
		1553	I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
		1554
		1555	/* Calc sr entries for one plane configs */
		1556	if (HAS_FW_BLC(dev) && enabled) {
		1557	/* self-refresh has much higher latency */
		1558	static const int sr_latency_ns = 6000;
		1559	int clock = enabled->mode.clock;
		1560	int htotal = enabled->mode.htotal;
		1561	int hdisplay = enabled->mode.hdisplay;
		1562	int pixel_size = enabled->fb->bits_per_pixel / 8;
		1563	unsigned long line_time_us;
		1564	int entries;
		1565
		1566	line_time_us = (htotal * 1000) / clock;
		1567
		1568	/* Use ns/us then divide to preserve precision */
		1569	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1570	pixel_size * hdisplay;
		1571	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1572	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1573	srwm = wm_info->fifo_size - entries;
		1574	if (srwm < 0)
		1575	srwm = 1;
		1576
		1577	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1578	I915_WRITE(FW_BLC_SELF,
		1579	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1580	else if (IS_I915GM(dev))
		1581	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1582	}
		1583
		1584	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1585	planea_wm, planeb_wm, cwm, srwm);
		1586
		1587	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1588	fwater_hi = (cwm & 0x1f);
		1589
		1590	/* Set request length to 8 cachelines per fetch */
		1591	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1592	fwater_hi = fwater_hi \| (1 << 8);
		1593
		1594	I915_WRITE(FW_BLC, fwater_lo);
		1595	I915_WRITE(FW_BLC2, fwater_hi);
		1596
		1597	if (HAS_FW_BLC(dev)) {
		1598	if (enabled) {
		1599	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1600	I915_WRITE(FW_BLC_SELF,
		1601	FW_BLC_SELF_EN_MASK \| FW_BLC_SELF_EN);
		1602	else if (IS_I915GM(dev))
		1603	I915_WRITE(INSTPM, I915_READ(INSTPM) \| INSTPM_SELF_EN);
		1604	DRM_DEBUG_KMS("memory self refresh enabled\n");
		1605	} else
		1606	DRM_DEBUG_KMS("memory self refresh disabled\n");
		1607	}
		1608	}
		1609
		1610	static void i830_update_wm(struct drm_device *dev)
		1611	{
		1612	struct drm_i915_private *dev_priv = dev->dev_private;
		1613	struct drm_crtc *crtc;
		1614	uint32_t fwater_lo;
		1615	int planea_wm;
		1616
		1617	crtc = single_enabled_crtc(dev);
		1618	if (crtc == NULL)
		1619	return;
		1620
		1621	planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
		1622	dev_priv->display.get_fifo_size(dev, 0),
3243	Serge	1623	4, latency_ns);
3031	serge	1624	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1625	fwater_lo \|= (3<<8) \| planea_wm;
		1626
		1627	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1628
		1629	I915_WRITE(FW_BLC, fwater_lo);
		1630	}
		1631
		1632	#define ILK_LP0_PLANE_LATENCY 700
		1633	#define ILK_LP0_CURSOR_LATENCY 1300
		1634
		1635	/*
		1636	* Check the wm result.
		1637	*
		1638	* If any calculated watermark values is larger than the maximum value that
		1639	* can be programmed into the associated watermark register, that watermark
		1640	* must be disabled.
		1641	*/
		1642	static bool ironlake_check_srwm(struct drm_device *dev, int level,
		1643	int fbc_wm, int display_wm, int cursor_wm,
		1644	const struct intel_watermark_params *display,
		1645	const struct intel_watermark_params *cursor)
		1646	{
		1647	struct drm_i915_private *dev_priv = dev->dev_private;
		1648
		1649	DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
		1650	" cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
		1651
		1652	if (fbc_wm > SNB_FBC_MAX_SRWM) {
		1653	DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
		1654	fbc_wm, SNB_FBC_MAX_SRWM, level);
		1655
		1656	/* fbc has it's own way to disable FBC WM */
		1657	I915_WRITE(DISP_ARB_CTL,
		1658	I915_READ(DISP_ARB_CTL) \| DISP_FBC_WM_DIS);
		1659	return false;
		1660	}
		1661
		1662	if (display_wm > display->max_wm) {
		1663	DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
		1664	display_wm, SNB_DISPLAY_MAX_SRWM, level);
		1665	return false;
		1666	}
		1667
		1668	if (cursor_wm > cursor->max_wm) {
		1669	DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
		1670	cursor_wm, SNB_CURSOR_MAX_SRWM, level);
		1671	return false;
		1672	}
		1673
		1674	if (!(fbc_wm \|\| display_wm \|\| cursor_wm)) {
		1675	DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
		1676	return false;
		1677	}
		1678
		1679	return true;
		1680	}
		1681
		1682	/*
		1683	* Compute watermark values of WM[1-3],
		1684	*/
		1685	static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
		1686	int latency_ns,
		1687	const struct intel_watermark_params *display,
		1688	const struct intel_watermark_params *cursor,
		1689	int fbc_wm, int display_wm, int *cursor_wm)
		1690	{
		1691	struct drm_crtc *crtc;
		1692	unsigned long line_time_us;
		1693	int hdisplay, htotal, pixel_size, clock;
		1694	int line_count, line_size;
		1695	int small, large;
		1696	int entries;
		1697
		1698	if (!latency_ns) {
		1699	fbc_wm = display_wm = *cursor_wm = 0;
		1700	return false;
		1701	}
		1702
		1703	crtc = intel_get_crtc_for_plane(dev, plane);
		1704	hdisplay = crtc->mode.hdisplay;
		1705	htotal = crtc->mode.htotal;
		1706	clock = crtc->mode.clock;
		1707	pixel_size = crtc->fb->bits_per_pixel / 8;
		1708
		1709	line_time_us = (htotal * 1000) / clock;
		1710	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1711	line_size = hdisplay * pixel_size;
		1712
		1713	/* Use the minimum of the small and large buffer method for primary */
		1714	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1715	large = line_count * line_size;
		1716
		1717	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1718	*display_wm = entries + display->guard_size;
		1719
		1720	/*
		1721	* Spec says:
		1722	* FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
		1723	*/
		1724	fbc_wm = DIV_ROUND_UP(display_wm * 64, line_size) + 2;
		1725
		1726	/* calculate the self-refresh watermark for display cursor */
		1727	entries = line_count * pixel_size * 64;
		1728	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1729	*cursor_wm = entries + cursor->guard_size;
		1730
		1731	return ironlake_check_srwm(dev, level,
		1732	fbc_wm, display_wm, *cursor_wm,
		1733	display, cursor);
		1734	}
		1735
		1736	static void ironlake_update_wm(struct drm_device *dev)
		1737	{
		1738	struct drm_i915_private *dev_priv = dev->dev_private;
		1739	int fbc_wm, plane_wm, cursor_wm;
		1740	unsigned int enabled;
		1741
		1742	enabled = 0;
		1743	if (g4x_compute_wm0(dev, 0,
		1744	&ironlake_display_wm_info,
		1745	ILK_LP0_PLANE_LATENCY,
		1746	&ironlake_cursor_wm_info,
		1747	ILK_LP0_CURSOR_LATENCY,
		1748	&plane_wm, &cursor_wm)) {
		1749	I915_WRITE(WM0_PIPEA_ILK,
		1750	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1751	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1752	" plane %d, " "cursor: %d\n",
		1753	plane_wm, cursor_wm);
		1754	enabled \|= 1;
		1755	}
		1756
		1757	if (g4x_compute_wm0(dev, 1,
		1758	&ironlake_display_wm_info,
		1759	ILK_LP0_PLANE_LATENCY,
		1760	&ironlake_cursor_wm_info,
		1761	ILK_LP0_CURSOR_LATENCY,
		1762	&plane_wm, &cursor_wm)) {
		1763	I915_WRITE(WM0_PIPEB_ILK,
		1764	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1765	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1766	" plane %d, cursor: %d\n",
		1767	plane_wm, cursor_wm);
		1768	enabled \|= 2;
		1769	}
		1770
		1771	/*
		1772	* Calculate and update the self-refresh watermark only when one
		1773	* display plane is used.
		1774	*/
		1775	I915_WRITE(WM3_LP_ILK, 0);
		1776	I915_WRITE(WM2_LP_ILK, 0);
		1777	I915_WRITE(WM1_LP_ILK, 0);
		1778
		1779	if (!single_plane_enabled(enabled))
		1780	return;
		1781	enabled = ffs(enabled) - 1;
		1782
		1783	/* WM1 */
		1784	if (!ironlake_compute_srwm(dev, 1, enabled,
		1785	ILK_READ_WM1_LATENCY() * 500,
		1786	&ironlake_display_srwm_info,
		1787	&ironlake_cursor_srwm_info,
		1788	&fbc_wm, &plane_wm, &cursor_wm))
		1789	return;
		1790
		1791	I915_WRITE(WM1_LP_ILK,
		1792	WM1_LP_SR_EN \|
		1793	(ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1794	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1795	(plane_wm << WM1_LP_SR_SHIFT) \|
		1796	cursor_wm);
		1797
		1798	/* WM2 */
		1799	if (!ironlake_compute_srwm(dev, 2, enabled,
		1800	ILK_READ_WM2_LATENCY() * 500,
		1801	&ironlake_display_srwm_info,
		1802	&ironlake_cursor_srwm_info,
		1803	&fbc_wm, &plane_wm, &cursor_wm))
		1804	return;
		1805
		1806	I915_WRITE(WM2_LP_ILK,
		1807	WM2_LP_EN \|
		1808	(ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1809	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1810	(plane_wm << WM1_LP_SR_SHIFT) \|
		1811	cursor_wm);
		1812
		1813	/*
		1814	* WM3 is unsupported on ILK, probably because we don't have latency
		1815	* data for that power state
		1816	*/
		1817	}
		1818
		1819	static void sandybridge_update_wm(struct drm_device *dev)
		1820	{
		1821	struct drm_i915_private *dev_priv = dev->dev_private;
		1822	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		1823	u32 val;
		1824	int fbc_wm, plane_wm, cursor_wm;
		1825	unsigned int enabled;
		1826
		1827	enabled = 0;
		1828	if (g4x_compute_wm0(dev, 0,
		1829	&sandybridge_display_wm_info, latency,
		1830	&sandybridge_cursor_wm_info, latency,
		1831	&plane_wm, &cursor_wm)) {
		1832	val = I915_READ(WM0_PIPEA_ILK);
		1833	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1834	I915_WRITE(WM0_PIPEA_ILK, val \|
		1835	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1836	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1837	" plane %d, " "cursor: %d\n",
		1838	plane_wm, cursor_wm);
		1839	enabled \|= 1;
		1840	}
		1841
		1842	if (g4x_compute_wm0(dev, 1,
		1843	&sandybridge_display_wm_info, latency,
		1844	&sandybridge_cursor_wm_info, latency,
		1845	&plane_wm, &cursor_wm)) {
		1846	val = I915_READ(WM0_PIPEB_ILK);
		1847	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1848	I915_WRITE(WM0_PIPEB_ILK, val \|
		1849	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1850	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1851	" plane %d, cursor: %d\n",
		1852	plane_wm, cursor_wm);
		1853	enabled \|= 2;
		1854	}
		1855
3243	Serge	1856	/*
		1857	* Calculate and update the self-refresh watermark only when one
		1858	* display plane is used.
		1859	*
		1860	* SNB support 3 levels of watermark.
		1861	*
		1862	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1863	* and disabled in the descending order
		1864	*
		1865	*/
		1866	I915_WRITE(WM3_LP_ILK, 0);
		1867	I915_WRITE(WM2_LP_ILK, 0);
		1868	I915_WRITE(WM1_LP_ILK, 0);
		1869
		1870	if (!single_plane_enabled(enabled) \|\|
		1871	dev_priv->sprite_scaling_enabled)
		1872	return;
		1873	enabled = ffs(enabled) - 1;
		1874
		1875	/* WM1 */
		1876	if (!ironlake_compute_srwm(dev, 1, enabled,
		1877	SNB_READ_WM1_LATENCY() * 500,
		1878	&sandybridge_display_srwm_info,
		1879	&sandybridge_cursor_srwm_info,
		1880	&fbc_wm, &plane_wm, &cursor_wm))
		1881	return;
		1882
		1883	I915_WRITE(WM1_LP_ILK,
		1884	WM1_LP_SR_EN \|
		1885	(SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1886	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1887	(plane_wm << WM1_LP_SR_SHIFT) \|
		1888	cursor_wm);
		1889
		1890	/* WM2 */
		1891	if (!ironlake_compute_srwm(dev, 2, enabled,
		1892	SNB_READ_WM2_LATENCY() * 500,
		1893	&sandybridge_display_srwm_info,
		1894	&sandybridge_cursor_srwm_info,
		1895	&fbc_wm, &plane_wm, &cursor_wm))
		1896	return;
		1897
		1898	I915_WRITE(WM2_LP_ILK,
		1899	WM2_LP_EN \|
		1900	(SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1901	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1902	(plane_wm << WM1_LP_SR_SHIFT) \|
		1903	cursor_wm);
		1904
		1905	/* WM3 */
		1906	if (!ironlake_compute_srwm(dev, 3, enabled,
		1907	SNB_READ_WM3_LATENCY() * 500,
		1908	&sandybridge_display_srwm_info,
		1909	&sandybridge_cursor_srwm_info,
		1910	&fbc_wm, &plane_wm, &cursor_wm))
		1911	return;
		1912
		1913	I915_WRITE(WM3_LP_ILK,
		1914	WM3_LP_EN \|
		1915	(SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1916	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1917	(plane_wm << WM1_LP_SR_SHIFT) \|
		1918	cursor_wm);
		1919	}
		1920
		1921	static void ivybridge_update_wm(struct drm_device *dev)
		1922	{
		1923	struct drm_i915_private *dev_priv = dev->dev_private;
		1924	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		1925	u32 val;
		1926	int fbc_wm, plane_wm, cursor_wm;
		1927	int ignore_fbc_wm, ignore_plane_wm, ignore_cursor_wm;
		1928	unsigned int enabled;
		1929
		1930	enabled = 0;
		1931	if (g4x_compute_wm0(dev, 0,
3031	serge	1932	&sandybridge_display_wm_info, latency,
		1933	&sandybridge_cursor_wm_info, latency,
		1934	&plane_wm, &cursor_wm)) {
3243	Serge	1935	val = I915_READ(WM0_PIPEA_ILK);
		1936	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1937	I915_WRITE(WM0_PIPEA_ILK, val \|
		1938	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1939	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1940	" plane %d, " "cursor: %d\n",
		1941	plane_wm, cursor_wm);
		1942	enabled \|= 1;
		1943	}
		1944
		1945	if (g4x_compute_wm0(dev, 1,
		1946	&sandybridge_display_wm_info, latency,
		1947	&sandybridge_cursor_wm_info, latency,
		1948	&plane_wm, &cursor_wm)) {
		1949	val = I915_READ(WM0_PIPEB_ILK);
		1950	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1951	I915_WRITE(WM0_PIPEB_ILK, val \|
		1952	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1953	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1954	" plane %d, cursor: %d\n",
		1955	plane_wm, cursor_wm);
		1956	enabled \|= 2;
		1957	}
		1958
		1959	if (g4x_compute_wm0(dev, 2,
		1960	&sandybridge_display_wm_info, latency,
		1961	&sandybridge_cursor_wm_info, latency,
		1962	&plane_wm, &cursor_wm)) {
3031	serge	1963	val = I915_READ(WM0_PIPEC_IVB);
		1964	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1965	I915_WRITE(WM0_PIPEC_IVB, val \|
		1966	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1967	DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
		1968	" plane %d, cursor: %d\n",
		1969	plane_wm, cursor_wm);
		1970	enabled \|= 3;
		1971	}
		1972
		1973	/*
		1974	* Calculate and update the self-refresh watermark only when one
		1975	* display plane is used.
		1976	*
		1977	* SNB support 3 levels of watermark.
		1978	*
		1979	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1980	* and disabled in the descending order
		1981	*
		1982	*/
		1983	I915_WRITE(WM3_LP_ILK, 0);
		1984	I915_WRITE(WM2_LP_ILK, 0);
		1985	I915_WRITE(WM1_LP_ILK, 0);
		1986
		1987	if (!single_plane_enabled(enabled) \|\|
		1988	dev_priv->sprite_scaling_enabled)
		1989	return;
		1990	enabled = ffs(enabled) - 1;
		1991
		1992	/* WM1 */
		1993	if (!ironlake_compute_srwm(dev, 1, enabled,
		1994	SNB_READ_WM1_LATENCY() * 500,
		1995	&sandybridge_display_srwm_info,
		1996	&sandybridge_cursor_srwm_info,
		1997	&fbc_wm, &plane_wm, &cursor_wm))
		1998	return;
		1999
		2000	I915_WRITE(WM1_LP_ILK,
		2001	WM1_LP_SR_EN \|
		2002	(SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2003	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2004	(plane_wm << WM1_LP_SR_SHIFT) \|
		2005	cursor_wm);
		2006
		2007	/* WM2 */
		2008	if (!ironlake_compute_srwm(dev, 2, enabled,
		2009	SNB_READ_WM2_LATENCY() * 500,
		2010	&sandybridge_display_srwm_info,
		2011	&sandybridge_cursor_srwm_info,
		2012	&fbc_wm, &plane_wm, &cursor_wm))
		2013	return;
		2014
		2015	I915_WRITE(WM2_LP_ILK,
		2016	WM2_LP_EN \|
		2017	(SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2018	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2019	(plane_wm << WM1_LP_SR_SHIFT) \|
		2020	cursor_wm);
		2021
3243	Serge	2022	/* WM3, note we have to correct the cursor latency */
3031	serge	2023	if (!ironlake_compute_srwm(dev, 3, enabled,
		2024	SNB_READ_WM3_LATENCY() * 500,
		2025	&sandybridge_display_srwm_info,
		2026	&sandybridge_cursor_srwm_info,
3243	Serge	2027	&fbc_wm, &plane_wm, &ignore_cursor_wm) \|\|
		2028	!ironlake_compute_srwm(dev, 3, enabled,
		2029	2 * SNB_READ_WM3_LATENCY() * 500,
		2030	&sandybridge_display_srwm_info,
		2031	&sandybridge_cursor_srwm_info,
		2032	&ignore_fbc_wm, &ignore_plane_wm, &cursor_wm))
3031	serge	2033	return;
		2034
		2035	I915_WRITE(WM3_LP_ILK,
		2036	WM3_LP_EN \|
		2037	(SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2038	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2039	(plane_wm << WM1_LP_SR_SHIFT) \|
		2040	cursor_wm);
		2041	}
		2042
		2043	static void
		2044	haswell_update_linetime_wm(struct drm_device *dev, int pipe,
		2045	struct drm_display_mode *mode)
		2046	{
		2047	struct drm_i915_private *dev_priv = dev->dev_private;
		2048	u32 temp;
		2049
		2050	temp = I915_READ(PIPE_WM_LINETIME(pipe));
		2051	temp &= ~PIPE_WM_LINETIME_MASK;
		2052
		2053	/* The WM are computed with base on how long it takes to fill a single
		2054	* row at the given clock rate, multiplied by 8.
		2055	* */
		2056	temp \|= PIPE_WM_LINETIME_TIME(
		2057	((mode->crtc_hdisplay * 1000) / mode->clock) * 8);
		2058
		2059	/* IPS watermarks are only used by pipe A, and are ignored by
		2060	* pipes B and C. They are calculated similarly to the common
		2061	* linetime values, except that we are using CD clock frequency
		2062	* in MHz instead of pixel rate for the division.
		2063	*
		2064	* This is a placeholder for the IPS watermark calculation code.
		2065	*/
		2066
		2067	I915_WRITE(PIPE_WM_LINETIME(pipe), temp);
		2068	}
		2069
		2070	static bool
		2071	sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
		2072	uint32_t sprite_width, int pixel_size,
		2073	const struct intel_watermark_params *display,
		2074	int display_latency_ns, int *sprite_wm)
		2075	{
		2076	struct drm_crtc *crtc;
		2077	int clock;
		2078	int entries, tlb_miss;
		2079
		2080	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	2081	if (!intel_crtc_active(crtc)) {
3031	serge	2082	*sprite_wm = display->guard_size;
		2083	return false;
		2084	}
		2085
		2086	clock = crtc->mode.clock;
		2087
		2088	/* Use the small buffer method to calculate the sprite watermark */
		2089	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		2090	tlb_miss = display->fifo_size*display->cacheline_size -
		2091	sprite_width * 8;
		2092	if (tlb_miss > 0)
		2093	entries += tlb_miss;
		2094	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		2095	*sprite_wm = entries + display->guard_size;
		2096	if (*sprite_wm > (int)display->max_wm)
		2097	*sprite_wm = display->max_wm;
		2098
		2099	return true;
		2100	}
		2101
		2102	static bool
		2103	sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
		2104	uint32_t sprite_width, int pixel_size,
		2105	const struct intel_watermark_params *display,
		2106	int latency_ns, int *sprite_wm)
		2107	{
		2108	struct drm_crtc *crtc;
		2109	unsigned long line_time_us;
		2110	int clock;
		2111	int line_count, line_size;
		2112	int small, large;
		2113	int entries;
		2114
		2115	if (!latency_ns) {
		2116	*sprite_wm = 0;
		2117	return false;
		2118	}
		2119
		2120	crtc = intel_get_crtc_for_plane(dev, plane);
		2121	clock = crtc->mode.clock;
		2122	if (!clock) {
		2123	*sprite_wm = 0;
		2124	return false;
		2125	}
		2126
		2127	line_time_us = (sprite_width * 1000) / clock;
		2128	if (!line_time_us) {
		2129	*sprite_wm = 0;
		2130	return false;
		2131	}
		2132
		2133	line_count = (latency_ns / line_time_us + 1000) / 1000;
		2134	line_size = sprite_width * pixel_size;
		2135
		2136	/* Use the minimum of the small and large buffer method for primary */
		2137	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		2138	large = line_count * line_size;
		2139
		2140	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		2141	*sprite_wm = entries + display->guard_size;
		2142
		2143	return *sprite_wm > 0x3ff ? false : true;
		2144	}
		2145
		2146	static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
		2147	uint32_t sprite_width, int pixel_size)
		2148	{
		2149	struct drm_i915_private *dev_priv = dev->dev_private;
		2150	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		2151	u32 val;
		2152	int sprite_wm, reg;
		2153	int ret;
		2154
		2155	switch (pipe) {
		2156	case 0:
		2157	reg = WM0_PIPEA_ILK;
		2158	break;
		2159	case 1:
		2160	reg = WM0_PIPEB_ILK;
		2161	break;
		2162	case 2:
		2163	reg = WM0_PIPEC_IVB;
		2164	break;
		2165	default:
		2166	return; /* bad pipe */
		2167	}
		2168
		2169	ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
		2170	&sandybridge_display_wm_info,
		2171	latency, &sprite_wm);
		2172	if (!ret) {
		2173	DRM_DEBUG_KMS("failed to compute sprite wm for pipe %d\n",
		2174	pipe);
		2175	return;
		2176	}
		2177
		2178	val = I915_READ(reg);
		2179	val &= ~WM0_PIPE_SPRITE_MASK;
		2180	I915_WRITE(reg, val \| (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
		2181	DRM_DEBUG_KMS("sprite watermarks For pipe %d - %d\n", pipe, sprite_wm);
		2182
		2183
		2184	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2185	pixel_size,
		2186	&sandybridge_display_srwm_info,
		2187	SNB_READ_WM1_LATENCY() * 500,
		2188	&sprite_wm);
		2189	if (!ret) {
		2190	DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %d\n",
		2191	pipe);
		2192	return;
		2193	}
		2194	I915_WRITE(WM1S_LP_ILK, sprite_wm);
		2195
		2196	/* Only IVB has two more LP watermarks for sprite */
		2197	if (!IS_IVYBRIDGE(dev))
		2198	return;
		2199
		2200	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2201	pixel_size,
		2202	&sandybridge_display_srwm_info,
		2203	SNB_READ_WM2_LATENCY() * 500,
		2204	&sprite_wm);
		2205	if (!ret) {
		2206	DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %d\n",
		2207	pipe);
		2208	return;
		2209	}
		2210	I915_WRITE(WM2S_LP_IVB, sprite_wm);
		2211
		2212	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2213	pixel_size,
		2214	&sandybridge_display_srwm_info,
		2215	SNB_READ_WM3_LATENCY() * 500,
		2216	&sprite_wm);
		2217	if (!ret) {
		2218	DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %d\n",
		2219	pipe);
		2220	return;
		2221	}
		2222	I915_WRITE(WM3S_LP_IVB, sprite_wm);
		2223	}
		2224
		2225	/**
		2226	* intel_update_watermarks - update FIFO watermark values based on current modes
		2227	*
		2228	* Calculate watermark values for the various WM regs based on current mode
		2229	* and plane configuration.
		2230	*
		2231	* There are several cases to deal with here:
		2232	* - normal (i.e. non-self-refresh)
		2233	* - self-refresh (SR) mode
		2234	* - lines are large relative to FIFO size (buffer can hold up to 2)
		2235	* - lines are small relative to FIFO size (buffer can hold more than 2
		2236	* lines), so need to account for TLB latency
		2237	*
		2238	* The normal calculation is:
		2239	* watermark = dotclock * bytes per pixel * latency
		2240	* where latency is platform & configuration dependent (we assume pessimal
		2241	* values here).
		2242	*
		2243	* The SR calculation is:
		2244	* watermark = (trunc(latency/line time)+1) * surface width *
		2245	* bytes per pixel
		2246	* where
		2247	* line time = htotal / dotclock
		2248	* surface width = hdisplay for normal plane and 64 for cursor
		2249	* and latency is assumed to be high, as above.
		2250	*
		2251	* The final value programmed to the register should always be rounded up,
		2252	* and include an extra 2 entries to account for clock crossings.
		2253	*
		2254	* We don't use the sprite, so we can ignore that. And on Crestline we have
		2255	* to set the non-SR watermarks to 8.
		2256	*/
		2257	void intel_update_watermarks(struct drm_device *dev)
		2258	{
		2259	struct drm_i915_private *dev_priv = dev->dev_private;
		2260
		2261	if (dev_priv->display.update_wm)
		2262	dev_priv->display.update_wm(dev);
		2263	}
		2264
		2265	void intel_update_linetime_watermarks(struct drm_device *dev,
		2266	int pipe, struct drm_display_mode *mode)
		2267	{
		2268	struct drm_i915_private *dev_priv = dev->dev_private;
		2269
		2270	if (dev_priv->display.update_linetime_wm)
		2271	dev_priv->display.update_linetime_wm(dev, pipe, mode);
		2272	}
		2273
		2274	void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
		2275	uint32_t sprite_width, int pixel_size)
		2276	{
		2277	struct drm_i915_private *dev_priv = dev->dev_private;
		2278
		2279	if (dev_priv->display.update_sprite_wm)
		2280	dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
		2281	pixel_size);
		2282	}
		2283
		2284	static struct drm_i915_gem_object *
		2285	intel_alloc_context_page(struct drm_device *dev)
		2286	{
		2287	struct drm_i915_gem_object *ctx;
		2288	int ret;
		2289
		2290	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2291
		2292	ctx = i915_gem_alloc_object(dev, 4096);
		2293	if (!ctx) {
		2294	DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
		2295	return NULL;
		2296	}
		2297
		2298	ret = i915_gem_object_pin(ctx, 4096, true, false);
		2299	if (ret) {
		2300	DRM_ERROR("failed to pin power context: %d\n", ret);
		2301	goto err_unref;
		2302	}
		2303
		2304	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
		2305	if (ret) {
		2306	DRM_ERROR("failed to set-domain on power context: %d\n", ret);
		2307	goto err_unpin;
		2308	}
		2309
		2310	return ctx;
		2311
		2312	err_unpin:
		2313	i915_gem_object_unpin(ctx);
		2314	err_unref:
		2315	drm_gem_object_unreference(&ctx->base);
		2316	return NULL;
		2317	}
		2318
		2319	/**
		2320	* Lock protecting IPS related data structures
		2321	*/
		2322	DEFINE_SPINLOCK(mchdev_lock);
		2323
		2324	/* Global for IPS driver to get at the current i915 device. Protected by
		2325	* mchdev_lock. */
		2326	static struct drm_i915_private *i915_mch_dev;
		2327
		2328	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		2329	{
		2330	struct drm_i915_private *dev_priv = dev->dev_private;
		2331	u16 rgvswctl;
		2332
		2333	assert_spin_locked(&mchdev_lock);
		2334
		2335	rgvswctl = I915_READ16(MEMSWCTL);
		2336	if (rgvswctl & MEMCTL_CMD_STS) {
		2337	DRM_DEBUG("gpu busy, RCS change rejected\n");
		2338	return false; /* still busy with another command */
		2339	}
		2340
		2341	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		2342	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		2343	I915_WRITE16(MEMSWCTL, rgvswctl);
		2344	POSTING_READ16(MEMSWCTL);
		2345
		2346	rgvswctl \|= MEMCTL_CMD_STS;
		2347	I915_WRITE16(MEMSWCTL, rgvswctl);
		2348
		2349	return true;
		2350	}
		2351
		2352	static void ironlake_enable_drps(struct drm_device *dev)
		2353	{
		2354	struct drm_i915_private *dev_priv = dev->dev_private;
		2355	u32 rgvmodectl = I915_READ(MEMMODECTL);
		2356	u8 fmax, fmin, fstart, vstart;
		2357
		2358	spin_lock_irq(&mchdev_lock);
		2359
		2360	/* Enable temp reporting */
		2361	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		2362	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		2363
		2364	/* 100ms RC evaluation intervals */
		2365	I915_WRITE(RCUPEI, 100000);
		2366	I915_WRITE(RCDNEI, 100000);
		2367
		2368	/* Set max/min thresholds to 90ms and 80ms respectively */
		2369	I915_WRITE(RCBMAXAVG, 90000);
		2370	I915_WRITE(RCBMINAVG, 80000);
		2371
		2372	I915_WRITE(MEMIHYST, 1);
		2373
		2374	/* Set up min, max, and cur for interrupt handling */
		2375	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		2376	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		2377	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		2378	MEMMODE_FSTART_SHIFT;
		2379
		2380	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
		2381	PXVFREQ_PX_SHIFT;
		2382
		2383	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		2384	dev_priv->ips.fstart = fstart;
		2385
		2386	dev_priv->ips.max_delay = fstart;
		2387	dev_priv->ips.min_delay = fmin;
		2388	dev_priv->ips.cur_delay = fstart;
		2389
		2390	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		2391	fmax, fmin, fstart);
		2392
		2393	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		2394
		2395	/*
		2396	* Interrupts will be enabled in ironlake_irq_postinstall
		2397	*/
		2398
		2399	I915_WRITE(VIDSTART, vstart);
		2400	POSTING_READ(VIDSTART);
		2401
		2402	rgvmodectl \|= MEMMODE_SWMODE_EN;
		2403	I915_WRITE(MEMMODECTL, rgvmodectl);
		2404
		2405	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		2406	DRM_ERROR("stuck trying to change perf mode\n");
		2407	mdelay(1);
		2408
		2409	ironlake_set_drps(dev, fstart);
		2410
		2411	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
		2412	I915_READ(0x112e0);
		2413	dev_priv->ips.last_time1 = jiffies_to_msecs(GetTimerTicks());
		2414	dev_priv->ips.last_count2 = I915_READ(0x112f4);
3482	Serge	2415	getrawmonotonic(&dev_priv->ips.last_time2);
3031	serge	2416
		2417	spin_unlock_irq(&mchdev_lock);
		2418	}
		2419
		2420	static void ironlake_disable_drps(struct drm_device *dev)
		2421	{
		2422	struct drm_i915_private *dev_priv = dev->dev_private;
		2423	u16 rgvswctl;
		2424
		2425	spin_lock_irq(&mchdev_lock);
		2426
		2427	rgvswctl = I915_READ16(MEMSWCTL);
		2428
		2429	/* Ack interrupts, disable EFC interrupt */
		2430	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		2431	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		2432	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		2433	I915_WRITE(DEIIR, DE_PCU_EVENT);
		2434	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		2435
		2436	/* Go back to the starting frequency */
		2437	ironlake_set_drps(dev, dev_priv->ips.fstart);
		2438	mdelay(1);
		2439	rgvswctl \|= MEMCTL_CMD_STS;
		2440	I915_WRITE(MEMSWCTL, rgvswctl);
		2441	mdelay(1);
		2442
		2443	spin_unlock_irq(&mchdev_lock);
		2444	}
		2445
		2446	/* There's a funny hw issue where the hw returns all 0 when reading from
		2447	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		2448	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		2449	* all limits and the gpu stuck at whatever frequency it is at atm).
		2450	*/
		2451	static u32 gen6_rps_limits(struct drm_i915_private dev_priv, u8 val)
		2452	{
		2453	u32 limits;
		2454
		2455	limits = 0;
		2456
		2457	if (*val >= dev_priv->rps.max_delay)
		2458	*val = dev_priv->rps.max_delay;
		2459	limits \|= dev_priv->rps.max_delay << 24;
		2460
		2461	/* Only set the down limit when we've reached the lowest level to avoid
		2462	* getting more interrupts, otherwise leave this clear. This prevents a
		2463	* race in the hw when coming out of rc6: There's a tiny window where
		2464	* the hw runs at the minimal clock before selecting the desired
		2465	* frequency, if the down threshold expires in that window we will not
		2466	* receive a down interrupt. */
		2467	if (*val <= dev_priv->rps.min_delay) {
		2468	*val = dev_priv->rps.min_delay;
		2469	limits \|= dev_priv->rps.min_delay << 16;
		2470	}
		2471
		2472	return limits;
		2473	}
		2474
		2475	void gen6_set_rps(struct drm_device *dev, u8 val)
		2476	{
		2477	struct drm_i915_private *dev_priv = dev->dev_private;
		2478	u32 limits = gen6_rps_limits(dev_priv, &val);
		2479
3243	Serge	2480	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2481	WARN_ON(val > dev_priv->rps.max_delay);
		2482	WARN_ON(val < dev_priv->rps.min_delay);
		2483
		2484	if (val == dev_priv->rps.cur_delay)
		2485	return;
		2486
		2487	I915_WRITE(GEN6_RPNSWREQ,
		2488	GEN6_FREQUENCY(val) \|
		2489	GEN6_OFFSET(0) \|
		2490	GEN6_AGGRESSIVE_TURBO);
		2491
		2492	/* Make sure we continue to get interrupts
		2493	* until we hit the minimum or maximum frequencies.
		2494	*/
		2495	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
		2496
		2497	POSTING_READ(GEN6_RPNSWREQ);
		2498
		2499	dev_priv->rps.cur_delay = val;
		2500
		2501	trace_intel_gpu_freq_change(val * 50);
		2502	}
		2503
		2504	static void gen6_disable_rps(struct drm_device *dev)
		2505	{
		2506	struct drm_i915_private *dev_priv = dev->dev_private;
		2507
		2508	I915_WRITE(GEN6_RC_CONTROL, 0);
		2509	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		2510	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
		2511	I915_WRITE(GEN6_PMIER, 0);
		2512	/* Complete PM interrupt masking here doesn't race with the rps work
		2513	* item again unmasking PM interrupts because that is using a different
		2514	* register (PMIMR) to mask PM interrupts. The only risk is in leaving
		2515	* stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
		2516
		2517	spin_lock_irq(&dev_priv->rps.lock);
		2518	dev_priv->rps.pm_iir = 0;
		2519	spin_unlock_irq(&dev_priv->rps.lock);
		2520
		2521	I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
		2522	}
		2523
		2524	int intel_enable_rc6(const struct drm_device *dev)
		2525	{
		2526	/* Respect the kernel parameter if it is set */
		2527	if (i915_enable_rc6 >= 0)
		2528	return i915_enable_rc6;
		2529
3120	serge	2530	/* Disable RC6 on Ironlake */
		2531	if (INTEL_INFO(dev)->gen == 5)
		2532	return 0;
3031	serge	2533
		2534	if (IS_HASWELL(dev)) {
		2535	DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
		2536	return INTEL_RC6_ENABLE;
		2537	}
		2538
		2539	/* snb/ivb have more than one rc6 state. */
		2540	if (INTEL_INFO(dev)->gen == 6) {
		2541	DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
		2542	return INTEL_RC6_ENABLE;
		2543	}
		2544
		2545	DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
		2546	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
		2547	}
		2548
		2549	static void gen6_enable_rps(struct drm_device *dev)
		2550	{
		2551	struct drm_i915_private *dev_priv = dev->dev_private;
		2552	struct intel_ring_buffer *ring;
		2553	u32 rp_state_cap;
		2554	u32 gt_perf_status;
3243	Serge	2555	u32 rc6vids, pcu_mbox, rc6_mask = 0;
3031	serge	2556	u32 gtfifodbg;
		2557	int rc6_mode;
3243	Serge	2558	int i, ret;
3031	serge	2559
3243	Serge	2560	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2561
		2562	/* Here begins a magic sequence of register writes to enable
		2563	* auto-downclocking.
		2564	*
		2565	* Perhaps there might be some value in exposing these to
		2566	* userspace...
		2567	*/
		2568	I915_WRITE(GEN6_RC_STATE, 0);
		2569
		2570	/* Clear the DBG now so we don't confuse earlier errors */
		2571	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		2572	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		2573	I915_WRITE(GTFIFODBG, gtfifodbg);
		2574	}
		2575
		2576	gen6_gt_force_wake_get(dev_priv);
		2577
		2578	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		2579	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
		2580
		2581	/* In units of 100MHz */
		2582	dev_priv->rps.max_delay = rp_state_cap & 0xff;
		2583	dev_priv->rps.min_delay = (rp_state_cap & 0xff0000) >> 16;
		2584	dev_priv->rps.cur_delay = 0;
		2585
		2586	/* disable the counters and set deterministic thresholds */
		2587	I915_WRITE(GEN6_RC_CONTROL, 0);
		2588
		2589	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		2590	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		2591	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		2592	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		2593	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		2594
		2595	for_each_ring(ring, dev_priv, i)
		2596	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		2597
		2598	I915_WRITE(GEN6_RC_SLEEP, 0);
		2599	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
		2600	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	2601	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	2602	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		2603
		2604	/* Check if we are enabling RC6 */
		2605	rc6_mode = intel_enable_rc6(dev_priv->dev);
		2606	if (rc6_mode & INTEL_RC6_ENABLE)
		2607	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		2608
		2609	/* We don't use those on Haswell */
		2610	if (!IS_HASWELL(dev)) {
		2611	if (rc6_mode & INTEL_RC6p_ENABLE)
		2612	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		2613
		2614	if (rc6_mode & INTEL_RC6pp_ENABLE)
		2615	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		2616	}
		2617
		2618	DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
		2619	(rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		2620	(rc6_mask & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		2621	(rc6_mask & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
		2622
		2623	I915_WRITE(GEN6_RC_CONTROL,
		2624	rc6_mask \|
		2625	GEN6_RC_CTL_EI_MODE(1) \|
		2626	GEN6_RC_CTL_HW_ENABLE);
		2627
		2628	I915_WRITE(GEN6_RPNSWREQ,
		2629	GEN6_FREQUENCY(10) \|
		2630	GEN6_OFFSET(0) \|
		2631	GEN6_AGGRESSIVE_TURBO);
		2632	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		2633	GEN6_FREQUENCY(12));
		2634
		2635	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
		2636	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
		2637	dev_priv->rps.max_delay << 24 \|
		2638	dev_priv->rps.min_delay << 16);
		2639
		2640	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		2641	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		2642	I915_WRITE(GEN6_RP_UP_EI, 66000);
		2643	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		2644
		2645	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		2646	I915_WRITE(GEN6_RP_CONTROL,
		2647	GEN6_RP_MEDIA_TURBO \|
		2648	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		2649	GEN6_RP_MEDIA_IS_GFX \|
		2650	GEN6_RP_ENABLE \|
		2651	GEN6_RP_UP_BUSY_AVG \|
		2652	(IS_HASWELL(dev) ? GEN7_RP_DOWN_IDLE_AVG : GEN6_RP_DOWN_IDLE_CONT));
		2653
3243	Serge	2654	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
		2655	if (!ret) {
		2656	pcu_mbox = 0;
		2657	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
		2658	if (ret && pcu_mbox & (1<<31)) { /* OC supported */
3031	serge	2659	dev_priv->rps.max_delay = pcu_mbox & 0xff;
		2660	DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
		2661	}
3243	Serge	2662	} else {
		2663	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		2664	}
3031	serge	2665
		2666	gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
		2667
		2668	/* requires MSI enabled */
		2669	I915_WRITE(GEN6_PMIER, GEN6_PM_DEFERRED_EVENTS);
		2670	spin_lock_irq(&dev_priv->rps.lock);
		2671	WARN_ON(dev_priv->rps.pm_iir != 0);
		2672	I915_WRITE(GEN6_PMIMR, 0);
		2673	spin_unlock_irq(&dev_priv->rps.lock);
		2674	/* enable all PM interrupts */
		2675	I915_WRITE(GEN6_PMINTRMSK, 0);
		2676
3243	Serge	2677	rc6vids = 0;
		2678	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		2679	if (IS_GEN6(dev) && ret) {
		2680	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		2681	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		2682	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		2683	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		2684	rc6vids &= 0xffff00;
		2685	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		2686	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		2687	if (ret)
		2688	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		2689	}
		2690
3031	serge	2691	gen6_gt_force_wake_put(dev_priv);
		2692	}
		2693
		2694	static void gen6_update_ring_freq(struct drm_device *dev)
		2695	{
		2696	struct drm_i915_private *dev_priv = dev->dev_private;
		2697	int min_freq = 15;
3243	Serge	2698	int gpu_freq;
		2699	unsigned int ia_freq, max_ia_freq;
3031	serge	2700	int scaling_factor = 180;
		2701
3243	Serge	2702	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2703
		2704	max_ia_freq = cpufreq_quick_get_max(0);
		2705	/*
		2706	* Default to measured freq if none found, PCU will ensure we don't go
		2707	* over
		2708	*/
		2709	if (!max_ia_freq)
		2710	max_ia_freq = tsc_khz;
		2711
		2712	/* Convert from kHz to MHz */
		2713	max_ia_freq /= 1000;
		2714
		2715	/*
		2716	* For each potential GPU frequency, load a ring frequency we'd like
		2717	* to use for memory access. We do this by specifying the IA frequency
		2718	* the PCU should use as a reference to determine the ring frequency.
		2719	*/
		2720	for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
		2721	gpu_freq--) {
		2722	int diff = dev_priv->rps.max_delay - gpu_freq;
		2723
		2724	/*
		2725	* For GPU frequencies less than 750MHz, just use the lowest
		2726	* ring freq.
		2727	*/
		2728	if (gpu_freq < min_freq)
		2729	ia_freq = 800;
		2730	else
		2731	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		2732	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3243	Serge	2733	ia_freq <<= GEN6_PCODE_FREQ_IA_RATIO_SHIFT;
3031	serge	2734
3243	Serge	2735	sandybridge_pcode_write(dev_priv,
		2736	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
		2737	ia_freq \| gpu_freq);
3031	serge	2738	}
		2739	}
		2740
		2741	void ironlake_teardown_rc6(struct drm_device *dev)
		2742	{
		2743	struct drm_i915_private *dev_priv = dev->dev_private;
		2744
3243	Serge	2745	if (dev_priv->ips.renderctx) {
		2746	i915_gem_object_unpin(dev_priv->ips.renderctx);
		2747	drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
		2748	dev_priv->ips.renderctx = NULL;
3031	serge	2749	}
		2750
3243	Serge	2751	if (dev_priv->ips.pwrctx) {
		2752	i915_gem_object_unpin(dev_priv->ips.pwrctx);
		2753	drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
		2754	dev_priv->ips.pwrctx = NULL;
3031	serge	2755	}
		2756	}
		2757
		2758	static void ironlake_disable_rc6(struct drm_device *dev)
		2759	{
		2760	struct drm_i915_private *dev_priv = dev->dev_private;
		2761
		2762	if (I915_READ(PWRCTXA)) {
		2763	/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
		2764	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) \| RCX_SW_EXIT);
		2765	wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
		2766	50);
		2767
		2768	I915_WRITE(PWRCTXA, 0);
		2769	POSTING_READ(PWRCTXA);
		2770
		2771	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		2772	POSTING_READ(RSTDBYCTL);
		2773	}
		2774	}
		2775
		2776	static int ironlake_setup_rc6(struct drm_device *dev)
		2777	{
		2778	struct drm_i915_private *dev_priv = dev->dev_private;
		2779
3243	Serge	2780	if (dev_priv->ips.renderctx == NULL)
		2781	dev_priv->ips.renderctx = intel_alloc_context_page(dev);
		2782	if (!dev_priv->ips.renderctx)
3031	serge	2783	return -ENOMEM;
		2784
3243	Serge	2785	if (dev_priv->ips.pwrctx == NULL)
		2786	dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
		2787	if (!dev_priv->ips.pwrctx) {
3031	serge	2788	ironlake_teardown_rc6(dev);
		2789	return -ENOMEM;
		2790	}
		2791
		2792	return 0;
		2793	}
		2794
		2795	static void ironlake_enable_rc6(struct drm_device *dev)
		2796	{
		2797	struct drm_i915_private *dev_priv = dev->dev_private;
		2798	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
3243	Serge	2799	bool was_interruptible;
3031	serge	2800	int ret;
		2801
		2802	/* rc6 disabled by default due to repeated reports of hanging during
		2803	* boot and resume.
		2804	*/
		2805	if (!intel_enable_rc6(dev))
		2806	return;
		2807
		2808	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2809
		2810	ret = ironlake_setup_rc6(dev);
		2811	if (ret)
		2812	return;
		2813
3243	Serge	2814	was_interruptible = dev_priv->mm.interruptible;
		2815	dev_priv->mm.interruptible = false;
		2816
3031	serge	2817	/*
		2818	* GPU can automatically power down the render unit if given a page
		2819	* to save state.
		2820	*/
		2821	ret = intel_ring_begin(ring, 6);
		2822	if (ret) {
		2823	ironlake_teardown_rc6(dev);
3243	Serge	2824	dev_priv->mm.interruptible = was_interruptible;
3031	serge	2825	return;
		2826	}
		2827
		2828	intel_ring_emit(ring, MI_SUSPEND_FLUSH \| MI_SUSPEND_FLUSH_EN);
		2829	intel_ring_emit(ring, MI_SET_CONTEXT);
3243	Serge	2830	intel_ring_emit(ring, dev_priv->ips.renderctx->gtt_offset \|
3031	serge	2831	MI_MM_SPACE_GTT \|
		2832	MI_SAVE_EXT_STATE_EN \|
		2833	MI_RESTORE_EXT_STATE_EN \|
		2834	MI_RESTORE_INHIBIT);
		2835	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
		2836	intel_ring_emit(ring, MI_NOOP);
		2837	intel_ring_emit(ring, MI_FLUSH);
		2838	intel_ring_advance(ring);
		2839
		2840	/*
		2841	* Wait for the command parser to advance past MI_SET_CONTEXT. The HW
		2842	* does an implicit flush, combined with MI_FLUSH above, it should be
		2843	* safe to assume that renderctx is valid
		2844	*/
3243	Serge	2845	ret = intel_ring_idle(ring);
		2846	dev_priv->mm.interruptible = was_interruptible;
3031	serge	2847	if (ret) {
		2848	DRM_ERROR("failed to enable ironlake power power savings\n");
		2849	ironlake_teardown_rc6(dev);
		2850	return;
		2851	}
		2852
3243	Serge	2853	I915_WRITE(PWRCTXA, dev_priv->ips.pwrctx->gtt_offset \| PWRCTX_EN);
3031	serge	2854	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		2855	}
		2856
		2857	static unsigned long intel_pxfreq(u32 vidfreq)
		2858	{
		2859	unsigned long freq;
		2860	int div = (vidfreq & 0x3f0000) >> 16;
		2861	int post = (vidfreq & 0x3000) >> 12;
		2862	int pre = (vidfreq & 0x7);
		2863
		2864	if (!pre)
		2865	return 0;
		2866
		2867	freq = ((div * 133333) / ((1<
		2868
		2869	return freq;
		2870	}
		2871
		2872	static const struct cparams {
		2873	u16 i;
		2874	u16 t;
		2875	u16 m;
		2876	u16 c;
		2877	} cparams[] = {
		2878	{ 1, 1333, 301, 28664 },
		2879	{ 1, 1066, 294, 24460 },
		2880	{ 1, 800, 294, 25192 },
		2881	{ 0, 1333, 276, 27605 },
		2882	{ 0, 1066, 276, 27605 },
		2883	{ 0, 800, 231, 23784 },
		2884	};
		2885
		2886	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		2887	{
		2888	u64 total_count, diff, ret;
		2889	u32 count1, count2, count3, m = 0, c = 0;
		2890	unsigned long now = jiffies_to_msecs(GetTimerTicks()), diff1;
		2891	int i;
		2892
		2893	assert_spin_locked(&mchdev_lock);
		2894
		2895	diff1 = now - dev_priv->ips.last_time1;
		2896
		2897	/* Prevent division-by-zero if we are asking too fast.
		2898	* Also, we don't get interesting results if we are polling
		2899	* faster than once in 10ms, so just return the saved value
		2900	* in such cases.
		2901	*/
		2902	if (diff1 <= 10)
		2903	return dev_priv->ips.chipset_power;
		2904
		2905	count1 = I915_READ(DMIEC);
		2906	count2 = I915_READ(DDREC);
		2907	count3 = I915_READ(CSIEC);
		2908
		2909	total_count = count1 + count2 + count3;
		2910
		2911	/* FIXME: handle per-counter overflow */
		2912	if (total_count < dev_priv->ips.last_count1) {
		2913	diff = ~0UL - dev_priv->ips.last_count1;
		2914	diff += total_count;
		2915	} else {
		2916	diff = total_count - dev_priv->ips.last_count1;
		2917	}
		2918
		2919	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		2920	if (cparams[i].i == dev_priv->ips.c_m &&
		2921	cparams[i].t == dev_priv->ips.r_t) {
		2922	m = cparams[i].m;
		2923	c = cparams[i].c;
		2924	break;
		2925	}
		2926	}
		2927
		2928	diff = div_u64(diff, diff1);
		2929	ret = ((m * diff) + c);
		2930	ret = div_u64(ret, 10);
		2931
		2932	dev_priv->ips.last_count1 = total_count;
		2933	dev_priv->ips.last_time1 = now;
		2934
		2935	dev_priv->ips.chipset_power = ret;
		2936
		2937	return ret;
		2938	}
		2939
		2940	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		2941	{
		2942	unsigned long val;
		2943
		2944	if (dev_priv->info->gen != 5)
		2945	return 0;
		2946
		2947	spin_lock_irq(&mchdev_lock);
		2948
		2949	val = __i915_chipset_val(dev_priv);
		2950
		2951	spin_unlock_irq(&mchdev_lock);
		2952
		2953	return val;
		2954	}
		2955
		2956	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		2957	{
		2958	unsigned long m, x, b;
		2959	u32 tsfs;
		2960
		2961	tsfs = I915_READ(TSFS);
		2962
		2963	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		2964	x = I915_READ8(TR1);
		2965
		2966	b = tsfs & TSFS_INTR_MASK;
		2967
		2968	return ((m * x) / 127) - b;
		2969	}
		2970
		2971	static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		2972	{
		2973	static const struct v_table {
		2974	u16 vd; /* in .1 mil */
		2975	u16 vm; /* in .1 mil */
		2976	} v_table[] = {
		2977	{ 0, 0, },
		2978	{ 375, 0, },
		2979	{ 500, 0, },
		2980	{ 625, 0, },
		2981	{ 750, 0, },
		2982	{ 875, 0, },
		2983	{ 1000, 0, },
		2984	{ 1125, 0, },
		2985	{ 4125, 3000, },
		2986	{ 4125, 3000, },
		2987	{ 4125, 3000, },
		2988	{ 4125, 3000, },
		2989	{ 4125, 3000, },
		2990	{ 4125, 3000, },
		2991	{ 4125, 3000, },
		2992	{ 4125, 3000, },
		2993	{ 4125, 3000, },
		2994	{ 4125, 3000, },
		2995	{ 4125, 3000, },
		2996	{ 4125, 3000, },
		2997	{ 4125, 3000, },
		2998	{ 4125, 3000, },
		2999	{ 4125, 3000, },
		3000	{ 4125, 3000, },
		3001	{ 4125, 3000, },
		3002	{ 4125, 3000, },
		3003	{ 4125, 3000, },
		3004	{ 4125, 3000, },
		3005	{ 4125, 3000, },
		3006	{ 4125, 3000, },
		3007	{ 4125, 3000, },
		3008	{ 4125, 3000, },
		3009	{ 4250, 3125, },
		3010	{ 4375, 3250, },
		3011	{ 4500, 3375, },
		3012	{ 4625, 3500, },
		3013	{ 4750, 3625, },
		3014	{ 4875, 3750, },
		3015	{ 5000, 3875, },
		3016	{ 5125, 4000, },
		3017	{ 5250, 4125, },
		3018	{ 5375, 4250, },
		3019	{ 5500, 4375, },
		3020	{ 5625, 4500, },
		3021	{ 5750, 4625, },
		3022	{ 5875, 4750, },
		3023	{ 6000, 4875, },
		3024	{ 6125, 5000, },
		3025	{ 6250, 5125, },
		3026	{ 6375, 5250, },
		3027	{ 6500, 5375, },
		3028	{ 6625, 5500, },
		3029	{ 6750, 5625, },
		3030	{ 6875, 5750, },
		3031	{ 7000, 5875, },
		3032	{ 7125, 6000, },
		3033	{ 7250, 6125, },
		3034	{ 7375, 6250, },
		3035	{ 7500, 6375, },
		3036	{ 7625, 6500, },
		3037	{ 7750, 6625, },
		3038	{ 7875, 6750, },
		3039	{ 8000, 6875, },
		3040	{ 8125, 7000, },
		3041	{ 8250, 7125, },
		3042	{ 8375, 7250, },
		3043	{ 8500, 7375, },
		3044	{ 8625, 7500, },
		3045	{ 8750, 7625, },
		3046	{ 8875, 7750, },
		3047	{ 9000, 7875, },
		3048	{ 9125, 8000, },
		3049	{ 9250, 8125, },
		3050	{ 9375, 8250, },
		3051	{ 9500, 8375, },
		3052	{ 9625, 8500, },
		3053	{ 9750, 8625, },
		3054	{ 9875, 8750, },
		3055	{ 10000, 8875, },
		3056	{ 10125, 9000, },
		3057	{ 10250, 9125, },
		3058	{ 10375, 9250, },
		3059	{ 10500, 9375, },
		3060	{ 10625, 9500, },
		3061	{ 10750, 9625, },
		3062	{ 10875, 9750, },
		3063	{ 11000, 9875, },
		3064	{ 11125, 10000, },
		3065	{ 11250, 10125, },
		3066	{ 11375, 10250, },
		3067	{ 11500, 10375, },
		3068	{ 11625, 10500, },
		3069	{ 11750, 10625, },
		3070	{ 11875, 10750, },
		3071	{ 12000, 10875, },
		3072	{ 12125, 11000, },
		3073	{ 12250, 11125, },
		3074	{ 12375, 11250, },
		3075	{ 12500, 11375, },
		3076	{ 12625, 11500, },
		3077	{ 12750, 11625, },
		3078	{ 12875, 11750, },
		3079	{ 13000, 11875, },
		3080	{ 13125, 12000, },
		3081	{ 13250, 12125, },
		3082	{ 13375, 12250, },
		3083	{ 13500, 12375, },
		3084	{ 13625, 12500, },
		3085	{ 13750, 12625, },
		3086	{ 13875, 12750, },
		3087	{ 14000, 12875, },
		3088	{ 14125, 13000, },
		3089	{ 14250, 13125, },
		3090	{ 14375, 13250, },
		3091	{ 14500, 13375, },
		3092	{ 14625, 13500, },
		3093	{ 14750, 13625, },
		3094	{ 14875, 13750, },
		3095	{ 15000, 13875, },
		3096	{ 15125, 14000, },
		3097	{ 15250, 14125, },
		3098	{ 15375, 14250, },
		3099	{ 15500, 14375, },
		3100	{ 15625, 14500, },
		3101	{ 15750, 14625, },
		3102	{ 15875, 14750, },
		3103	{ 16000, 14875, },
		3104	{ 16125, 15000, },
		3105	};
		3106	if (dev_priv->info->is_mobile)
		3107	return v_table[pxvid].vm;
		3108	else
		3109	return v_table[pxvid].vd;
		3110	}
		3111
		3112	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		3113	{
		3114	struct timespec now, diff1;
		3115	u64 diff;
		3116	unsigned long diffms;
		3117	u32 count;
		3118
		3119	assert_spin_locked(&mchdev_lock);
		3120
		3121	getrawmonotonic(&now);
		3122	diff1 = timespec_sub(now, dev_priv->ips.last_time2);
		3123
		3124	/* Don't divide by 0 */
		3125	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
		3126	if (!diffms)
		3127	return;
		3128
		3129	count = I915_READ(GFXEC);
		3130
		3131	if (count < dev_priv->ips.last_count2) {
		3132	diff = ~0UL - dev_priv->ips.last_count2;
		3133	diff += count;
		3134	} else {
		3135	diff = count - dev_priv->ips.last_count2;
		3136	}
		3137
		3138	dev_priv->ips.last_count2 = count;
		3139	dev_priv->ips.last_time2 = now;
		3140
		3141	/* More magic constants... */
		3142	diff = diff * 1181;
		3143	diff = div_u64(diff, diffms * 10);
		3144	dev_priv->ips.gfx_power = diff;
		3145	}
		3146
		3147	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		3148	{
		3149	if (dev_priv->info->gen != 5)
		3150	return;
		3151
		3152	spin_lock_irq(&mchdev_lock);
		3153
		3154	__i915_update_gfx_val(dev_priv);
		3155
		3156	spin_unlock_irq(&mchdev_lock);
		3157	}
		3158
		3159	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		3160	{
		3161	unsigned long t, corr, state1, corr2, state2;
		3162	u32 pxvid, ext_v;
		3163
		3164	assert_spin_locked(&mchdev_lock);
		3165
		3166	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
		3167	pxvid = (pxvid >> 24) & 0x7f;
		3168	ext_v = pvid_to_extvid(dev_priv, pxvid);
		3169
		3170	state1 = ext_v;
		3171
		3172	t = i915_mch_val(dev_priv);
		3173
		3174	/* Revel in the empirically derived constants */
		3175
		3176	/* Correction factor in 1/100000 units */
		3177	if (t > 80)
		3178	corr = ((t * 2349) + 135940);
		3179	else if (t >= 50)
		3180	corr = ((t * 964) + 29317);
		3181	else /* < 50 */
		3182	corr = ((t * 301) + 1004);
		3183
		3184	corr = corr * ((150142 * state1) / 10000 - 78642);
		3185	corr /= 100000;
		3186	corr2 = (corr * dev_priv->ips.corr);
		3187
		3188	state2 = (corr2 * state1) / 10000;
		3189	state2 /= 100; /* convert to mW */
		3190
		3191	__i915_update_gfx_val(dev_priv);
		3192
		3193	return dev_priv->ips.gfx_power + state2;
		3194	}
		3195
		3196	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		3197	{
		3198	unsigned long val;
		3199
		3200	if (dev_priv->info->gen != 5)
		3201	return 0;
		3202
		3203	spin_lock_irq(&mchdev_lock);
		3204
		3205	val = __i915_gfx_val(dev_priv);
		3206
		3207	spin_unlock_irq(&mchdev_lock);
		3208
		3209	return val;
		3210	}
		3211
		3212	/**
		3213	* i915_read_mch_val - return value for IPS use
		3214	*
		3215	* Calculate and return a value for the IPS driver to use when deciding whether
		3216	* we have thermal and power headroom to increase CPU or GPU power budget.
		3217	*/
		3218	unsigned long i915_read_mch_val(void)
		3219	{
		3220	struct drm_i915_private *dev_priv;
		3221	unsigned long chipset_val, graphics_val, ret = 0;
		3222
		3223	spin_lock_irq(&mchdev_lock);
		3224	if (!i915_mch_dev)
		3225	goto out_unlock;
		3226	dev_priv = i915_mch_dev;
		3227
		3228	chipset_val = __i915_chipset_val(dev_priv);
		3229	graphics_val = __i915_gfx_val(dev_priv);
		3230
		3231	ret = chipset_val + graphics_val;
		3232
		3233	out_unlock:
		3234	spin_unlock_irq(&mchdev_lock);
		3235
		3236	return ret;
		3237	}
		3238	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		3239
		3240	/**
		3241	* i915_gpu_raise - raise GPU frequency limit
		3242	*
		3243	* Raise the limit; IPS indicates we have thermal headroom.
		3244	*/
		3245	bool i915_gpu_raise(void)
		3246	{
		3247	struct drm_i915_private *dev_priv;
		3248	bool ret = true;
		3249
		3250	spin_lock_irq(&mchdev_lock);
		3251	if (!i915_mch_dev) {
		3252	ret = false;
		3253	goto out_unlock;
		3254	}
		3255	dev_priv = i915_mch_dev;
		3256
		3257	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		3258	dev_priv->ips.max_delay--;
		3259
		3260	out_unlock:
		3261	spin_unlock_irq(&mchdev_lock);
		3262
		3263	return ret;
		3264	}
		3265	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		3266
		3267	/**
		3268	* i915_gpu_lower - lower GPU frequency limit
		3269	*
		3270	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		3271	* frequency maximum.
		3272	*/
		3273	bool i915_gpu_lower(void)
		3274	{
		3275	struct drm_i915_private *dev_priv;
		3276	bool ret = true;
		3277
		3278	spin_lock_irq(&mchdev_lock);
		3279	if (!i915_mch_dev) {
		3280	ret = false;
		3281	goto out_unlock;
		3282	}
		3283	dev_priv = i915_mch_dev;
		3284
		3285	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		3286	dev_priv->ips.max_delay++;
		3287
		3288	out_unlock:
		3289	spin_unlock_irq(&mchdev_lock);
		3290
		3291	return ret;
		3292	}
		3293	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		3294
		3295	/**
		3296	* i915_gpu_busy - indicate GPU business to IPS
		3297	*
		3298	* Tell the IPS driver whether or not the GPU is busy.
		3299	*/
		3300	bool i915_gpu_busy(void)
		3301	{
		3302	struct drm_i915_private *dev_priv;
		3303	struct intel_ring_buffer *ring;
		3304	bool ret = false;
		3305	int i;
		3306
		3307	spin_lock_irq(&mchdev_lock);
		3308	if (!i915_mch_dev)
		3309	goto out_unlock;
		3310	dev_priv = i915_mch_dev;
		3311
		3312	for_each_ring(ring, dev_priv, i)
		3313	ret \|= !list_empty(&ring->request_list);
		3314
		3315	out_unlock:
		3316	spin_unlock_irq(&mchdev_lock);
		3317
		3318	return ret;
		3319	}
		3320	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		3321
		3322	/**
		3323	* i915_gpu_turbo_disable - disable graphics turbo
		3324	*
		3325	* Disable graphics turbo by resetting the max frequency and setting the
		3326	* current frequency to the default.
		3327	*/
		3328	bool i915_gpu_turbo_disable(void)
		3329	{
		3330	struct drm_i915_private *dev_priv;
		3331	bool ret = true;
		3332
		3333	spin_lock_irq(&mchdev_lock);
		3334	if (!i915_mch_dev) {
		3335	ret = false;
		3336	goto out_unlock;
		3337	}
		3338	dev_priv = i915_mch_dev;
		3339
		3340	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		3341
		3342	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		3343	ret = false;
		3344
		3345	out_unlock:
		3346	spin_unlock_irq(&mchdev_lock);
		3347
		3348	return ret;
		3349	}
		3350	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		3351
		3352	/**
		3353	* Tells the intel_ips driver that the i915 driver is now loaded, if
		3354	* IPS got loaded first.
		3355	*
		3356	* This awkward dance is so that neither module has to depend on the
		3357	* other in order for IPS to do the appropriate communication of
		3358	* GPU turbo limits to i915.
		3359	*/
		3360	static void
		3361	ips_ping_for_i915_load(void)
		3362	{
		3363	void (*link)(void);
		3364
		3365	// link = symbol_get(ips_link_to_i915_driver);
		3366	// if (link) {
		3367	// link();
		3368	// symbol_put(ips_link_to_i915_driver);
		3369	// }
		3370	}
		3371
		3372	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		3373	{
		3374	/* We only register the i915 ips part with intel-ips once everything is
		3375	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		3376	spin_lock_irq(&mchdev_lock);
		3377	i915_mch_dev = dev_priv;
		3378	spin_unlock_irq(&mchdev_lock);
		3379
		3380	ips_ping_for_i915_load();
		3381	}
		3382
		3383	void intel_gpu_ips_teardown(void)
		3384	{
		3385	spin_lock_irq(&mchdev_lock);
		3386	i915_mch_dev = NULL;
		3387	spin_unlock_irq(&mchdev_lock);
		3388	}
		3389	static void intel_init_emon(struct drm_device *dev)
		3390	{
		3391	struct drm_i915_private *dev_priv = dev->dev_private;
		3392	u32 lcfuse;
		3393	u8 pxw[16];
		3394	int i;
		3395
		3396	/* Disable to program */
		3397	I915_WRITE(ECR, 0);
		3398	POSTING_READ(ECR);
		3399
		3400	/* Program energy weights for various events */
		3401	I915_WRITE(SDEW, 0x15040d00);
		3402	I915_WRITE(CSIEW0, 0x007f0000);
		3403	I915_WRITE(CSIEW1, 0x1e220004);
		3404	I915_WRITE(CSIEW2, 0x04000004);
		3405
		3406	for (i = 0; i < 5; i++)
		3407	I915_WRITE(PEW + (i * 4), 0);
		3408	for (i = 0; i < 3; i++)
		3409	I915_WRITE(DEW + (i * 4), 0);
		3410
		3411	/* Program P-state weights to account for frequency power adjustment */
		3412	for (i = 0; i < 16; i++) {
		3413	u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
		3414	unsigned long freq = intel_pxfreq(pxvidfreq);
		3415	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		3416	PXVFREQ_PX_SHIFT;
		3417	unsigned long val;
		3418
		3419	val = vid * vid;
		3420	val *= (freq / 1000);
		3421	val *= 255;
		3422	val /= (127127900);
		3423	if (val > 0xff)
		3424	DRM_ERROR("bad pxval: %ld\n", val);
		3425	pxw[i] = val;
		3426	}
		3427	/* Render standby states get 0 weight */
		3428	pxw[14] = 0;
		3429	pxw[15] = 0;
		3430
		3431	for (i = 0; i < 4; i++) {
		3432	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		3433	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
		3434	I915_WRITE(PXW + (i * 4), val);
		3435	}
		3436
		3437	/* Adjust magic regs to magic values (more experimental results) */
		3438	I915_WRITE(OGW0, 0);
		3439	I915_WRITE(OGW1, 0);
		3440	I915_WRITE(EG0, 0x00007f00);
		3441	I915_WRITE(EG1, 0x0000000e);
		3442	I915_WRITE(EG2, 0x000e0000);
		3443	I915_WRITE(EG3, 0x68000300);
		3444	I915_WRITE(EG4, 0x42000000);
		3445	I915_WRITE(EG5, 0x00140031);
		3446	I915_WRITE(EG6, 0);
		3447	I915_WRITE(EG7, 0);
		3448
		3449	for (i = 0; i < 8; i++)
		3450	I915_WRITE(PXWL + (i * 4), 0);
		3451
		3452	/* Enable PMON + select events */
		3453	I915_WRITE(ECR, 0x80000019);
		3454
		3455	lcfuse = I915_READ(LCFUSE02);
		3456
		3457	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		3458	}
		3459
		3460	void intel_disable_gt_powersave(struct drm_device *dev)
		3461	{
3243	Serge	3462	struct drm_i915_private *dev_priv = dev->dev_private;
		3463
3031	serge	3464	if (IS_IRONLAKE_M(dev)) {
		3465	ironlake_disable_drps(dev);
		3466	ironlake_disable_rc6(dev);
		3467	} else if (INTEL_INFO(dev)->gen >= 6 && !IS_VALLEYVIEW(dev)) {
3482	Serge	3468	// cancel_delayed_work_sync(&dev_priv->rps.delayed_resume_work);
		3469	mutex_lock(&dev_priv->rps.hw_lock);
3031	serge	3470	gen6_disable_rps(dev);
3480	Serge	3471	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	3472	}
		3473	}
		3474
3482	Serge	3475	static void intel_gen6_powersave_work(struct work_struct *work)
		3476	{
		3477	struct drm_i915_private *dev_priv =
		3478	container_of(work, struct drm_i915_private,
		3479	rps.delayed_resume_work.work);
		3480	struct drm_device *dev = dev_priv->dev;
		3481
		3482	ENTER();
		3483
		3484	mutex_lock(&dev_priv->rps.hw_lock);
		3485	gen6_enable_rps(dev);
		3486	gen6_update_ring_freq(dev);
		3487	mutex_unlock(&dev_priv->rps.hw_lock);
		3488
		3489	LEAVE();
		3490	}
		3491
3031	serge	3492	void intel_enable_gt_powersave(struct drm_device *dev)
		3493	{
3243	Serge	3494	struct drm_i915_private *dev_priv = dev->dev_private;
		3495
3031	serge	3496	if (IS_IRONLAKE_M(dev)) {
		3497	ironlake_enable_drps(dev);
		3498	ironlake_enable_rc6(dev);
		3499	intel_init_emon(dev);
		3500	} else if ((IS_GEN6(dev) \|\| IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) {
3243	Serge	3501	/*
		3502	* PCU communication is slow and this doesn't need to be
		3503	* done at any specific time, so do this out of our fast path
		3504	* to make resume and init faster.
		3505	*/
3482	Serge	3506	schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		3507	round_jiffies_up_relative(HZ));
3031	serge	3508	}
		3509	}
		3510
3243	Serge	3511	static void ibx_init_clock_gating(struct drm_device *dev)
		3512	{
		3513	struct drm_i915_private *dev_priv = dev->dev_private;
		3514
		3515	/*
		3516	* On Ibex Peak and Cougar Point, we need to disable clock
		3517	* gating for the panel power sequencer or it will fail to
		3518	* start up when no ports are active.
		3519	*/
		3520	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		3521	}
		3522
3031	serge	3523	static void ironlake_init_clock_gating(struct drm_device *dev)
		3524	{
		3525	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	3526	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	3527
		3528	/* Required for FBC */
3243	Serge	3529	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		3530	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		3531	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	3532
		3533	I915_WRITE(PCH_3DCGDIS0,
		3534	MARIUNIT_CLOCK_GATE_DISABLE \|
		3535	SVSMUNIT_CLOCK_GATE_DISABLE);
		3536	I915_WRITE(PCH_3DCGDIS1,
		3537	VFMUNIT_CLOCK_GATE_DISABLE);
		3538
		3539	/*
		3540	* According to the spec the following bits should be set in
		3541	* order to enable memory self-refresh
		3542	* The bit 22/21 of 0x42004
		3543	* The bit 5 of 0x42020
		3544	* The bit 15 of 0x45000
		3545	*/
		3546	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3547	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3548	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	3549	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	3550	I915_WRITE(DISP_ARB_CTL,
		3551	(I915_READ(DISP_ARB_CTL) \|
		3552	DISP_FBC_WM_DIS));
		3553	I915_WRITE(WM3_LP_ILK, 0);
		3554	I915_WRITE(WM2_LP_ILK, 0);
		3555	I915_WRITE(WM1_LP_ILK, 0);
		3556
		3557	/*
		3558	* Based on the document from hardware guys the following bits
		3559	* should be set unconditionally in order to enable FBC.
		3560	* The bit 22 of 0x42000
		3561	* The bit 22 of 0x42004
		3562	* The bit 7,8,9 of 0x42020.
		3563	*/
		3564	if (IS_IRONLAKE_M(dev)) {
		3565	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		3566	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		3567	ILK_FBCQ_DIS);
		3568	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3569	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3570	ILK_DPARB_GATE);
		3571	}
		3572
3243	Serge	3573	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		3574
3031	serge	3575	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3576	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3577	ILK_ELPIN_409_SELECT);
		3578	I915_WRITE(_3D_CHICKEN2,
		3579	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		3580	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	3581
		3582	/* WaDisableRenderCachePipelinedFlush */
		3583	I915_WRITE(CACHE_MODE_0,
		3584	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		3585
		3586	ibx_init_clock_gating(dev);
3031	serge	3587	}
		3588
3243	Serge	3589	static void cpt_init_clock_gating(struct drm_device *dev)
		3590	{
		3591	struct drm_i915_private *dev_priv = dev->dev_private;
		3592	int pipe;
		3593
		3594	/*
		3595	* On Ibex Peak and Cougar Point, we need to disable clock
		3596	* gating for the panel power sequencer or it will fail to
		3597	* start up when no ports are active.
		3598	*/
		3599	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		3600	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		3601	DPLS_EDP_PPS_FIX_DIS);
		3602	/* The below fixes the weird display corruption, a few pixels shifted
		3603	* downward, on (only) LVDS of some HP laptops with IVY.
		3604	*/
		3605	for_each_pipe(pipe)
		3606	I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_CHICKEN2_TIMING_OVERRIDE);
		3607	/* WADP0ClockGatingDisable */
		3608	for_each_pipe(pipe) {
		3609	I915_WRITE(TRANS_CHICKEN1(pipe),
		3610	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		3611	}
		3612	}
		3613
3480	Serge	3614	static void gen6_check_mch_setup(struct drm_device *dev)
		3615	{
		3616	struct drm_i915_private *dev_priv = dev->dev_private;
		3617	uint32_t tmp;
		3618
		3619	tmp = I915_READ(MCH_SSKPD);
		3620	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) {
		3621	DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp);
		3622	DRM_INFO("This can cause pipe underruns and display issues.\n");
		3623	DRM_INFO("Please upgrade your BIOS to fix this.\n");
		3624	}
		3625	}
		3626
3031	serge	3627	static void gen6_init_clock_gating(struct drm_device *dev)
		3628	{
		3629	struct drm_i915_private *dev_priv = dev->dev_private;
		3630	int pipe;
3243	Serge	3631	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	3632
3243	Serge	3633	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	3634
		3635	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3636	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3637	ILK_ELPIN_409_SELECT);
		3638
3243	Serge	3639	/* WaDisableHiZPlanesWhenMSAAEnabled */
		3640	I915_WRITE(_3D_CHICKEN,
		3641	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		3642
		3643	/* WaSetupGtModeTdRowDispatch */
		3644	if (IS_SNB_GT1(dev))
		3645	I915_WRITE(GEN6_GT_MODE,
		3646	_MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
		3647
3031	serge	3648	I915_WRITE(WM3_LP_ILK, 0);
		3649	I915_WRITE(WM2_LP_ILK, 0);
		3650	I915_WRITE(WM1_LP_ILK, 0);
		3651
		3652	I915_WRITE(CACHE_MODE_0,
		3653	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		3654
		3655	I915_WRITE(GEN6_UCGCTL1,
		3656	I915_READ(GEN6_UCGCTL1) \|
		3657	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		3658	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		3659
		3660	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3661	* gating disable must be set. Failure to set it results in
		3662	* flickering pixels due to Z write ordering failures after
		3663	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3664	* Sanctuary and Tropics, and apparently anything else with
		3665	* alpha test or pixel discard.
		3666	*
		3667	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3668	* but we didn't debug actual testcases to find it out.
		3669	*
		3670	* Also apply WaDisableVDSUnitClockGating and
		3671	* WaDisableRCPBUnitClockGating.
		3672	*/
		3673	I915_WRITE(GEN6_UCGCTL2,
		3674	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		3675	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		3676	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3677
		3678	/* Bspec says we need to always set all mask bits. */
		3679	I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) \|
		3680	_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
		3681
		3682	/*
		3683	* According to the spec the following bits should be
		3684	* set in order to enable memory self-refresh and fbc:
		3685	* The bit21 and bit22 of 0x42000
		3686	* The bit21 and bit22 of 0x42004
		3687	* The bit5 and bit7 of 0x42020
		3688	* The bit14 of 0x70180
		3689	* The bit14 of 0x71180
		3690	*/
		3691	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		3692	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		3693	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		3694	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3695	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3696	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	3697	I915_WRITE(ILK_DSPCLK_GATE_D,
		3698	I915_READ(ILK_DSPCLK_GATE_D) \|
		3699	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		3700	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	3701
3243	Serge	3702	/* WaMbcDriverBootEnable */
3031	serge	3703	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3704	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3705
		3706	for_each_pipe(pipe) {
		3707	I915_WRITE(DSPCNTR(pipe),
		3708	I915_READ(DSPCNTR(pipe)) \|
		3709	DISPPLANE_TRICKLE_FEED_DISABLE);
		3710	intel_flush_display_plane(dev_priv, pipe);
		3711	}
		3712
		3713	/* The default value should be 0x200 according to docs, but the two
		3714	* platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
		3715	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
		3716	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
3243	Serge	3717
		3718	cpt_init_clock_gating(dev);
3480	Serge	3719
		3720	gen6_check_mch_setup(dev);
3031	serge	3721	}
		3722
		3723	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		3724	{
		3725	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		3726
		3727	reg &= ~GEN7_FF_SCHED_MASK;
		3728	reg \|= GEN7_FF_TS_SCHED_HW;
		3729	reg \|= GEN7_FF_VS_SCHED_HW;
		3730	reg \|= GEN7_FF_DS_SCHED_HW;
		3731
3480	Serge	3732	/* WaVSRefCountFullforceMissDisable */
		3733	if (IS_HASWELL(dev_priv->dev))
		3734	reg &= ~GEN7_FF_VS_REF_CNT_FFME;
		3735
3031	serge	3736	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		3737	}
		3738
3243	Serge	3739	static void lpt_init_clock_gating(struct drm_device *dev)
		3740	{
		3741	struct drm_i915_private *dev_priv = dev->dev_private;
		3742
		3743	/*
		3744	* TODO: this bit should only be enabled when really needed, then
		3745	* disabled when not needed anymore in order to save power.
		3746	*/
		3747	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
		3748	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		3749	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		3750	PCH_LP_PARTITION_LEVEL_DISABLE);
		3751	}
		3752
3031	serge	3753	static void haswell_init_clock_gating(struct drm_device *dev)
		3754	{
		3755	struct drm_i915_private *dev_priv = dev->dev_private;
		3756	int pipe;
		3757
		3758	I915_WRITE(WM3_LP_ILK, 0);
		3759	I915_WRITE(WM2_LP_ILK, 0);
		3760	I915_WRITE(WM1_LP_ILK, 0);
		3761
		3762	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3763	* This implements the WaDisableRCZUnitClockGating workaround.
		3764	*/
		3765	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
		3766
		3767	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3768	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3769	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3770
		3771	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3772	I915_WRITE(GEN7_L3CNTLREG1,
		3773	GEN7_WA_FOR_GEN7_L3_CONTROL);
		3774	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		3775	GEN7_WA_L3_CHICKEN_MODE);
		3776
		3777	/* This is required by WaCatErrorRejectionIssue */
		3778	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3779	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3780	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3781
		3782	for_each_pipe(pipe) {
		3783	I915_WRITE(DSPCNTR(pipe),
		3784	I915_READ(DSPCNTR(pipe)) \|
		3785	DISPPLANE_TRICKLE_FEED_DISABLE);
		3786	intel_flush_display_plane(dev_priv, pipe);
		3787	}
		3788
		3789	gen7_setup_fixed_func_scheduler(dev_priv);
		3790
		3791	/* WaDisable4x2SubspanOptimization */
		3792	I915_WRITE(CACHE_MODE_1,
		3793	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3794
3243	Serge	3795	/* WaMbcDriverBootEnable */
		3796	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3797	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3798
3031	serge	3799	/* XXX: This is a workaround for early silicon revisions and should be
		3800	* removed later.
		3801	*/
		3802	I915_WRITE(WM_DBG,
		3803	I915_READ(WM_DBG) \|
		3804	WM_DBG_DISALLOW_MULTIPLE_LP \|
		3805	WM_DBG_DISALLOW_SPRITE \|
		3806	WM_DBG_DISALLOW_MAXFIFO);
		3807
3243	Serge	3808	lpt_init_clock_gating(dev);
3031	serge	3809	}
		3810
		3811	static void ivybridge_init_clock_gating(struct drm_device *dev)
		3812	{
		3813	struct drm_i915_private *dev_priv = dev->dev_private;
		3814	int pipe;
		3815	uint32_t snpcr;
		3816
		3817	I915_WRITE(WM3_LP_ILK, 0);
		3818	I915_WRITE(WM2_LP_ILK, 0);
		3819	I915_WRITE(WM1_LP_ILK, 0);
		3820
3243	Serge	3821	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	3822
3243	Serge	3823	/* WaDisableEarlyCull */
		3824	I915_WRITE(_3D_CHICKEN3,
		3825	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		3826
		3827	/* WaDisableBackToBackFlipFix */
3031	serge	3828	I915_WRITE(IVB_CHICKEN3,
		3829	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3830	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3831
3243	Serge	3832	/* WaDisablePSDDualDispatchEnable */
		3833	if (IS_IVB_GT1(dev))
		3834	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		3835	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3836	else
		3837	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
		3838	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3839
3031	serge	3840	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3841	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3842	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3843
		3844	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3845	I915_WRITE(GEN7_L3CNTLREG1,
		3846	GEN7_WA_FOR_GEN7_L3_CONTROL);
		3847	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		3848	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	3849	if (IS_IVB_GT1(dev))
		3850	I915_WRITE(GEN7_ROW_CHICKEN2,
		3851	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		3852	else
		3853	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		3854	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3031	serge	3855
3243	Serge	3856
		3857	/* WaForceL3Serialization */
		3858	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3859	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3860
3031	serge	3861	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3862	* gating disable must be set. Failure to set it results in
		3863	* flickering pixels due to Z write ordering failures after
		3864	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3865	* Sanctuary and Tropics, and apparently anything else with
		3866	* alpha test or pixel discard.
		3867	*
		3868	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3869	* but we didn't debug actual testcases to find it out.
		3870	*
		3871	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3872	* This implements the WaDisableRCZUnitClockGating workaround.
		3873	*/
		3874	I915_WRITE(GEN6_UCGCTL2,
		3875	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		3876	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3877
		3878	/* This is required by WaCatErrorRejectionIssue */
		3879	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3880	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3881	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3882
		3883	for_each_pipe(pipe) {
		3884	I915_WRITE(DSPCNTR(pipe),
		3885	I915_READ(DSPCNTR(pipe)) \|
		3886	DISPPLANE_TRICKLE_FEED_DISABLE);
		3887	intel_flush_display_plane(dev_priv, pipe);
		3888	}
		3889
3243	Serge	3890	/* WaMbcDriverBootEnable */
3031	serge	3891	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3892	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3893
		3894	gen7_setup_fixed_func_scheduler(dev_priv);
		3895
		3896	/* WaDisable4x2SubspanOptimization */
		3897	I915_WRITE(CACHE_MODE_1,
		3898	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3899
		3900	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		3901	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		3902	snpcr \|= GEN6_MBC_SNPCR_MED;
		3903	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	3904
		3905	cpt_init_clock_gating(dev);
3480	Serge	3906
		3907	gen6_check_mch_setup(dev);
3031	serge	3908	}
		3909
		3910	static void valleyview_init_clock_gating(struct drm_device *dev)
		3911	{
		3912	struct drm_i915_private *dev_priv = dev->dev_private;
		3913	int pipe;
		3914
		3915	I915_WRITE(WM3_LP_ILK, 0);
		3916	I915_WRITE(WM2_LP_ILK, 0);
		3917	I915_WRITE(WM1_LP_ILK, 0);
		3918
3243	Serge	3919	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	3920
3243	Serge	3921	/* WaDisableEarlyCull */
		3922	I915_WRITE(_3D_CHICKEN3,
		3923	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		3924
		3925	/* WaDisableBackToBackFlipFix */
3031	serge	3926	I915_WRITE(IVB_CHICKEN3,
		3927	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3928	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3929
3243	Serge	3930	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		3931	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3932
3031	serge	3933	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3934	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3935	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3936
		3937	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
3243	Serge	3938	I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) \| GEN7_L3AGDIS);
3031	serge	3939	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
		3940
3243	Serge	3941	/* WaForceL3Serialization */
		3942	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3943	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3944
		3945	/* WaDisableDopClockGating */
		3946	I915_WRITE(GEN7_ROW_CHICKEN2,
		3947	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		3948
		3949	/* WaForceL3Serialization */
		3950	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3951	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3952
3031	serge	3953	/* This is required by WaCatErrorRejectionIssue */
		3954	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3955	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3956	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3957
3243	Serge	3958	/* WaMbcDriverBootEnable */
3031	serge	3959	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3960	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3961
		3962
		3963	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3964	* gating disable must be set. Failure to set it results in
		3965	* flickering pixels due to Z write ordering failures after
		3966	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3967	* Sanctuary and Tropics, and apparently anything else with
		3968	* alpha test or pixel discard.
		3969	*
		3970	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3971	* but we didn't debug actual testcases to find it out.
		3972	*
		3973	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3974	* This implements the WaDisableRCZUnitClockGating workaround.
		3975	*
		3976	* Also apply WaDisableVDSUnitClockGating and
		3977	* WaDisableRCPBUnitClockGating.
		3978	*/
		3979	I915_WRITE(GEN6_UCGCTL2,
		3980	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		3981	GEN7_TDLUNIT_CLOCK_GATE_DISABLE \|
		3982	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		3983	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		3984	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3985
		3986	I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
		3987
		3988	for_each_pipe(pipe) {
		3989	I915_WRITE(DSPCNTR(pipe),
		3990	I915_READ(DSPCNTR(pipe)) \|
		3991	DISPPLANE_TRICKLE_FEED_DISABLE);
		3992	intel_flush_display_plane(dev_priv, pipe);
		3993	}
		3994
		3995	I915_WRITE(CACHE_MODE_1,
		3996	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3997
		3998	/*
		3999	* On ValleyView, the GUnit needs to signal the GT
		4000	* when flip and other events complete. So enable
		4001	* all the GUnit->GT interrupts here
		4002	*/
		4003	I915_WRITE(VLV_DPFLIPSTAT, PIPEB_LINE_COMPARE_INT_EN \|
		4004	PIPEB_HLINE_INT_EN \| PIPEB_VBLANK_INT_EN \|
		4005	SPRITED_FLIPDONE_INT_EN \| SPRITEC_FLIPDONE_INT_EN \|
		4006	PLANEB_FLIPDONE_INT_EN \| PIPEA_LINE_COMPARE_INT_EN \|
		4007	PIPEA_HLINE_INT_EN \| PIPEA_VBLANK_INT_EN \|
		4008	SPRITEB_FLIPDONE_INT_EN \| SPRITEA_FLIPDONE_INT_EN \|
		4009	PLANEA_FLIPDONE_INT_EN);
3243	Serge	4010
		4011	/*
		4012	* WaDisableVLVClockGating_VBIIssue
		4013	* Disable clock gating on th GCFG unit to prevent a delay
		4014	* in the reporting of vblank events.
		4015	*/
		4016	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
3031	serge	4017	}
		4018
		4019	static void g4x_init_clock_gating(struct drm_device *dev)
		4020	{
		4021	struct drm_i915_private *dev_priv = dev->dev_private;
		4022	uint32_t dspclk_gate;
		4023
		4024	I915_WRITE(RENCLK_GATE_D1, 0);
		4025	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		4026	GS_UNIT_CLOCK_GATE_DISABLE \|
		4027	CL_UNIT_CLOCK_GATE_DISABLE);
		4028	I915_WRITE(RAMCLK_GATE_D, 0);
		4029	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		4030	OVRUNIT_CLOCK_GATE_DISABLE \|
		4031	OVCUNIT_CLOCK_GATE_DISABLE;
		4032	if (IS_GM45(dev))
		4033	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		4034	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	4035
		4036	/* WaDisableRenderCachePipelinedFlush */
		4037	I915_WRITE(CACHE_MODE_0,
		4038	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
3031	serge	4039	}
		4040
		4041	static void crestline_init_clock_gating(struct drm_device *dev)
		4042	{
		4043	struct drm_i915_private *dev_priv = dev->dev_private;
		4044
		4045	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		4046	I915_WRITE(RENCLK_GATE_D2, 0);
		4047	I915_WRITE(DSPCLK_GATE_D, 0);
		4048	I915_WRITE(RAMCLK_GATE_D, 0);
		4049	I915_WRITE16(DEUC, 0);
		4050	}
		4051
		4052	static void broadwater_init_clock_gating(struct drm_device *dev)
		4053	{
		4054	struct drm_i915_private *dev_priv = dev->dev_private;
		4055
		4056	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		4057	I965_RCC_CLOCK_GATE_DISABLE \|
		4058	I965_RCPB_CLOCK_GATE_DISABLE \|
		4059	I965_ISC_CLOCK_GATE_DISABLE \|
		4060	I965_FBC_CLOCK_GATE_DISABLE);
		4061	I915_WRITE(RENCLK_GATE_D2, 0);
		4062	}
		4063
		4064	static void gen3_init_clock_gating(struct drm_device *dev)
		4065	{
		4066	struct drm_i915_private *dev_priv = dev->dev_private;
		4067	u32 dstate = I915_READ(D_STATE);
		4068
		4069	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		4070	DSTATE_DOT_CLOCK_GATING;
		4071	I915_WRITE(D_STATE, dstate);
		4072
		4073	if (IS_PINEVIEW(dev))
		4074	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		4075
		4076	/* IIR "flip pending" means done if this bit is set */
		4077	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
		4078	}
		4079
		4080	static void i85x_init_clock_gating(struct drm_device *dev)
		4081	{
		4082	struct drm_i915_private *dev_priv = dev->dev_private;
		4083
		4084	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
		4085	}
		4086
		4087	static void i830_init_clock_gating(struct drm_device *dev)
		4088	{
		4089	struct drm_i915_private *dev_priv = dev->dev_private;
		4090
		4091	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
		4092	}
		4093
		4094	void intel_init_clock_gating(struct drm_device *dev)
		4095	{
		4096	struct drm_i915_private *dev_priv = dev->dev_private;
		4097
		4098	dev_priv->display.init_clock_gating(dev);
		4099	}
		4100
3480	Serge	4101	void intel_set_power_well(struct drm_device *dev, bool enable)
3031	serge	4102	{
		4103	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	4104	bool is_enabled, enable_requested;
		4105	uint32_t tmp;
3031	serge	4106
		4107	if (!IS_HASWELL(dev))
		4108	return;
		4109
3482	Serge	4110	if (!i915_disable_power_well && !enable)
		4111	return;
		4112
3480	Serge	4113	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
		4114	is_enabled = tmp & HSW_PWR_WELL_STATE;
		4115	enable_requested = tmp & HSW_PWR_WELL_ENABLE;
3031	serge	4116
3480	Serge	4117	if (enable) {
		4118	if (!enable_requested)
		4119	I915_WRITE(HSW_PWR_WELL_DRIVER, HSW_PWR_WELL_ENABLE);
3031	serge	4120
3480	Serge	4121	if (!is_enabled) {
		4122	DRM_DEBUG_KMS("Enabling power well\n");
		4123	if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
		4124	HSW_PWR_WELL_STATE), 20))
		4125	DRM_ERROR("Timeout enabling power well\n");
3031	serge	4126	}
3480	Serge	4127	} else {
		4128	if (enable_requested) {
		4129	I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
		4130	DRM_DEBUG_KMS("Requesting to disable the power well\n");
3031	serge	4131	}
3480	Serge	4132	}
		4133	}
3031	serge	4134
3480	Serge	4135	/*
		4136	* Starting with Haswell, we have a "Power Down Well" that can be turned off
		4137	* when not needed anymore. We have 4 registers that can request the power well
		4138	* to be enabled, and it will only be disabled if none of the registers is
		4139	* requesting it to be enabled.
		4140	*/
		4141	void intel_init_power_well(struct drm_device *dev)
		4142	{
		4143	struct drm_i915_private *dev_priv = dev->dev_private;
		4144
		4145	if (!IS_HASWELL(dev))
		4146	return;
		4147
		4148	/* For now, we need the power well to be always enabled. */
		4149	intel_set_power_well(dev, true);
		4150
		4151	/* We're taking over the BIOS, so clear any requests made by it since
		4152	* the driver is in charge now. */
		4153	if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE)
		4154	I915_WRITE(HSW_PWR_WELL_BIOS, 0);
3031	serge	4155	}
		4156
		4157	/* Set up chip specific power management-related functions */
		4158	void intel_init_pm(struct drm_device *dev)
		4159	{
		4160	struct drm_i915_private *dev_priv = dev->dev_private;
		4161
		4162	if (I915_HAS_FBC(dev)) {
		4163	if (HAS_PCH_SPLIT(dev)) {
		4164	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
		4165	dev_priv->display.enable_fbc = ironlake_enable_fbc;
		4166	dev_priv->display.disable_fbc = ironlake_disable_fbc;
		4167	} else if (IS_GM45(dev)) {
		4168	dev_priv->display.fbc_enabled = g4x_fbc_enabled;
		4169	dev_priv->display.enable_fbc = g4x_enable_fbc;
		4170	dev_priv->display.disable_fbc = g4x_disable_fbc;
		4171	} else if (IS_CRESTLINE(dev)) {
		4172	dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
		4173	dev_priv->display.enable_fbc = i8xx_enable_fbc;
		4174	dev_priv->display.disable_fbc = i8xx_disable_fbc;
		4175	}
		4176	/* 855GM needs testing */
		4177	}
		4178
		4179	/* For cxsr */
		4180	if (IS_PINEVIEW(dev))
		4181	i915_pineview_get_mem_freq(dev);
		4182	else if (IS_GEN5(dev))
		4183	i915_ironlake_get_mem_freq(dev);
		4184
		4185	/* For FIFO watermark updates */
		4186	if (HAS_PCH_SPLIT(dev)) {
		4187	if (IS_GEN5(dev)) {
		4188	if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
		4189	dev_priv->display.update_wm = ironlake_update_wm;
		4190	else {
		4191	DRM_DEBUG_KMS("Failed to get proper latency. "
		4192	"Disable CxSR\n");
		4193	dev_priv->display.update_wm = NULL;
		4194	}
		4195	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		4196	} else if (IS_GEN6(dev)) {
		4197	if (SNB_READ_WM0_LATENCY()) {
		4198	dev_priv->display.update_wm = sandybridge_update_wm;
		4199	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4200	} else {
		4201	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4202	"Disable CxSR\n");
		4203	dev_priv->display.update_wm = NULL;
		4204	}
		4205	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
		4206	} else if (IS_IVYBRIDGE(dev)) {
		4207	/* FIXME: detect B0+ stepping and use auto training */
		4208	if (SNB_READ_WM0_LATENCY()) {
3243	Serge	4209	dev_priv->display.update_wm = ivybridge_update_wm;
3031	serge	4210	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4211	} else {
		4212	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4213	"Disable CxSR\n");
		4214	dev_priv->display.update_wm = NULL;
		4215	}
		4216	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
		4217	} else if (IS_HASWELL(dev)) {
		4218	if (SNB_READ_WM0_LATENCY()) {
		4219	dev_priv->display.update_wm = sandybridge_update_wm;
		4220	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4221	dev_priv->display.update_linetime_wm = haswell_update_linetime_wm;
		4222	} else {
		4223	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4224	"Disable CxSR\n");
		4225	dev_priv->display.update_wm = NULL;
		4226	}
		4227	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
		4228	} else
		4229	dev_priv->display.update_wm = NULL;
		4230	} else if (IS_VALLEYVIEW(dev)) {
		4231	dev_priv->display.update_wm = valleyview_update_wm;
		4232	dev_priv->display.init_clock_gating =
		4233	valleyview_init_clock_gating;
		4234	} else if (IS_PINEVIEW(dev)) {
		4235	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		4236	dev_priv->is_ddr3,
		4237	dev_priv->fsb_freq,
		4238	dev_priv->mem_freq)) {
		4239	DRM_INFO("failed to find known CxSR latency "
		4240	"(found ddr%s fsb freq %d, mem freq %d), "
		4241	"disabling CxSR\n",
		4242	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		4243	dev_priv->fsb_freq, dev_priv->mem_freq);
		4244	/* Disable CxSR and never update its watermark again */
		4245	pineview_disable_cxsr(dev);
		4246	dev_priv->display.update_wm = NULL;
		4247	} else
		4248	dev_priv->display.update_wm = pineview_update_wm;
		4249	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		4250	} else if (IS_G4X(dev)) {
		4251	dev_priv->display.update_wm = g4x_update_wm;
		4252	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		4253	} else if (IS_GEN4(dev)) {
		4254	dev_priv->display.update_wm = i965_update_wm;
		4255	if (IS_CRESTLINE(dev))
		4256	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		4257	else if (IS_BROADWATER(dev))
		4258	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		4259	} else if (IS_GEN3(dev)) {
		4260	dev_priv->display.update_wm = i9xx_update_wm;
		4261	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		4262	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		4263	} else if (IS_I865G(dev)) {
		4264	dev_priv->display.update_wm = i830_update_wm;
		4265	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		4266	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		4267	} else if (IS_I85X(dev)) {
		4268	dev_priv->display.update_wm = i9xx_update_wm;
		4269	dev_priv->display.get_fifo_size = i85x_get_fifo_size;
		4270	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		4271	} else {
		4272	dev_priv->display.update_wm = i830_update_wm;
		4273	dev_priv->display.init_clock_gating = i830_init_clock_gating;
		4274	if (IS_845G(dev))
		4275	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		4276	else
		4277	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		4278	}
		4279	}
		4280
		4281	static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
		4282	{
		4283	u32 gt_thread_status_mask;
		4284
		4285	if (IS_HASWELL(dev_priv->dev))
		4286	gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
		4287	else
		4288	gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
		4289
		4290	/* w/a for a sporadic read returning 0 by waiting for the GT
		4291	* thread to wake up.
		4292	*/
		4293	if (wait_for_atomic_us((I915_READ_NOTRACE(GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
		4294	DRM_ERROR("GT thread status wait timed out\n");
		4295	}
		4296
3243	Serge	4297	static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
		4298	{
		4299	I915_WRITE_NOTRACE(FORCEWAKE, 0);
		4300	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
		4301	}
		4302
3031	serge	4303	static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
		4304	{
		4305	u32 forcewake_ack;
		4306
		4307	if (IS_HASWELL(dev_priv->dev))
		4308	forcewake_ack = FORCEWAKE_ACK_HSW;
		4309	else
		4310	forcewake_ack = FORCEWAKE_ACK;
		4311
		4312	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1) == 0,
		4313	FORCEWAKE_ACK_TIMEOUT_MS))
		4314	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4315
3243	Serge	4316	I915_WRITE_NOTRACE(FORCEWAKE, FORCEWAKE_KERNEL);
3031	serge	4317	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
		4318
		4319	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
		4320	FORCEWAKE_ACK_TIMEOUT_MS))
		4321	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4322
		4323	__gen6_gt_wait_for_thread_c0(dev_priv);
		4324	}
		4325
3243	Serge	4326	static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
		4327	{
		4328	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
		4329	/* something from same cacheline, but !FORCEWAKE_MT */
		4330	POSTING_READ(ECOBUS);
		4331	}
		4332
3031	serge	4333	static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
		4334	{
		4335	u32 forcewake_ack;
		4336
		4337	if (IS_HASWELL(dev_priv->dev))
		4338	forcewake_ack = FORCEWAKE_ACK_HSW;
		4339	else
		4340	forcewake_ack = FORCEWAKE_MT_ACK;
		4341
		4342	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1) == 0,
		4343	FORCEWAKE_ACK_TIMEOUT_MS))
		4344	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4345
3243	Serge	4346	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
		4347	/* something from same cacheline, but !FORCEWAKE_MT */
		4348	POSTING_READ(ECOBUS);
3031	serge	4349
		4350	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
		4351	FORCEWAKE_ACK_TIMEOUT_MS))
		4352	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4353
		4354	__gen6_gt_wait_for_thread_c0(dev_priv);
		4355	}
		4356
		4357	/*
		4358	* Generally this is called implicitly by the register read function. However,
		4359	* if some sequence requires the GT to not power down then this function should
		4360	* be called at the beginning of the sequence followed by a call to
		4361	* gen6_gt_force_wake_put() at the end of the sequence.
		4362	*/
		4363	void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
		4364	{
		4365	unsigned long irqflags;
		4366
		4367	spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
		4368	if (dev_priv->forcewake_count++ == 0)
		4369	dev_priv->gt.force_wake_get(dev_priv);
		4370	spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
		4371	}
		4372
		4373	void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
		4374	{
		4375	u32 gtfifodbg;
		4376	gtfifodbg = I915_READ_NOTRACE(GTFIFODBG);
		4377	if (WARN(gtfifodbg & GT_FIFO_CPU_ERROR_MASK,
		4378	"MMIO read or write has been dropped %x\n", gtfifodbg))
		4379	I915_WRITE_NOTRACE(GTFIFODBG, GT_FIFO_CPU_ERROR_MASK);
		4380	}
		4381
		4382	static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
		4383	{
		4384	I915_WRITE_NOTRACE(FORCEWAKE, 0);
3243	Serge	4385	/* something from same cacheline, but !FORCEWAKE */
		4386	POSTING_READ(ECOBUS);
3031	serge	4387	gen6_gt_check_fifodbg(dev_priv);
		4388	}
		4389
		4390	static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
		4391	{
3243	Serge	4392	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
		4393	/* something from same cacheline, but !FORCEWAKE_MT */
		4394	POSTING_READ(ECOBUS);
3031	serge	4395	gen6_gt_check_fifodbg(dev_priv);
		4396	}
		4397
		4398	/*
		4399	* see gen6_gt_force_wake_get()
		4400	*/
		4401	void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
		4402	{
		4403	unsigned long irqflags;
		4404
		4405	spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
		4406	if (--dev_priv->forcewake_count == 0)
		4407	dev_priv->gt.force_wake_put(dev_priv);
		4408	spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
		4409	}
		4410
		4411	int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
		4412	{
		4413	int ret = 0;
		4414
		4415	if (dev_priv->gt_fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
		4416	int loop = 500;
		4417	u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
		4418	while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
		4419	udelay(10);
		4420	fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
		4421	}
		4422	if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
		4423	++ret;
		4424	dev_priv->gt_fifo_count = fifo;
		4425	}
		4426	dev_priv->gt_fifo_count--;
		4427
		4428	return ret;
		4429	}
		4430
3243	Serge	4431	static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
		4432	{
		4433	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(0xffff));
		4434	/* something from same cacheline, but !FORCEWAKE_VLV */
		4435	POSTING_READ(FORCEWAKE_ACK_VLV);
		4436	}
		4437
3031	serge	4438	static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
		4439	{
		4440	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1) == 0,
		4441	FORCEWAKE_ACK_TIMEOUT_MS))
		4442	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4443
3243	Serge	4444	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
3031	serge	4445
		4446	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1),
		4447	FORCEWAKE_ACK_TIMEOUT_MS))
		4448	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4449
		4450	__gen6_gt_wait_for_thread_c0(dev_priv);
		4451	}
		4452
		4453	static void vlv_force_wake_put(struct drm_i915_private *dev_priv)
		4454	{
3243	Serge	4455	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
		4456	/* something from same cacheline, but !FORCEWAKE_VLV */
		4457	POSTING_READ(FORCEWAKE_ACK_VLV);
3031	serge	4458	gen6_gt_check_fifodbg(dev_priv);
		4459	}
		4460
3243	Serge	4461	void intel_gt_reset(struct drm_device *dev)
		4462	{
		4463	struct drm_i915_private *dev_priv = dev->dev_private;
		4464
		4465	if (IS_VALLEYVIEW(dev)) {
		4466	vlv_force_wake_reset(dev_priv);
		4467	} else if (INTEL_INFO(dev)->gen >= 6) {
		4468	__gen6_gt_force_wake_reset(dev_priv);
		4469	if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev))
		4470	__gen6_gt_force_wake_mt_reset(dev_priv);
		4471	}
		4472	}
		4473
3031	serge	4474	void intel_gt_init(struct drm_device *dev)
		4475	{
		4476	struct drm_i915_private *dev_priv = dev->dev_private;
		4477
		4478	spin_lock_init(&dev_priv->gt_lock);
		4479
3243	Serge	4480	intel_gt_reset(dev);
		4481
3031	serge	4482	if (IS_VALLEYVIEW(dev)) {
		4483	dev_priv->gt.force_wake_get = vlv_force_wake_get;
		4484	dev_priv->gt.force_wake_put = vlv_force_wake_put;
3243	Serge	4485	} else if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev)) {
		4486	dev_priv->gt.force_wake_get = __gen6_gt_force_wake_mt_get;
		4487	dev_priv->gt.force_wake_put = __gen6_gt_force_wake_mt_put;
		4488	} else if (IS_GEN6(dev)) {
3031	serge	4489	dev_priv->gt.force_wake_get = __gen6_gt_force_wake_get;
		4490	dev_priv->gt.force_wake_put = __gen6_gt_force_wake_put;
3243	Serge	4491	}
3482	Serge	4492	INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
		4493	intel_gen6_powersave_work);
3243	Serge	4494	}
3031	serge	4495
3243	Serge	4496	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u8 mbox, u32 val)
		4497	{
		4498	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4499
3243	Serge	4500	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		4501	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		4502	return -EAGAIN;
		4503	}
3031	serge	4504
3243	Serge	4505	I915_WRITE(GEN6_PCODE_DATA, *val);
		4506	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		4507
		4508	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		4509	500)) {
		4510	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		4511	return -ETIMEDOUT;
3031	serge	4512	}
3243	Serge	4513
		4514	*val = I915_READ(GEN6_PCODE_DATA);
		4515	I915_WRITE(GEN6_PCODE_DATA, 0);
		4516
		4517	return 0;
		4518	}
		4519
		4520	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
		4521	{
		4522	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4523
		4524	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		4525	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		4526	return -EAGAIN;
3031	serge	4527	}
3243	Serge	4528
		4529	I915_WRITE(GEN6_PCODE_DATA, val);
		4530	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		4531
		4532	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		4533	500)) {
		4534	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		4535	return -ETIMEDOUT;
3031	serge	4536	}
3243	Serge	4537
		4538	I915_WRITE(GEN6_PCODE_DATA, 0);
		4539
		4540	return 0;
3031	serge	4541	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 3482