WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	#define iowrite32(v, addr) writel((v), (addr))
		29	#define ioread32(addr) readl(addr)
		30
		31	//#include
		32	#include "i915_drv.h"
		33	#include "intel_drv.h"
		34	#include
		35	//#include "../../../platform/x86/intel_ips.h"
		36	#include
		37
		38	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		39
		40	#define assert_spin_locked(x)
		41
		42	void getrawmonotonic(struct timespec *ts);
		43	void set_normalized_timespec(struct timespec *ts, time_t sec, long nsec);
		44
		45	static inline struct timespec timespec_sub(struct timespec lhs,
		46	struct timespec rhs)
		47	{
		48	struct timespec ts_delta;
		49	set_normalized_timespec(&ts_delta, lhs.tv_sec - rhs.tv_sec,
		50	lhs.tv_nsec - rhs.tv_nsec);
		51	return ts_delta;
		52	}
		53
		54
		55	/* FBC, or Frame Buffer Compression, is a technique employed to compress the
		56	* framebuffer contents in-memory, aiming at reducing the required bandwidth
		57	* during in-memory transfers and, therefore, reduce the power packet.
		58	*
		59	* The benefits of FBC are mostly visible with solid backgrounds and
		60	* variation-less patterns.
		61	*
		62	* FBC-related functionality can be enabled by the means of the
		63	* i915.i915_enable_fbc parameter
		64	*/
		65
3243	Serge	66	static bool intel_crtc_active(struct drm_crtc *crtc)
		67	{
		68	/* Be paranoid as we can arrive here with only partial
		69	* state retrieved from the hardware during setup.
		70	*/
		71	return to_intel_crtc(crtc)->active && crtc->fb && crtc->mode.clock;
		72	}
		73
3031	serge	74	static void i8xx_disable_fbc(struct drm_device *dev)
		75	{
		76	struct drm_i915_private *dev_priv = dev->dev_private;
		77	u32 fbc_ctl;
		78
		79	/* Disable compression */
		80	fbc_ctl = I915_READ(FBC_CONTROL);
		81	if ((fbc_ctl & FBC_CTL_EN) == 0)
		82	return;
		83
		84	fbc_ctl &= ~FBC_CTL_EN;
		85	I915_WRITE(FBC_CONTROL, fbc_ctl);
		86
		87	/* Wait for compressing bit to clear */
		88	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
		89	DRM_DEBUG_KMS("FBC idle timed out\n");
		90	return;
		91	}
		92
		93	DRM_DEBUG_KMS("disabled FBC\n");
		94	}
		95
		96	static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		97	{
		98	struct drm_device *dev = crtc->dev;
		99	struct drm_i915_private *dev_priv = dev->dev_private;
		100	struct drm_framebuffer *fb = crtc->fb;
		101	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		102	struct drm_i915_gem_object *obj = intel_fb->obj;
		103	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		104	int cfb_pitch;
		105	int plane, i;
		106	u32 fbc_ctl, fbc_ctl2;
		107
		108	cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
		109	if (fb->pitches[0] < cfb_pitch)
		110	cfb_pitch = fb->pitches[0];
		111
		112	/* FBC_CTL wants 64B units */
		113	cfb_pitch = (cfb_pitch / 64) - 1;
		114	plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
		115
		116	/* Clear old tags */
		117	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
		118	I915_WRITE(FBC_TAG + (i * 4), 0);
		119
		120	/* Set it up... */
		121	fbc_ctl2 = FBC_CTL_FENCE_DBL \| FBC_CTL_IDLE_IMM \| FBC_CTL_CPU_FENCE;
		122	fbc_ctl2 \|= plane;
		123	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
		124	I915_WRITE(FBC_FENCE_OFF, crtc->y);
		125
		126	/* enable it... */
		127	fbc_ctl = FBC_CTL_EN \| FBC_CTL_PERIODIC;
		128	if (IS_I945GM(dev))
		129	fbc_ctl \|= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
		130	fbc_ctl \|= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
		131	fbc_ctl \|= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
		132	fbc_ctl \|= obj->fence_reg;
		133	I915_WRITE(FBC_CONTROL, fbc_ctl);
		134
		135	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
		136	cfb_pitch, crtc->y, intel_crtc->plane);
		137	}
		138
		139	static bool i8xx_fbc_enabled(struct drm_device *dev)
		140	{
		141	struct drm_i915_private *dev_priv = dev->dev_private;
		142
		143	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
		144	}
		145
		146	static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		147	{
		148	struct drm_device *dev = crtc->dev;
		149	struct drm_i915_private *dev_priv = dev->dev_private;
		150	struct drm_framebuffer *fb = crtc->fb;
		151	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		152	struct drm_i915_gem_object *obj = intel_fb->obj;
		153	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		154	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		155	unsigned long stall_watermark = 200;
		156	u32 dpfc_ctl;
		157
		158	dpfc_ctl = plane \| DPFC_SR_EN \| DPFC_CTL_LIMIT_1X;
		159	dpfc_ctl \|= DPFC_CTL_FENCE_EN \| obj->fence_reg;
		160	I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
		161
		162	I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		163	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		164	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		165	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
		166
		167	/* enable it... */
		168	I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) \| DPFC_CTL_EN);
		169
		170	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		171	}
		172
		173	static void g4x_disable_fbc(struct drm_device *dev)
		174	{
		175	struct drm_i915_private *dev_priv = dev->dev_private;
		176	u32 dpfc_ctl;
		177
		178	/* Disable compression */
		179	dpfc_ctl = I915_READ(DPFC_CONTROL);
		180	if (dpfc_ctl & DPFC_CTL_EN) {
		181	dpfc_ctl &= ~DPFC_CTL_EN;
		182	I915_WRITE(DPFC_CONTROL, dpfc_ctl);
		183
		184	DRM_DEBUG_KMS("disabled FBC\n");
		185	}
		186	}
		187
		188	static bool g4x_fbc_enabled(struct drm_device *dev)
		189	{
		190	struct drm_i915_private *dev_priv = dev->dev_private;
		191
		192	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
		193	}
		194
		195	static void sandybridge_blit_fbc_update(struct drm_device *dev)
		196	{
		197	struct drm_i915_private *dev_priv = dev->dev_private;
		198	u32 blt_ecoskpd;
		199
		200	/* Make sure blitter notifies FBC of writes */
		201	gen6_gt_force_wake_get(dev_priv);
		202	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
		203	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY <<
		204	GEN6_BLITTER_LOCK_SHIFT;
		205	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		206	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY;
		207	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		208	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
		209	GEN6_BLITTER_LOCK_SHIFT);
		210	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		211	POSTING_READ(GEN6_BLITTER_ECOSKPD);
		212	gen6_gt_force_wake_put(dev_priv);
		213	}
		214
		215	static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		216	{
		217	struct drm_device *dev = crtc->dev;
		218	struct drm_i915_private *dev_priv = dev->dev_private;
		219	struct drm_framebuffer *fb = crtc->fb;
		220	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		221	struct drm_i915_gem_object *obj = intel_fb->obj;
		222	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		223	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		224	unsigned long stall_watermark = 200;
		225	u32 dpfc_ctl;
		226
		227	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		228	dpfc_ctl &= DPFC_RESERVED;
		229	dpfc_ctl \|= (plane \| DPFC_CTL_LIMIT_1X);
		230	/* Set persistent mode for front-buffer rendering, ala X. */
		231	dpfc_ctl \|= DPFC_CTL_PERSISTENT_MODE;
		232	dpfc_ctl \|= (DPFC_CTL_FENCE_EN \| obj->fence_reg);
		233	I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
		234
		235	I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		236	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		237	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		238	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
		239	I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset \| ILK_FBC_RT_VALID);
		240	/* enable it... */
		241	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		242
		243	if (IS_GEN6(dev)) {
		244	I915_WRITE(SNB_DPFC_CTL_SA,
		245	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		246	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		247	sandybridge_blit_fbc_update(dev);
		248	}
		249
		250	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		251	}
		252
		253	static void ironlake_disable_fbc(struct drm_device *dev)
		254	{
		255	struct drm_i915_private *dev_priv = dev->dev_private;
		256	u32 dpfc_ctl;
		257
		258	/* Disable compression */
		259	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		260	if (dpfc_ctl & DPFC_CTL_EN) {
		261	dpfc_ctl &= ~DPFC_CTL_EN;
		262	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
		263
		264	DRM_DEBUG_KMS("disabled FBC\n");
		265	}
		266	}
		267
		268	static bool ironlake_fbc_enabled(struct drm_device *dev)
		269	{
		270	struct drm_i915_private *dev_priv = dev->dev_private;
		271
		272	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
		273	}
		274
		275	bool intel_fbc_enabled(struct drm_device *dev)
		276	{
		277	struct drm_i915_private *dev_priv = dev->dev_private;
		278
		279	if (!dev_priv->display.fbc_enabled)
		280	return false;
		281
		282	return dev_priv->display.fbc_enabled(dev);
		283	}
		284
		285	#if 0
		286	static void intel_fbc_work_fn(struct work_struct *__work)
		287	{
		288	struct intel_fbc_work *work =
		289	container_of(to_delayed_work(__work),
		290	struct intel_fbc_work, work);
		291	struct drm_device *dev = work->crtc->dev;
		292	struct drm_i915_private *dev_priv = dev->dev_private;
		293
		294	mutex_lock(&dev->struct_mutex);
		295	if (work == dev_priv->fbc_work) {
		296	/* Double check that we haven't switched fb without cancelling
		297	* the prior work.
		298	*/
		299	if (work->crtc->fb == work->fb) {
		300	dev_priv->display.enable_fbc(work->crtc,
		301	work->interval);
		302
		303	dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
		304	dev_priv->cfb_fb = work->crtc->fb->base.id;
		305	dev_priv->cfb_y = work->crtc->y;
		306	}
		307
		308	dev_priv->fbc_work = NULL;
		309	}
		310	mutex_unlock(&dev->struct_mutex);
		311
		312	kfree(work);
		313	}
		314
		315	static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
		316	{
		317	if (dev_priv->fbc_work == NULL)
		318	return;
		319
		320	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
		321
		322	/* Synchronisation is provided by struct_mutex and checking of
		323	* dev_priv->fbc_work, so we can perform the cancellation
		324	* entirely asynchronously.
		325	*/
		326	if (cancel_delayed_work(&dev_priv->fbc_work->work))
		327	/* tasklet was killed before being run, clean up */
		328	kfree(dev_priv->fbc_work);
		329
		330	/* Mark the work as no longer wanted so that if it does
		331	* wake-up (because the work was already running and waiting
		332	* for our mutex), it will discover that is no longer
		333	* necessary to run.
		334	*/
		335	dev_priv->fbc_work = NULL;
		336	}
		337	#endif
		338
		339	void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		340	{
		341	struct intel_fbc_work *work;
		342	struct drm_device *dev = crtc->dev;
		343	struct drm_i915_private *dev_priv = dev->dev_private;
		344
		345	// if (!dev_priv->display.enable_fbc)
		346	return;
		347	#if 0
		348	intel_cancel_fbc_work(dev_priv);
		349
		350	work = kzalloc(sizeof *work, GFP_KERNEL);
		351	if (work == NULL) {
		352	dev_priv->display.enable_fbc(crtc, interval);
		353	return;
		354	}
		355
		356	work->crtc = crtc;
		357	work->fb = crtc->fb;
		358	work->interval = interval;
		359	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
		360
		361	dev_priv->fbc_work = work;
		362
		363	DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
		364
		365	/* Delay the actual enabling to let pageflipping cease and the
		366	* display to settle before starting the compression. Note that
		367	* this delay also serves a second purpose: it allows for a
		368	* vblank to pass after disabling the FBC before we attempt
		369	* to modify the control registers.
		370	*
		371	* A more complicated solution would involve tracking vblanks
		372	* following the termination of the page-flipping sequence
		373	* and indeed performing the enable as a co-routine and not
		374	* waiting synchronously upon the vblank.
		375	*/
		376	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
		377	#endif
		378
		379	}
		380
		381	void intel_disable_fbc(struct drm_device *dev)
		382	{
		383	struct drm_i915_private *dev_priv = dev->dev_private;
		384
		385	// intel_cancel_fbc_work(dev_priv);
		386
		387	// if (!dev_priv->display.disable_fbc)
		388	// return;
		389
		390	// dev_priv->display.disable_fbc(dev);
		391	dev_priv->cfb_plane = -1;
		392	}
		393
		394	/**
		395	* intel_update_fbc - enable/disable FBC as needed
		396	* @dev: the drm_device
		397	*
		398	* Set up the framebuffer compression hardware at mode set time. We
		399	* enable it if possible:
		400	* - plane A only (on pre-965)
		401	* - no pixel mulitply/line duplication
		402	* - no alpha buffer discard
		403	* - no dual wide
		404	* - framebuffer <= 2048 in width, 1536 in height
		405	*
		406	* We can't assume that any compression will take place (worst case),
		407	* so the compressed buffer has to be the same size as the uncompressed
		408	* one. It also must reside (along with the line length buffer) in
		409	* stolen memory.
		410	*
		411	* We need to enable/disable FBC on a global basis.
		412	*/
		413	void intel_update_fbc(struct drm_device *dev)
		414	{
		415	struct drm_i915_private *dev_priv = dev->dev_private;
		416	struct drm_crtc crtc = NULL, tmp_crtc;
		417	struct intel_crtc *intel_crtc;
		418	struct drm_framebuffer *fb;
		419	struct intel_framebuffer *intel_fb;
		420	struct drm_i915_gem_object *obj;
		421	int enable_fbc;
		422
		423	if (!i915_powersave)
		424	return;
		425
		426	if (!I915_HAS_FBC(dev))
		427	return;
		428
		429	/*
		430	* If FBC is already on, we just have to verify that we can
		431	* keep it that way...
		432	* Need to disable if:
		433	* - more than one pipe is active
		434	* - changing FBC params (stride, fence, mode)
		435	* - new fb is too large to fit in compressed buffer
		436	* - going to an unsupported config (interlace, pixel multiply, etc.)
		437	*/
		438	list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	439	if (intel_crtc_active(tmp_crtc) &&
		440	!to_intel_crtc(tmp_crtc)->primary_disabled) {
3031	serge	441	if (crtc) {
		442	DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
		443	dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
		444	goto out_disable;
		445	}
		446	crtc = tmp_crtc;
		447	}
		448	}
		449
		450	if (!crtc \|\| crtc->fb == NULL) {
		451	DRM_DEBUG_KMS("no output, disabling\n");
		452	dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
		453	goto out_disable;
		454	}
		455
		456	intel_crtc = to_intel_crtc(crtc);
		457	fb = crtc->fb;
		458	intel_fb = to_intel_framebuffer(fb);
		459	obj = intel_fb->obj;
		460
		461	enable_fbc = i915_enable_fbc;
		462	if (enable_fbc < 0) {
		463	DRM_DEBUG_KMS("fbc set to per-chip default\n");
		464	enable_fbc = 1;
		465	if (INTEL_INFO(dev)->gen <= 6)
		466	enable_fbc = 0;
		467	}
		468	if (!enable_fbc) {
		469	DRM_DEBUG_KMS("fbc disabled per module param\n");
		470	dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
		471	goto out_disable;
		472	}
		473	if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) \|\|
		474	(crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
		475	DRM_DEBUG_KMS("mode incompatible with compression, "
		476	"disabling\n");
		477	dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
		478	goto out_disable;
		479	}
		480	if ((crtc->mode.hdisplay > 2048) \|\|
		481	(crtc->mode.vdisplay > 1536)) {
		482	DRM_DEBUG_KMS("mode too large for compression, disabling\n");
		483	dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
		484	goto out_disable;
		485	}
		486	if ((IS_I915GM(dev) \|\| IS_I945GM(dev)) && intel_crtc->plane != 0) {
		487	DRM_DEBUG_KMS("plane not 0, disabling compression\n");
		488	dev_priv->no_fbc_reason = FBC_BAD_PLANE;
		489	goto out_disable;
		490	}
		491
		492	/* The use of a CPU fence is mandatory in order to detect writes
		493	* by the CPU to the scanout and trigger updates to the FBC.
		494	*/
		495	if (obj->tiling_mode != I915_TILING_X \|\|
		496	obj->fence_reg == I915_FENCE_REG_NONE) {
		497	DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
		498	dev_priv->no_fbc_reason = FBC_NOT_TILED;
		499	goto out_disable;
		500	}
		501
		502	/* If the kernel debugger is active, always disable compression */
		503	if (in_dbg_master())
		504	goto out_disable;
		505
3480	Serge	506	if (i915_gem_stolen_setup_compression(dev, intel_fb->obj->base.size)) {
		507	DRM_INFO("not enough stolen space for compressed buffer (need %zd bytes), disabling\n", intel_fb->obj->base.size);
		508	DRM_INFO("hint: you may be able to increase stolen memory size in the BIOS to avoid this\n");
		509	DRM_DEBUG_KMS("framebuffer too large, disabling compression\n");
		510	dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
		511	goto out_disable;
		512	}
		513
3031	serge	514	/* If the scanout has not changed, don't modify the FBC settings.
		515	* Note that we make the fundamental assumption that the fb->obj
		516	* cannot be unpinned (and have its GTT offset and fence revoked)
		517	* without first being decoupled from the scanout and FBC disabled.
		518	*/
		519	if (dev_priv->cfb_plane == intel_crtc->plane &&
		520	dev_priv->cfb_fb == fb->base.id &&
		521	dev_priv->cfb_y == crtc->y)
		522	return;
		523
		524	if (intel_fbc_enabled(dev)) {
		525	/* We update FBC along two paths, after changing fb/crtc
		526	* configuration (modeswitching) and after page-flipping
		527	* finishes. For the latter, we know that not only did
		528	* we disable the FBC at the start of the page-flip
		529	* sequence, but also more than one vblank has passed.
		530	*
		531	* For the former case of modeswitching, it is possible
		532	* to switch between two FBC valid configurations
		533	* instantaneously so we do need to disable the FBC
		534	* before we can modify its control registers. We also
		535	* have to wait for the next vblank for that to take
		536	* effect. However, since we delay enabling FBC we can
		537	* assume that a vblank has passed since disabling and
		538	* that we can safely alter the registers in the deferred
		539	* callback.
		540	*
		541	* In the scenario that we go from a valid to invalid
		542	* and then back to valid FBC configuration we have
		543	* no strict enforcement that a vblank occurred since
		544	* disabling the FBC. However, along all current pipe
		545	* disabling paths we do need to wait for a vblank at
		546	* some point. And we wait before enabling FBC anyway.
		547	*/
		548	DRM_DEBUG_KMS("disabling active FBC for update\n");
		549	intel_disable_fbc(dev);
		550	}
		551
		552	intel_enable_fbc(crtc, 500);
		553	return;
		554
		555	out_disable:
		556	/* Multiple disables should be harmless */
		557	if (intel_fbc_enabled(dev)) {
		558	DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
		559	intel_disable_fbc(dev);
		560	}
3480	Serge	561	i915_gem_stolen_cleanup_compression(dev);
3031	serge	562	}
		563
		564	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		565	{
		566	drm_i915_private_t *dev_priv = dev->dev_private;
		567	u32 tmp;
		568
		569	tmp = I915_READ(CLKCFG);
		570
		571	switch (tmp & CLKCFG_FSB_MASK) {
		572	case CLKCFG_FSB_533:
		573	dev_priv->fsb_freq = 533; /* 1334 /
		574	break;
		575	case CLKCFG_FSB_800:
		576	dev_priv->fsb_freq = 800; /* 2004 /
		577	break;
		578	case CLKCFG_FSB_667:
		579	dev_priv->fsb_freq = 667; /* 1674 /
		580	break;
		581	case CLKCFG_FSB_400:
		582	dev_priv->fsb_freq = 400; /* 1004 /
		583	break;
		584	}
		585
		586	switch (tmp & CLKCFG_MEM_MASK) {
		587	case CLKCFG_MEM_533:
		588	dev_priv->mem_freq = 533;
		589	break;
		590	case CLKCFG_MEM_667:
		591	dev_priv->mem_freq = 667;
		592	break;
		593	case CLKCFG_MEM_800:
		594	dev_priv->mem_freq = 800;
		595	break;
		596	}
		597
		598	/* detect pineview DDR3 setting */
		599	tmp = I915_READ(CSHRDDR3CTL);
		600	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		601	}
		602
		603	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		604	{
		605	drm_i915_private_t *dev_priv = dev->dev_private;
		606	u16 ddrpll, csipll;
		607
		608	ddrpll = I915_READ16(DDRMPLL1);
		609	csipll = I915_READ16(CSIPLL0);
		610
		611	switch (ddrpll & 0xff) {
		612	case 0xc:
		613	dev_priv->mem_freq = 800;
		614	break;
		615	case 0x10:
		616	dev_priv->mem_freq = 1066;
		617	break;
		618	case 0x14:
		619	dev_priv->mem_freq = 1333;
		620	break;
		621	case 0x18:
		622	dev_priv->mem_freq = 1600;
		623	break;
		624	default:
		625	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		626	ddrpll & 0xff);
		627	dev_priv->mem_freq = 0;
		628	break;
		629	}
		630
		631	dev_priv->ips.r_t = dev_priv->mem_freq;
		632
		633	switch (csipll & 0x3ff) {
		634	case 0x00c:
		635	dev_priv->fsb_freq = 3200;
		636	break;
		637	case 0x00e:
		638	dev_priv->fsb_freq = 3733;
		639	break;
		640	case 0x010:
		641	dev_priv->fsb_freq = 4266;
		642	break;
		643	case 0x012:
		644	dev_priv->fsb_freq = 4800;
		645	break;
		646	case 0x014:
		647	dev_priv->fsb_freq = 5333;
		648	break;
		649	case 0x016:
		650	dev_priv->fsb_freq = 5866;
		651	break;
		652	case 0x018:
		653	dev_priv->fsb_freq = 6400;
		654	break;
		655	default:
		656	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		657	csipll & 0x3ff);
		658	dev_priv->fsb_freq = 0;
		659	break;
		660	}
		661
		662	if (dev_priv->fsb_freq == 3200) {
		663	dev_priv->ips.c_m = 0;
		664	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		665	dev_priv->ips.c_m = 1;
		666	} else {
		667	dev_priv->ips.c_m = 2;
		668	}
		669	}
		670
		671	static const struct cxsr_latency cxsr_latency_table[] = {
		672	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		673	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		674	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		675	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		676	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		677
		678	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		679	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		680	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		681	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		682	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		683
		684	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		685	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		686	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		687	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		688	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		689
		690	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		691	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		692	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		693	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		694	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		695
		696	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		697	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		698	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		699	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		700	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		701
		702	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		703	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		704	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		705	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		706	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		707	};
		708
		709	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		710	int is_ddr3,
		711	int fsb,
		712	int mem)
		713	{
		714	const struct cxsr_latency *latency;
		715	int i;
		716
		717	if (fsb == 0 \|\| mem == 0)
		718	return NULL;
		719
		720	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		721	latency = &cxsr_latency_table[i];
		722	if (is_desktop == latency->is_desktop &&
		723	is_ddr3 == latency->is_ddr3 &&
		724	fsb == latency->fsb_freq && mem == latency->mem_freq)
		725	return latency;
		726	}
		727
		728	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		729
		730	return NULL;
		731	}
		732
		733	static void pineview_disable_cxsr(struct drm_device *dev)
		734	{
		735	struct drm_i915_private *dev_priv = dev->dev_private;
		736
		737	/* deactivate cxsr */
		738	I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
		739	}
		740
		741	/*
		742	* Latency for FIFO fetches is dependent on several factors:
		743	* - memory configuration (speed, channels)
		744	* - chipset
		745	* - current MCH state
		746	* It can be fairly high in some situations, so here we assume a fairly
		747	* pessimal value. It's a tradeoff between extra memory fetches (if we
		748	* set this value too high, the FIFO will fetch frequently to stay full)
		749	* and power consumption (set it too low to save power and we might see
		750	* FIFO underruns and display "flicker").
		751	*
		752	* A value of 5us seems to be a good balance; safe for very low end
		753	* platforms but not overly aggressive on lower latency configs.
		754	*/
		755	static const int latency_ns = 5000;
		756
		757	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		758	{
		759	struct drm_i915_private *dev_priv = dev->dev_private;
		760	uint32_t dsparb = I915_READ(DSPARB);
		761	int size;
		762
		763	size = dsparb & 0x7f;
		764	if (plane)
		765	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		766
		767	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		768	plane ? "B" : "A", size);
		769
		770	return size;
		771	}
		772
		773	static int i85x_get_fifo_size(struct drm_device *dev, int plane)
		774	{
		775	struct drm_i915_private *dev_priv = dev->dev_private;
		776	uint32_t dsparb = I915_READ(DSPARB);
		777	int size;
		778
		779	size = dsparb & 0x1ff;
		780	if (plane)
		781	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		782	size >>= 1; /* Convert to cachelines */
		783
		784	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		785	plane ? "B" : "A", size);
		786
		787	return size;
		788	}
		789
		790	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		791	{
		792	struct drm_i915_private *dev_priv = dev->dev_private;
		793	uint32_t dsparb = I915_READ(DSPARB);
		794	int size;
		795
		796	size = dsparb & 0x7f;
		797	size >>= 2; /* Convert to cachelines */
		798
		799	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		800	plane ? "B" : "A",
		801	size);
		802
		803	return size;
		804	}
		805
		806	static int i830_get_fifo_size(struct drm_device *dev, int plane)
		807	{
		808	struct drm_i915_private *dev_priv = dev->dev_private;
		809	uint32_t dsparb = I915_READ(DSPARB);
		810	int size;
		811
		812	size = dsparb & 0x7f;
		813	size >>= 1; /* Convert to cachelines */
		814
		815	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		816	plane ? "B" : "A", size);
		817
		818	return size;
		819	}
		820
		821	/* Pineview has different values for various configs */
		822	static const struct intel_watermark_params pineview_display_wm = {
		823	PINEVIEW_DISPLAY_FIFO,
		824	PINEVIEW_MAX_WM,
		825	PINEVIEW_DFT_WM,
		826	PINEVIEW_GUARD_WM,
		827	PINEVIEW_FIFO_LINE_SIZE
		828	};
		829	static const struct intel_watermark_params pineview_display_hplloff_wm = {
		830	PINEVIEW_DISPLAY_FIFO,
		831	PINEVIEW_MAX_WM,
		832	PINEVIEW_DFT_HPLLOFF_WM,
		833	PINEVIEW_GUARD_WM,
		834	PINEVIEW_FIFO_LINE_SIZE
		835	};
		836	static const struct intel_watermark_params pineview_cursor_wm = {
		837	PINEVIEW_CURSOR_FIFO,
		838	PINEVIEW_CURSOR_MAX_WM,
		839	PINEVIEW_CURSOR_DFT_WM,
		840	PINEVIEW_CURSOR_GUARD_WM,
		841	PINEVIEW_FIFO_LINE_SIZE,
		842	};
		843	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
		844	PINEVIEW_CURSOR_FIFO,
		845	PINEVIEW_CURSOR_MAX_WM,
		846	PINEVIEW_CURSOR_DFT_WM,
		847	PINEVIEW_CURSOR_GUARD_WM,
		848	PINEVIEW_FIFO_LINE_SIZE
		849	};
		850	static const struct intel_watermark_params g4x_wm_info = {
		851	G4X_FIFO_SIZE,
		852	G4X_MAX_WM,
		853	G4X_MAX_WM,
		854	2,
		855	G4X_FIFO_LINE_SIZE,
		856	};
		857	static const struct intel_watermark_params g4x_cursor_wm_info = {
		858	I965_CURSOR_FIFO,
		859	I965_CURSOR_MAX_WM,
		860	I965_CURSOR_DFT_WM,
		861	2,
		862	G4X_FIFO_LINE_SIZE,
		863	};
		864	static const struct intel_watermark_params valleyview_wm_info = {
		865	VALLEYVIEW_FIFO_SIZE,
		866	VALLEYVIEW_MAX_WM,
		867	VALLEYVIEW_MAX_WM,
		868	2,
		869	G4X_FIFO_LINE_SIZE,
		870	};
		871	static const struct intel_watermark_params valleyview_cursor_wm_info = {
		872	I965_CURSOR_FIFO,
		873	VALLEYVIEW_CURSOR_MAX_WM,
		874	I965_CURSOR_DFT_WM,
		875	2,
		876	G4X_FIFO_LINE_SIZE,
		877	};
		878	static const struct intel_watermark_params i965_cursor_wm_info = {
		879	I965_CURSOR_FIFO,
		880	I965_CURSOR_MAX_WM,
		881	I965_CURSOR_DFT_WM,
		882	2,
		883	I915_FIFO_LINE_SIZE,
		884	};
		885	static const struct intel_watermark_params i945_wm_info = {
		886	I945_FIFO_SIZE,
		887	I915_MAX_WM,
		888	1,
		889	2,
		890	I915_FIFO_LINE_SIZE
		891	};
		892	static const struct intel_watermark_params i915_wm_info = {
		893	I915_FIFO_SIZE,
		894	I915_MAX_WM,
		895	1,
		896	2,
		897	I915_FIFO_LINE_SIZE
		898	};
		899	static const struct intel_watermark_params i855_wm_info = {
		900	I855GM_FIFO_SIZE,
		901	I915_MAX_WM,
		902	1,
		903	2,
		904	I830_FIFO_LINE_SIZE
		905	};
		906	static const struct intel_watermark_params i830_wm_info = {
		907	I830_FIFO_SIZE,
		908	I915_MAX_WM,
		909	1,
		910	2,
		911	I830_FIFO_LINE_SIZE
		912	};
		913
		914	static const struct intel_watermark_params ironlake_display_wm_info = {
		915	ILK_DISPLAY_FIFO,
		916	ILK_DISPLAY_MAXWM,
		917	ILK_DISPLAY_DFTWM,
		918	2,
		919	ILK_FIFO_LINE_SIZE
		920	};
		921	static const struct intel_watermark_params ironlake_cursor_wm_info = {
		922	ILK_CURSOR_FIFO,
		923	ILK_CURSOR_MAXWM,
		924	ILK_CURSOR_DFTWM,
		925	2,
		926	ILK_FIFO_LINE_SIZE
		927	};
		928	static const struct intel_watermark_params ironlake_display_srwm_info = {
		929	ILK_DISPLAY_SR_FIFO,
		930	ILK_DISPLAY_MAX_SRWM,
		931	ILK_DISPLAY_DFT_SRWM,
		932	2,
		933	ILK_FIFO_LINE_SIZE
		934	};
		935	static const struct intel_watermark_params ironlake_cursor_srwm_info = {
		936	ILK_CURSOR_SR_FIFO,
		937	ILK_CURSOR_MAX_SRWM,
		938	ILK_CURSOR_DFT_SRWM,
		939	2,
		940	ILK_FIFO_LINE_SIZE
		941	};
		942
		943	static const struct intel_watermark_params sandybridge_display_wm_info = {
		944	SNB_DISPLAY_FIFO,
		945	SNB_DISPLAY_MAXWM,
		946	SNB_DISPLAY_DFTWM,
		947	2,
		948	SNB_FIFO_LINE_SIZE
		949	};
		950	static const struct intel_watermark_params sandybridge_cursor_wm_info = {
		951	SNB_CURSOR_FIFO,
		952	SNB_CURSOR_MAXWM,
		953	SNB_CURSOR_DFTWM,
		954	2,
		955	SNB_FIFO_LINE_SIZE
		956	};
		957	static const struct intel_watermark_params sandybridge_display_srwm_info = {
		958	SNB_DISPLAY_SR_FIFO,
		959	SNB_DISPLAY_MAX_SRWM,
		960	SNB_DISPLAY_DFT_SRWM,
		961	2,
		962	SNB_FIFO_LINE_SIZE
		963	};
		964	static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
		965	SNB_CURSOR_SR_FIFO,
		966	SNB_CURSOR_MAX_SRWM,
		967	SNB_CURSOR_DFT_SRWM,
		968	2,
		969	SNB_FIFO_LINE_SIZE
		970	};
		971
		972
		973	/**
		974	* intel_calculate_wm - calculate watermark level
		975	* @clock_in_khz: pixel clock
		976	* @wm: chip FIFO params
		977	* @pixel_size: display pixel size
		978	* @latency_ns: memory latency for the platform
		979	*
		980	* Calculate the watermark level (the level at which the display plane will
		981	* start fetching from memory again). Each chip has a different display
		982	* FIFO size and allocation, so the caller needs to figure that out and pass
		983	* in the correct intel_watermark_params structure.
		984	*
		985	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		986	* on the pixel size. When it reaches the watermark level, it'll start
		987	* fetching FIFO line sized based chunks from memory until the FIFO fills
		988	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		989	* will occur, and a display engine hang could result.
		990	*/
		991	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		992	const struct intel_watermark_params *wm,
		993	int fifo_size,
		994	int pixel_size,
		995	unsigned long latency_ns)
		996	{
		997	long entries_required, wm_size;
		998
		999	/*
		1000	* Note: we need to make sure we don't overflow for various clock &
		1001	* latency values.
		1002	* clocks go from a few thousand to several hundred thousand.
		1003	* latency is usually a few thousand
		1004	*/
		1005	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		1006	1000;
		1007	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		1008
		1009	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		1010
		1011	wm_size = fifo_size - (entries_required + wm->guard_size);
		1012
		1013	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		1014
		1015	/* Don't promote wm_size to unsigned... */
		1016	if (wm_size > (long)wm->max_wm)
		1017	wm_size = wm->max_wm;
		1018	if (wm_size <= 0)
		1019	wm_size = wm->default_wm;
		1020	return wm_size;
		1021	}
		1022
		1023	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		1024	{
		1025	struct drm_crtc crtc, enabled = NULL;
		1026
		1027	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	1028	if (intel_crtc_active(crtc)) {
3031	serge	1029	if (enabled)
		1030	return NULL;
		1031	enabled = crtc;
		1032	}
		1033	}
		1034
		1035	return enabled;
		1036	}
		1037
		1038	static void pineview_update_wm(struct drm_device *dev)
		1039	{
		1040	struct drm_i915_private *dev_priv = dev->dev_private;
		1041	struct drm_crtc *crtc;
		1042	const struct cxsr_latency *latency;
		1043	u32 reg;
		1044	unsigned long wm;
		1045
		1046	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		1047	dev_priv->fsb_freq, dev_priv->mem_freq);
		1048	if (!latency) {
		1049	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		1050	pineview_disable_cxsr(dev);
		1051	return;
		1052	}
		1053
		1054	crtc = single_enabled_crtc(dev);
		1055	if (crtc) {
		1056	int clock = crtc->mode.clock;
		1057	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1058
		1059	/* Display SR */
		1060	wm = intel_calculate_wm(clock, &pineview_display_wm,
		1061	pineview_display_wm.fifo_size,
		1062	pixel_size, latency->display_sr);
		1063	reg = I915_READ(DSPFW1);
		1064	reg &= ~DSPFW_SR_MASK;
		1065	reg \|= wm << DSPFW_SR_SHIFT;
		1066	I915_WRITE(DSPFW1, reg);
		1067	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		1068
		1069	/* cursor SR */
		1070	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		1071	pineview_display_wm.fifo_size,
		1072	pixel_size, latency->cursor_sr);
		1073	reg = I915_READ(DSPFW3);
		1074	reg &= ~DSPFW_CURSOR_SR_MASK;
		1075	reg \|= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
		1076	I915_WRITE(DSPFW3, reg);
		1077
		1078	/* Display HPLL off SR */
		1079	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		1080	pineview_display_hplloff_wm.fifo_size,
		1081	pixel_size, latency->display_hpll_disable);
		1082	reg = I915_READ(DSPFW3);
		1083	reg &= ~DSPFW_HPLL_SR_MASK;
		1084	reg \|= wm & DSPFW_HPLL_SR_MASK;
		1085	I915_WRITE(DSPFW3, reg);
		1086
		1087	/* cursor HPLL off SR */
		1088	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		1089	pineview_display_hplloff_wm.fifo_size,
		1090	pixel_size, latency->cursor_hpll_disable);
		1091	reg = I915_READ(DSPFW3);
		1092	reg &= ~DSPFW_HPLL_CURSOR_MASK;
		1093	reg \|= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
		1094	I915_WRITE(DSPFW3, reg);
		1095	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		1096
		1097	/* activate cxsr */
		1098	I915_WRITE(DSPFW3,
		1099	I915_READ(DSPFW3) \| PINEVIEW_SELF_REFRESH_EN);
		1100	DRM_DEBUG_KMS("Self-refresh is enabled\n");
		1101	} else {
		1102	pineview_disable_cxsr(dev);
		1103	DRM_DEBUG_KMS("Self-refresh is disabled\n");
		1104	}
		1105	}
		1106
		1107	static bool g4x_compute_wm0(struct drm_device *dev,
		1108	int plane,
		1109	const struct intel_watermark_params *display,
		1110	int display_latency_ns,
		1111	const struct intel_watermark_params *cursor,
		1112	int cursor_latency_ns,
		1113	int *plane_wm,
		1114	int *cursor_wm)
		1115	{
		1116	struct drm_crtc *crtc;
		1117	int htotal, hdisplay, clock, pixel_size;
		1118	int line_time_us, line_count;
		1119	int entries, tlb_miss;
		1120
		1121	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1122	if (!intel_crtc_active(crtc)) {
3031	serge	1123	*cursor_wm = cursor->guard_size;
		1124	*plane_wm = display->guard_size;
		1125	return false;
		1126	}
		1127
		1128	htotal = crtc->mode.htotal;
		1129	hdisplay = crtc->mode.hdisplay;
		1130	clock = crtc->mode.clock;
		1131	pixel_size = crtc->fb->bits_per_pixel / 8;
		1132
		1133	/* Use the small buffer method to calculate plane watermark */
		1134	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1135	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		1136	if (tlb_miss > 0)
		1137	entries += tlb_miss;
		1138	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1139	*plane_wm = entries + display->guard_size;
		1140	if (*plane_wm > (int)display->max_wm)
		1141	*plane_wm = display->max_wm;
		1142
		1143	/* Use the large buffer method to calculate cursor watermark */
		1144	line_time_us = ((htotal * 1000) / clock);
		1145	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
		1146	entries = line_count * 64 * pixel_size;
		1147	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		1148	if (tlb_miss > 0)
		1149	entries += tlb_miss;
		1150	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1151	*cursor_wm = entries + cursor->guard_size;
		1152	if (*cursor_wm > (int)cursor->max_wm)
		1153	*cursor_wm = (int)cursor->max_wm;
		1154
		1155	return true;
		1156	}
		1157
		1158	/*
		1159	* Check the wm result.
		1160	*
		1161	* If any calculated watermark values is larger than the maximum value that
		1162	* can be programmed into the associated watermark register, that watermark
		1163	* must be disabled.
		1164	*/
		1165	static bool g4x_check_srwm(struct drm_device *dev,
		1166	int display_wm, int cursor_wm,
		1167	const struct intel_watermark_params *display,
		1168	const struct intel_watermark_params *cursor)
		1169	{
		1170	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		1171	display_wm, cursor_wm);
		1172
		1173	if (display_wm > display->max_wm) {
		1174	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		1175	display_wm, display->max_wm);
		1176	return false;
		1177	}
		1178
		1179	if (cursor_wm > cursor->max_wm) {
		1180	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		1181	cursor_wm, cursor->max_wm);
		1182	return false;
		1183	}
		1184
		1185	if (!(display_wm \|\| cursor_wm)) {
		1186	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		1187	return false;
		1188	}
		1189
		1190	return true;
		1191	}
		1192
		1193	static bool g4x_compute_srwm(struct drm_device *dev,
		1194	int plane,
		1195	int latency_ns,
		1196	const struct intel_watermark_params *display,
		1197	const struct intel_watermark_params *cursor,
		1198	int display_wm, int cursor_wm)
		1199	{
		1200	struct drm_crtc *crtc;
		1201	int hdisplay, htotal, pixel_size, clock;
		1202	unsigned long line_time_us;
		1203	int line_count, line_size;
		1204	int small, large;
		1205	int entries;
		1206
		1207	if (!latency_ns) {
		1208	display_wm = cursor_wm = 0;
		1209	return false;
		1210	}
		1211
		1212	crtc = intel_get_crtc_for_plane(dev, plane);
		1213	hdisplay = crtc->mode.hdisplay;
		1214	htotal = crtc->mode.htotal;
		1215	clock = crtc->mode.clock;
		1216	pixel_size = crtc->fb->bits_per_pixel / 8;
		1217
		1218	line_time_us = (htotal * 1000) / clock;
		1219	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1220	line_size = hdisplay * pixel_size;
		1221
		1222	/* Use the minimum of the small and large buffer method for primary */
		1223	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1224	large = line_count * line_size;
		1225
		1226	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1227	*display_wm = entries + display->guard_size;
		1228
		1229	/* calculate the self-refresh watermark for display cursor */
		1230	entries = line_count * pixel_size * 64;
		1231	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1232	*cursor_wm = entries + cursor->guard_size;
		1233
		1234	return g4x_check_srwm(dev,
		1235	display_wm, cursor_wm,
		1236	display, cursor);
		1237	}
		1238
		1239	static bool vlv_compute_drain_latency(struct drm_device *dev,
		1240	int plane,
		1241	int *plane_prec_mult,
		1242	int *plane_dl,
		1243	int *cursor_prec_mult,
		1244	int *cursor_dl)
		1245	{
		1246	struct drm_crtc *crtc;
		1247	int clock, pixel_size;
		1248	int entries;
		1249
		1250	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1251	if (!intel_crtc_active(crtc))
3031	serge	1252	return false;
		1253
		1254	clock = crtc->mode.clock; /* VESA DOT Clock */
		1255	pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */
		1256
		1257	entries = (clock / 1000) * pixel_size;
		1258	*plane_prec_mult = (entries > 256) ?
		1259	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1260	plane_dl = (64 (plane_prec_mult) 4) / ((clock / 1000) *
		1261	pixel_size);
		1262
		1263	entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
		1264	*cursor_prec_mult = (entries > 256) ?
		1265	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1266	cursor_dl = (64 (cursor_prec_mult) 4) / ((clock / 1000) * 4);
		1267
		1268	return true;
		1269	}
		1270
		1271	/*
		1272	* Update drain latency registers of memory arbiter
		1273	*
		1274	* Valleyview SoC has a new memory arbiter and needs drain latency registers
		1275	* to be programmed. Each plane has a drain latency multiplier and a drain
		1276	* latency value.
		1277	*/
		1278
		1279	static void vlv_update_drain_latency(struct drm_device *dev)
		1280	{
		1281	struct drm_i915_private *dev_priv = dev->dev_private;
		1282	int planea_prec, planea_dl, planeb_prec, planeb_dl;
		1283	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
		1284	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
		1285	either 16 or 32 */
		1286
		1287	/* For plane A, Cursor A */
		1288	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
		1289	&cursor_prec_mult, &cursora_dl)) {
		1290	cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1291	DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
		1292	planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1293	DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
		1294
		1295	I915_WRITE(VLV_DDL1, cursora_prec \|
		1296	(cursora_dl << DDL_CURSORA_SHIFT) \|
		1297	planea_prec \| planea_dl);
		1298	}
		1299
		1300	/* For plane B, Cursor B */
		1301	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
		1302	&cursor_prec_mult, &cursorb_dl)) {
		1303	cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1304	DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
		1305	planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1306	DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
		1307
		1308	I915_WRITE(VLV_DDL2, cursorb_prec \|
		1309	(cursorb_dl << DDL_CURSORB_SHIFT) \|
		1310	planeb_prec \| planeb_dl);
		1311	}
		1312	}
		1313
		1314	#define single_plane_enabled(mask) is_power_of_2(mask)
		1315
		1316	static void valleyview_update_wm(struct drm_device *dev)
		1317	{
		1318	static const int sr_latency_ns = 12000;
		1319	struct drm_i915_private *dev_priv = dev->dev_private;
		1320	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1321	int plane_sr, cursor_sr;
3243	Serge	1322	int ignore_plane_sr, ignore_cursor_sr;
3031	serge	1323	unsigned int enabled = 0;
		1324
		1325	vlv_update_drain_latency(dev);
		1326
		1327	if (g4x_compute_wm0(dev, 0,
		1328	&valleyview_wm_info, latency_ns,
		1329	&valleyview_cursor_wm_info, latency_ns,
		1330	&planea_wm, &cursora_wm))
		1331	enabled \|= 1;
		1332
		1333	if (g4x_compute_wm0(dev, 1,
		1334	&valleyview_wm_info, latency_ns,
		1335	&valleyview_cursor_wm_info, latency_ns,
		1336	&planeb_wm, &cursorb_wm))
		1337	enabled \|= 2;
		1338
		1339	if (single_plane_enabled(enabled) &&
		1340	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1341	sr_latency_ns,
		1342	&valleyview_wm_info,
		1343	&valleyview_cursor_wm_info,
3243	Serge	1344	&plane_sr, &ignore_cursor_sr) &&
		1345	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1346	2*sr_latency_ns,
		1347	&valleyview_wm_info,
		1348	&valleyview_cursor_wm_info,
		1349	&ignore_plane_sr, &cursor_sr)) {
3031	serge	1350	I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
3243	Serge	1351	} else {
3031	serge	1352	I915_WRITE(FW_BLC_SELF_VLV,
		1353	I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
3243	Serge	1354	plane_sr = cursor_sr = 0;
		1355	}
3031	serge	1356
		1357	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1358	planea_wm, cursora_wm,
		1359	planeb_wm, cursorb_wm,
		1360	plane_sr, cursor_sr);
		1361
		1362	I915_WRITE(DSPFW1,
		1363	(plane_sr << DSPFW_SR_SHIFT) \|
		1364	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1365	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1366	planea_wm);
		1367	I915_WRITE(DSPFW2,
3243	Serge	1368	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1369	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1370	I915_WRITE(DSPFW3,
3243	Serge	1371	(I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) \|
		1372	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
3031	serge	1373	}
		1374
		1375	static void g4x_update_wm(struct drm_device *dev)
		1376	{
		1377	static const int sr_latency_ns = 12000;
		1378	struct drm_i915_private *dev_priv = dev->dev_private;
		1379	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1380	int plane_sr, cursor_sr;
		1381	unsigned int enabled = 0;
		1382
		1383	if (g4x_compute_wm0(dev, 0,
		1384	&g4x_wm_info, latency_ns,
		1385	&g4x_cursor_wm_info, latency_ns,
		1386	&planea_wm, &cursora_wm))
		1387	enabled \|= 1;
		1388
		1389	if (g4x_compute_wm0(dev, 1,
		1390	&g4x_wm_info, latency_ns,
		1391	&g4x_cursor_wm_info, latency_ns,
		1392	&planeb_wm, &cursorb_wm))
		1393	enabled \|= 2;
		1394
		1395	if (single_plane_enabled(enabled) &&
		1396	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1397	sr_latency_ns,
		1398	&g4x_wm_info,
		1399	&g4x_cursor_wm_info,
3243	Serge	1400	&plane_sr, &cursor_sr)) {
3031	serge	1401	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
3243	Serge	1402	} else {
3031	serge	1403	I915_WRITE(FW_BLC_SELF,
		1404	I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
3243	Serge	1405	plane_sr = cursor_sr = 0;
		1406	}
3031	serge	1407
		1408	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1409	planea_wm, cursora_wm,
		1410	planeb_wm, cursorb_wm,
		1411	plane_sr, cursor_sr);
		1412
		1413	I915_WRITE(DSPFW1,
		1414	(plane_sr << DSPFW_SR_SHIFT) \|
		1415	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1416	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1417	planea_wm);
		1418	I915_WRITE(DSPFW2,
3243	Serge	1419	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1420	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1421	/* HPLL off in SR has some issues on G4x... disable it */
		1422	I915_WRITE(DSPFW3,
3243	Serge	1423	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
3031	serge	1424	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1425	}
		1426
		1427	static void i965_update_wm(struct drm_device *dev)
		1428	{
		1429	struct drm_i915_private *dev_priv = dev->dev_private;
		1430	struct drm_crtc *crtc;
		1431	int srwm = 1;
		1432	int cursor_sr = 16;
		1433
		1434	/* Calc sr entries for one plane configs */
		1435	crtc = single_enabled_crtc(dev);
		1436	if (crtc) {
		1437	/* self-refresh has much higher latency */
		1438	static const int sr_latency_ns = 12000;
		1439	int clock = crtc->mode.clock;
		1440	int htotal = crtc->mode.htotal;
		1441	int hdisplay = crtc->mode.hdisplay;
		1442	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1443	unsigned long line_time_us;
		1444	int entries;
		1445
		1446	line_time_us = ((htotal * 1000) / clock);
		1447
		1448	/* Use ns/us then divide to preserve precision */
		1449	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1450	pixel_size * hdisplay;
		1451	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1452	srwm = I965_FIFO_SIZE - entries;
		1453	if (srwm < 0)
		1454	srwm = 1;
		1455	srwm &= 0x1ff;
		1456	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1457	entries, srwm);
		1458
		1459	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1460	pixel_size * 64;
		1461	entries = DIV_ROUND_UP(entries,
		1462	i965_cursor_wm_info.cacheline_size);
		1463	cursor_sr = i965_cursor_wm_info.fifo_size -
		1464	(entries + i965_cursor_wm_info.guard_size);
		1465
		1466	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1467	cursor_sr = i965_cursor_wm_info.max_wm;
		1468
		1469	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1470	"cursor %d\n", srwm, cursor_sr);
		1471
		1472	if (IS_CRESTLINE(dev))
		1473	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
		1474	} else {
		1475	/* Turn off self refresh if both pipes are enabled */
		1476	if (IS_CRESTLINE(dev))
		1477	I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
		1478	& ~FW_BLC_SELF_EN);
		1479	}
		1480
		1481	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1482	srwm);
		1483
		1484	/* 965 has limitations... */
		1485	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) \|
		1486	(8 << 16) \| (8 << 8) \| (8 << 0));
		1487	I915_WRITE(DSPFW2, (8 << 8) \| (8 << 0));
		1488	/* update cursor SR watermark */
		1489	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1490	}
		1491
		1492	static void i9xx_update_wm(struct drm_device *dev)
		1493	{
		1494	struct drm_i915_private *dev_priv = dev->dev_private;
		1495	const struct intel_watermark_params *wm_info;
		1496	uint32_t fwater_lo;
		1497	uint32_t fwater_hi;
		1498	int cwm, srwm = 1;
		1499	int fifo_size;
		1500	int planea_wm, planeb_wm;
		1501	struct drm_crtc crtc, enabled = NULL;
		1502
		1503	if (IS_I945GM(dev))
		1504	wm_info = &i945_wm_info;
		1505	else if (!IS_GEN2(dev))
		1506	wm_info = &i915_wm_info;
		1507	else
		1508	wm_info = &i855_wm_info;
		1509
		1510	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1511	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1512	if (intel_crtc_active(crtc)) {
		1513	int cpp = crtc->fb->bits_per_pixel / 8;
		1514	if (IS_GEN2(dev))
		1515	cpp = 4;
		1516
3031	serge	1517	planea_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1518	wm_info, fifo_size, cpp,
3031	serge	1519	latency_ns);
		1520	enabled = crtc;
		1521	} else
		1522	planea_wm = fifo_size - wm_info->guard_size;
		1523
		1524	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1525	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1526	if (intel_crtc_active(crtc)) {
		1527	int cpp = crtc->fb->bits_per_pixel / 8;
		1528	if (IS_GEN2(dev))
		1529	cpp = 4;
		1530
3031	serge	1531	planeb_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1532	wm_info, fifo_size, cpp,
3031	serge	1533	latency_ns);
		1534	if (enabled == NULL)
		1535	enabled = crtc;
		1536	else
		1537	enabled = NULL;
		1538	} else
		1539	planeb_wm = fifo_size - wm_info->guard_size;
		1540
		1541	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1542
		1543	/*
		1544	* Overlay gets an aggressive default since video jitter is bad.
		1545	*/
		1546	cwm = 2;
		1547
		1548	/* Play safe and disable self-refresh before adjusting watermarks. */
		1549	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1550	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK \| 0);
		1551	else if (IS_I915GM(dev))
		1552	I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
		1553
		1554	/* Calc sr entries for one plane configs */
		1555	if (HAS_FW_BLC(dev) && enabled) {
		1556	/* self-refresh has much higher latency */
		1557	static const int sr_latency_ns = 6000;
		1558	int clock = enabled->mode.clock;
		1559	int htotal = enabled->mode.htotal;
		1560	int hdisplay = enabled->mode.hdisplay;
		1561	int pixel_size = enabled->fb->bits_per_pixel / 8;
		1562	unsigned long line_time_us;
		1563	int entries;
		1564
		1565	line_time_us = (htotal * 1000) / clock;
		1566
		1567	/* Use ns/us then divide to preserve precision */
		1568	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1569	pixel_size * hdisplay;
		1570	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1571	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1572	srwm = wm_info->fifo_size - entries;
		1573	if (srwm < 0)
		1574	srwm = 1;
		1575
		1576	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1577	I915_WRITE(FW_BLC_SELF,
		1578	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1579	else if (IS_I915GM(dev))
		1580	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1581	}
		1582
		1583	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1584	planea_wm, planeb_wm, cwm, srwm);
		1585
		1586	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1587	fwater_hi = (cwm & 0x1f);
		1588
		1589	/* Set request length to 8 cachelines per fetch */
		1590	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1591	fwater_hi = fwater_hi \| (1 << 8);
		1592
		1593	I915_WRITE(FW_BLC, fwater_lo);
		1594	I915_WRITE(FW_BLC2, fwater_hi);
		1595
		1596	if (HAS_FW_BLC(dev)) {
		1597	if (enabled) {
		1598	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1599	I915_WRITE(FW_BLC_SELF,
		1600	FW_BLC_SELF_EN_MASK \| FW_BLC_SELF_EN);
		1601	else if (IS_I915GM(dev))
		1602	I915_WRITE(INSTPM, I915_READ(INSTPM) \| INSTPM_SELF_EN);
		1603	DRM_DEBUG_KMS("memory self refresh enabled\n");
		1604	} else
		1605	DRM_DEBUG_KMS("memory self refresh disabled\n");
		1606	}
		1607	}
		1608
		1609	static void i830_update_wm(struct drm_device *dev)
		1610	{
		1611	struct drm_i915_private *dev_priv = dev->dev_private;
		1612	struct drm_crtc *crtc;
		1613	uint32_t fwater_lo;
		1614	int planea_wm;
		1615
		1616	crtc = single_enabled_crtc(dev);
		1617	if (crtc == NULL)
		1618	return;
		1619
		1620	planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
		1621	dev_priv->display.get_fifo_size(dev, 0),
3243	Serge	1622	4, latency_ns);
3031	serge	1623	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1624	fwater_lo \|= (3<<8) \| planea_wm;
		1625
		1626	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1627
		1628	I915_WRITE(FW_BLC, fwater_lo);
		1629	}
		1630
		1631	#define ILK_LP0_PLANE_LATENCY 700
		1632	#define ILK_LP0_CURSOR_LATENCY 1300
		1633
		1634	/*
		1635	* Check the wm result.
		1636	*
		1637	* If any calculated watermark values is larger than the maximum value that
		1638	* can be programmed into the associated watermark register, that watermark
		1639	* must be disabled.
		1640	*/
		1641	static bool ironlake_check_srwm(struct drm_device *dev, int level,
		1642	int fbc_wm, int display_wm, int cursor_wm,
		1643	const struct intel_watermark_params *display,
		1644	const struct intel_watermark_params *cursor)
		1645	{
		1646	struct drm_i915_private *dev_priv = dev->dev_private;
		1647
		1648	DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
		1649	" cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
		1650
		1651	if (fbc_wm > SNB_FBC_MAX_SRWM) {
		1652	DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
		1653	fbc_wm, SNB_FBC_MAX_SRWM, level);
		1654
		1655	/* fbc has it's own way to disable FBC WM */
		1656	I915_WRITE(DISP_ARB_CTL,
		1657	I915_READ(DISP_ARB_CTL) \| DISP_FBC_WM_DIS);
		1658	return false;
		1659	}
		1660
		1661	if (display_wm > display->max_wm) {
		1662	DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
		1663	display_wm, SNB_DISPLAY_MAX_SRWM, level);
		1664	return false;
		1665	}
		1666
		1667	if (cursor_wm > cursor->max_wm) {
		1668	DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
		1669	cursor_wm, SNB_CURSOR_MAX_SRWM, level);
		1670	return false;
		1671	}
		1672
		1673	if (!(fbc_wm \|\| display_wm \|\| cursor_wm)) {
		1674	DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
		1675	return false;
		1676	}
		1677
		1678	return true;
		1679	}
		1680
		1681	/*
		1682	* Compute watermark values of WM[1-3],
		1683	*/
		1684	static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
		1685	int latency_ns,
		1686	const struct intel_watermark_params *display,
		1687	const struct intel_watermark_params *cursor,
		1688	int fbc_wm, int display_wm, int *cursor_wm)
		1689	{
		1690	struct drm_crtc *crtc;
		1691	unsigned long line_time_us;
		1692	int hdisplay, htotal, pixel_size, clock;
		1693	int line_count, line_size;
		1694	int small, large;
		1695	int entries;
		1696
		1697	if (!latency_ns) {
		1698	fbc_wm = display_wm = *cursor_wm = 0;
		1699	return false;
		1700	}
		1701
		1702	crtc = intel_get_crtc_for_plane(dev, plane);
		1703	hdisplay = crtc->mode.hdisplay;
		1704	htotal = crtc->mode.htotal;
		1705	clock = crtc->mode.clock;
		1706	pixel_size = crtc->fb->bits_per_pixel / 8;
		1707
		1708	line_time_us = (htotal * 1000) / clock;
		1709	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1710	line_size = hdisplay * pixel_size;
		1711
		1712	/* Use the minimum of the small and large buffer method for primary */
		1713	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1714	large = line_count * line_size;
		1715
		1716	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1717	*display_wm = entries + display->guard_size;
		1718
		1719	/*
		1720	* Spec says:
		1721	* FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
		1722	*/
		1723	fbc_wm = DIV_ROUND_UP(display_wm * 64, line_size) + 2;
		1724
		1725	/* calculate the self-refresh watermark for display cursor */
		1726	entries = line_count * pixel_size * 64;
		1727	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1728	*cursor_wm = entries + cursor->guard_size;
		1729
		1730	return ironlake_check_srwm(dev, level,
		1731	fbc_wm, display_wm, *cursor_wm,
		1732	display, cursor);
		1733	}
		1734
		1735	static void ironlake_update_wm(struct drm_device *dev)
		1736	{
		1737	struct drm_i915_private *dev_priv = dev->dev_private;
		1738	int fbc_wm, plane_wm, cursor_wm;
		1739	unsigned int enabled;
		1740
		1741	enabled = 0;
		1742	if (g4x_compute_wm0(dev, 0,
		1743	&ironlake_display_wm_info,
		1744	ILK_LP0_PLANE_LATENCY,
		1745	&ironlake_cursor_wm_info,
		1746	ILK_LP0_CURSOR_LATENCY,
		1747	&plane_wm, &cursor_wm)) {
		1748	I915_WRITE(WM0_PIPEA_ILK,
		1749	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1750	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1751	" plane %d, " "cursor: %d\n",
		1752	plane_wm, cursor_wm);
		1753	enabled \|= 1;
		1754	}
		1755
		1756	if (g4x_compute_wm0(dev, 1,
		1757	&ironlake_display_wm_info,
		1758	ILK_LP0_PLANE_LATENCY,
		1759	&ironlake_cursor_wm_info,
		1760	ILK_LP0_CURSOR_LATENCY,
		1761	&plane_wm, &cursor_wm)) {
		1762	I915_WRITE(WM0_PIPEB_ILK,
		1763	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1764	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1765	" plane %d, cursor: %d\n",
		1766	plane_wm, cursor_wm);
		1767	enabled \|= 2;
		1768	}
		1769
		1770	/*
		1771	* Calculate and update the self-refresh watermark only when one
		1772	* display plane is used.
		1773	*/
		1774	I915_WRITE(WM3_LP_ILK, 0);
		1775	I915_WRITE(WM2_LP_ILK, 0);
		1776	I915_WRITE(WM1_LP_ILK, 0);
		1777
		1778	if (!single_plane_enabled(enabled))
		1779	return;
		1780	enabled = ffs(enabled) - 1;
		1781
		1782	/* WM1 */
		1783	if (!ironlake_compute_srwm(dev, 1, enabled,
		1784	ILK_READ_WM1_LATENCY() * 500,
		1785	&ironlake_display_srwm_info,
		1786	&ironlake_cursor_srwm_info,
		1787	&fbc_wm, &plane_wm, &cursor_wm))
		1788	return;
		1789
		1790	I915_WRITE(WM1_LP_ILK,
		1791	WM1_LP_SR_EN \|
		1792	(ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1793	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1794	(plane_wm << WM1_LP_SR_SHIFT) \|
		1795	cursor_wm);
		1796
		1797	/* WM2 */
		1798	if (!ironlake_compute_srwm(dev, 2, enabled,
		1799	ILK_READ_WM2_LATENCY() * 500,
		1800	&ironlake_display_srwm_info,
		1801	&ironlake_cursor_srwm_info,
		1802	&fbc_wm, &plane_wm, &cursor_wm))
		1803	return;
		1804
		1805	I915_WRITE(WM2_LP_ILK,
		1806	WM2_LP_EN \|
		1807	(ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1808	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1809	(plane_wm << WM1_LP_SR_SHIFT) \|
		1810	cursor_wm);
		1811
		1812	/*
		1813	* WM3 is unsupported on ILK, probably because we don't have latency
		1814	* data for that power state
		1815	*/
		1816	}
		1817
		1818	static void sandybridge_update_wm(struct drm_device *dev)
		1819	{
		1820	struct drm_i915_private *dev_priv = dev->dev_private;
		1821	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		1822	u32 val;
		1823	int fbc_wm, plane_wm, cursor_wm;
		1824	unsigned int enabled;
		1825
		1826	enabled = 0;
		1827	if (g4x_compute_wm0(dev, 0,
		1828	&sandybridge_display_wm_info, latency,
		1829	&sandybridge_cursor_wm_info, latency,
		1830	&plane_wm, &cursor_wm)) {
		1831	val = I915_READ(WM0_PIPEA_ILK);
		1832	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1833	I915_WRITE(WM0_PIPEA_ILK, val \|
		1834	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1835	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1836	" plane %d, " "cursor: %d\n",
		1837	plane_wm, cursor_wm);
		1838	enabled \|= 1;
		1839	}
		1840
		1841	if (g4x_compute_wm0(dev, 1,
		1842	&sandybridge_display_wm_info, latency,
		1843	&sandybridge_cursor_wm_info, latency,
		1844	&plane_wm, &cursor_wm)) {
		1845	val = I915_READ(WM0_PIPEB_ILK);
		1846	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1847	I915_WRITE(WM0_PIPEB_ILK, val \|
		1848	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1849	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1850	" plane %d, cursor: %d\n",
		1851	plane_wm, cursor_wm);
		1852	enabled \|= 2;
		1853	}
		1854
3243	Serge	1855	/*
		1856	* Calculate and update the self-refresh watermark only when one
		1857	* display plane is used.
		1858	*
		1859	* SNB support 3 levels of watermark.
		1860	*
		1861	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1862	* and disabled in the descending order
		1863	*
		1864	*/
		1865	I915_WRITE(WM3_LP_ILK, 0);
		1866	I915_WRITE(WM2_LP_ILK, 0);
		1867	I915_WRITE(WM1_LP_ILK, 0);
		1868
		1869	if (!single_plane_enabled(enabled) \|\|
		1870	dev_priv->sprite_scaling_enabled)
		1871	return;
		1872	enabled = ffs(enabled) - 1;
		1873
		1874	/* WM1 */
		1875	if (!ironlake_compute_srwm(dev, 1, enabled,
		1876	SNB_READ_WM1_LATENCY() * 500,
		1877	&sandybridge_display_srwm_info,
		1878	&sandybridge_cursor_srwm_info,
		1879	&fbc_wm, &plane_wm, &cursor_wm))
		1880	return;
		1881
		1882	I915_WRITE(WM1_LP_ILK,
		1883	WM1_LP_SR_EN \|
		1884	(SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1885	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1886	(plane_wm << WM1_LP_SR_SHIFT) \|
		1887	cursor_wm);
		1888
		1889	/* WM2 */
		1890	if (!ironlake_compute_srwm(dev, 2, enabled,
		1891	SNB_READ_WM2_LATENCY() * 500,
		1892	&sandybridge_display_srwm_info,
		1893	&sandybridge_cursor_srwm_info,
		1894	&fbc_wm, &plane_wm, &cursor_wm))
		1895	return;
		1896
		1897	I915_WRITE(WM2_LP_ILK,
		1898	WM2_LP_EN \|
		1899	(SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1900	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1901	(plane_wm << WM1_LP_SR_SHIFT) \|
		1902	cursor_wm);
		1903
		1904	/* WM3 */
		1905	if (!ironlake_compute_srwm(dev, 3, enabled,
		1906	SNB_READ_WM3_LATENCY() * 500,
		1907	&sandybridge_display_srwm_info,
		1908	&sandybridge_cursor_srwm_info,
		1909	&fbc_wm, &plane_wm, &cursor_wm))
		1910	return;
		1911
		1912	I915_WRITE(WM3_LP_ILK,
		1913	WM3_LP_EN \|
		1914	(SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1915	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1916	(plane_wm << WM1_LP_SR_SHIFT) \|
		1917	cursor_wm);
		1918	}
		1919
		1920	static void ivybridge_update_wm(struct drm_device *dev)
		1921	{
		1922	struct drm_i915_private *dev_priv = dev->dev_private;
		1923	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		1924	u32 val;
		1925	int fbc_wm, plane_wm, cursor_wm;
		1926	int ignore_fbc_wm, ignore_plane_wm, ignore_cursor_wm;
		1927	unsigned int enabled;
		1928
		1929	enabled = 0;
		1930	if (g4x_compute_wm0(dev, 0,
3031	serge	1931	&sandybridge_display_wm_info, latency,
		1932	&sandybridge_cursor_wm_info, latency,
		1933	&plane_wm, &cursor_wm)) {
3243	Serge	1934	val = I915_READ(WM0_PIPEA_ILK);
		1935	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1936	I915_WRITE(WM0_PIPEA_ILK, val \|
		1937	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1938	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1939	" plane %d, " "cursor: %d\n",
		1940	plane_wm, cursor_wm);
		1941	enabled \|= 1;
		1942	}
		1943
		1944	if (g4x_compute_wm0(dev, 1,
		1945	&sandybridge_display_wm_info, latency,
		1946	&sandybridge_cursor_wm_info, latency,
		1947	&plane_wm, &cursor_wm)) {
		1948	val = I915_READ(WM0_PIPEB_ILK);
		1949	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1950	I915_WRITE(WM0_PIPEB_ILK, val \|
		1951	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1952	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1953	" plane %d, cursor: %d\n",
		1954	plane_wm, cursor_wm);
		1955	enabled \|= 2;
		1956	}
		1957
		1958	if (g4x_compute_wm0(dev, 2,
		1959	&sandybridge_display_wm_info, latency,
		1960	&sandybridge_cursor_wm_info, latency,
		1961	&plane_wm, &cursor_wm)) {
3031	serge	1962	val = I915_READ(WM0_PIPEC_IVB);
		1963	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1964	I915_WRITE(WM0_PIPEC_IVB, val \|
		1965	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1966	DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
		1967	" plane %d, cursor: %d\n",
		1968	plane_wm, cursor_wm);
		1969	enabled \|= 3;
		1970	}
		1971
		1972	/*
		1973	* Calculate and update the self-refresh watermark only when one
		1974	* display plane is used.
		1975	*
		1976	* SNB support 3 levels of watermark.
		1977	*
		1978	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1979	* and disabled in the descending order
		1980	*
		1981	*/
		1982	I915_WRITE(WM3_LP_ILK, 0);
		1983	I915_WRITE(WM2_LP_ILK, 0);
		1984	I915_WRITE(WM1_LP_ILK, 0);
		1985
		1986	if (!single_plane_enabled(enabled) \|\|
		1987	dev_priv->sprite_scaling_enabled)
		1988	return;
		1989	enabled = ffs(enabled) - 1;
		1990
		1991	/* WM1 */
		1992	if (!ironlake_compute_srwm(dev, 1, enabled,
		1993	SNB_READ_WM1_LATENCY() * 500,
		1994	&sandybridge_display_srwm_info,
		1995	&sandybridge_cursor_srwm_info,
		1996	&fbc_wm, &plane_wm, &cursor_wm))
		1997	return;
		1998
		1999	I915_WRITE(WM1_LP_ILK,
		2000	WM1_LP_SR_EN \|
		2001	(SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2002	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2003	(plane_wm << WM1_LP_SR_SHIFT) \|
		2004	cursor_wm);
		2005
		2006	/* WM2 */
		2007	if (!ironlake_compute_srwm(dev, 2, enabled,
		2008	SNB_READ_WM2_LATENCY() * 500,
		2009	&sandybridge_display_srwm_info,
		2010	&sandybridge_cursor_srwm_info,
		2011	&fbc_wm, &plane_wm, &cursor_wm))
		2012	return;
		2013
		2014	I915_WRITE(WM2_LP_ILK,
		2015	WM2_LP_EN \|
		2016	(SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2017	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2018	(plane_wm << WM1_LP_SR_SHIFT) \|
		2019	cursor_wm);
		2020
3243	Serge	2021	/* WM3, note we have to correct the cursor latency */
3031	serge	2022	if (!ironlake_compute_srwm(dev, 3, enabled,
		2023	SNB_READ_WM3_LATENCY() * 500,
		2024	&sandybridge_display_srwm_info,
		2025	&sandybridge_cursor_srwm_info,
3243	Serge	2026	&fbc_wm, &plane_wm, &ignore_cursor_wm) \|\|
		2027	!ironlake_compute_srwm(dev, 3, enabled,
		2028	2 * SNB_READ_WM3_LATENCY() * 500,
		2029	&sandybridge_display_srwm_info,
		2030	&sandybridge_cursor_srwm_info,
		2031	&ignore_fbc_wm, &ignore_plane_wm, &cursor_wm))
3031	serge	2032	return;
		2033
		2034	I915_WRITE(WM3_LP_ILK,
		2035	WM3_LP_EN \|
		2036	(SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2037	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2038	(plane_wm << WM1_LP_SR_SHIFT) \|
		2039	cursor_wm);
		2040	}
		2041
		2042	static void
		2043	haswell_update_linetime_wm(struct drm_device *dev, int pipe,
		2044	struct drm_display_mode *mode)
		2045	{
		2046	struct drm_i915_private *dev_priv = dev->dev_private;
		2047	u32 temp;
		2048
		2049	temp = I915_READ(PIPE_WM_LINETIME(pipe));
		2050	temp &= ~PIPE_WM_LINETIME_MASK;
		2051
		2052	/* The WM are computed with base on how long it takes to fill a single
		2053	* row at the given clock rate, multiplied by 8.
		2054	* */
		2055	temp \|= PIPE_WM_LINETIME_TIME(
		2056	((mode->crtc_hdisplay * 1000) / mode->clock) * 8);
		2057
		2058	/* IPS watermarks are only used by pipe A, and are ignored by
		2059	* pipes B and C. They are calculated similarly to the common
		2060	* linetime values, except that we are using CD clock frequency
		2061	* in MHz instead of pixel rate for the division.
		2062	*
		2063	* This is a placeholder for the IPS watermark calculation code.
		2064	*/
		2065
		2066	I915_WRITE(PIPE_WM_LINETIME(pipe), temp);
		2067	}
		2068
		2069	static bool
		2070	sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
		2071	uint32_t sprite_width, int pixel_size,
		2072	const struct intel_watermark_params *display,
		2073	int display_latency_ns, int *sprite_wm)
		2074	{
		2075	struct drm_crtc *crtc;
		2076	int clock;
		2077	int entries, tlb_miss;
		2078
		2079	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	2080	if (!intel_crtc_active(crtc)) {
3031	serge	2081	*sprite_wm = display->guard_size;
		2082	return false;
		2083	}
		2084
		2085	clock = crtc->mode.clock;
		2086
		2087	/* Use the small buffer method to calculate the sprite watermark */
		2088	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		2089	tlb_miss = display->fifo_size*display->cacheline_size -
		2090	sprite_width * 8;
		2091	if (tlb_miss > 0)
		2092	entries += tlb_miss;
		2093	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		2094	*sprite_wm = entries + display->guard_size;
		2095	if (*sprite_wm > (int)display->max_wm)
		2096	*sprite_wm = display->max_wm;
		2097
		2098	return true;
		2099	}
		2100
		2101	static bool
		2102	sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
		2103	uint32_t sprite_width, int pixel_size,
		2104	const struct intel_watermark_params *display,
		2105	int latency_ns, int *sprite_wm)
		2106	{
		2107	struct drm_crtc *crtc;
		2108	unsigned long line_time_us;
		2109	int clock;
		2110	int line_count, line_size;
		2111	int small, large;
		2112	int entries;
		2113
		2114	if (!latency_ns) {
		2115	*sprite_wm = 0;
		2116	return false;
		2117	}
		2118
		2119	crtc = intel_get_crtc_for_plane(dev, plane);
		2120	clock = crtc->mode.clock;
		2121	if (!clock) {
		2122	*sprite_wm = 0;
		2123	return false;
		2124	}
		2125
		2126	line_time_us = (sprite_width * 1000) / clock;
		2127	if (!line_time_us) {
		2128	*sprite_wm = 0;
		2129	return false;
		2130	}
		2131
		2132	line_count = (latency_ns / line_time_us + 1000) / 1000;
		2133	line_size = sprite_width * pixel_size;
		2134
		2135	/* Use the minimum of the small and large buffer method for primary */
		2136	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		2137	large = line_count * line_size;
		2138
		2139	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		2140	*sprite_wm = entries + display->guard_size;
		2141
		2142	return *sprite_wm > 0x3ff ? false : true;
		2143	}
		2144
		2145	static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
		2146	uint32_t sprite_width, int pixel_size)
		2147	{
		2148	struct drm_i915_private *dev_priv = dev->dev_private;
		2149	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		2150	u32 val;
		2151	int sprite_wm, reg;
		2152	int ret;
		2153
		2154	switch (pipe) {
		2155	case 0:
		2156	reg = WM0_PIPEA_ILK;
		2157	break;
		2158	case 1:
		2159	reg = WM0_PIPEB_ILK;
		2160	break;
		2161	case 2:
		2162	reg = WM0_PIPEC_IVB;
		2163	break;
		2164	default:
		2165	return; /* bad pipe */
		2166	}
		2167
		2168	ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
		2169	&sandybridge_display_wm_info,
		2170	latency, &sprite_wm);
		2171	if (!ret) {
		2172	DRM_DEBUG_KMS("failed to compute sprite wm for pipe %d\n",
		2173	pipe);
		2174	return;
		2175	}
		2176
		2177	val = I915_READ(reg);
		2178	val &= ~WM0_PIPE_SPRITE_MASK;
		2179	I915_WRITE(reg, val \| (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
		2180	DRM_DEBUG_KMS("sprite watermarks For pipe %d - %d\n", pipe, sprite_wm);
		2181
		2182
		2183	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2184	pixel_size,
		2185	&sandybridge_display_srwm_info,
		2186	SNB_READ_WM1_LATENCY() * 500,
		2187	&sprite_wm);
		2188	if (!ret) {
		2189	DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %d\n",
		2190	pipe);
		2191	return;
		2192	}
		2193	I915_WRITE(WM1S_LP_ILK, sprite_wm);
		2194
		2195	/* Only IVB has two more LP watermarks for sprite */
		2196	if (!IS_IVYBRIDGE(dev))
		2197	return;
		2198
		2199	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2200	pixel_size,
		2201	&sandybridge_display_srwm_info,
		2202	SNB_READ_WM2_LATENCY() * 500,
		2203	&sprite_wm);
		2204	if (!ret) {
		2205	DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %d\n",
		2206	pipe);
		2207	return;
		2208	}
		2209	I915_WRITE(WM2S_LP_IVB, sprite_wm);
		2210
		2211	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2212	pixel_size,
		2213	&sandybridge_display_srwm_info,
		2214	SNB_READ_WM3_LATENCY() * 500,
		2215	&sprite_wm);
		2216	if (!ret) {
		2217	DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %d\n",
		2218	pipe);
		2219	return;
		2220	}
		2221	I915_WRITE(WM3S_LP_IVB, sprite_wm);
		2222	}
		2223
		2224	/**
		2225	* intel_update_watermarks - update FIFO watermark values based on current modes
		2226	*
		2227	* Calculate watermark values for the various WM regs based on current mode
		2228	* and plane configuration.
		2229	*
		2230	* There are several cases to deal with here:
		2231	* - normal (i.e. non-self-refresh)
		2232	* - self-refresh (SR) mode
		2233	* - lines are large relative to FIFO size (buffer can hold up to 2)
		2234	* - lines are small relative to FIFO size (buffer can hold more than 2
		2235	* lines), so need to account for TLB latency
		2236	*
		2237	* The normal calculation is:
		2238	* watermark = dotclock * bytes per pixel * latency
		2239	* where latency is platform & configuration dependent (we assume pessimal
		2240	* values here).
		2241	*
		2242	* The SR calculation is:
		2243	* watermark = (trunc(latency/line time)+1) * surface width *
		2244	* bytes per pixel
		2245	* where
		2246	* line time = htotal / dotclock
		2247	* surface width = hdisplay for normal plane and 64 for cursor
		2248	* and latency is assumed to be high, as above.
		2249	*
		2250	* The final value programmed to the register should always be rounded up,
		2251	* and include an extra 2 entries to account for clock crossings.
		2252	*
		2253	* We don't use the sprite, so we can ignore that. And on Crestline we have
		2254	* to set the non-SR watermarks to 8.
		2255	*/
		2256	void intel_update_watermarks(struct drm_device *dev)
		2257	{
		2258	struct drm_i915_private *dev_priv = dev->dev_private;
		2259
		2260	if (dev_priv->display.update_wm)
		2261	dev_priv->display.update_wm(dev);
		2262	}
		2263
		2264	void intel_update_linetime_watermarks(struct drm_device *dev,
		2265	int pipe, struct drm_display_mode *mode)
		2266	{
		2267	struct drm_i915_private *dev_priv = dev->dev_private;
		2268
		2269	if (dev_priv->display.update_linetime_wm)
		2270	dev_priv->display.update_linetime_wm(dev, pipe, mode);
		2271	}
		2272
		2273	void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
		2274	uint32_t sprite_width, int pixel_size)
		2275	{
		2276	struct drm_i915_private *dev_priv = dev->dev_private;
		2277
		2278	if (dev_priv->display.update_sprite_wm)
		2279	dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
		2280	pixel_size);
		2281	}
		2282
		2283	static struct drm_i915_gem_object *
		2284	intel_alloc_context_page(struct drm_device *dev)
		2285	{
		2286	struct drm_i915_gem_object *ctx;
		2287	int ret;
		2288
		2289	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2290
		2291	ctx = i915_gem_alloc_object(dev, 4096);
		2292	if (!ctx) {
		2293	DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
		2294	return NULL;
		2295	}
		2296
		2297	ret = i915_gem_object_pin(ctx, 4096, true, false);
		2298	if (ret) {
		2299	DRM_ERROR("failed to pin power context: %d\n", ret);
		2300	goto err_unref;
		2301	}
		2302
		2303	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
		2304	if (ret) {
		2305	DRM_ERROR("failed to set-domain on power context: %d\n", ret);
		2306	goto err_unpin;
		2307	}
		2308
		2309	return ctx;
		2310
		2311	err_unpin:
		2312	i915_gem_object_unpin(ctx);
		2313	err_unref:
		2314	drm_gem_object_unreference(&ctx->base);
		2315	return NULL;
		2316	}
		2317
		2318	/**
		2319	* Lock protecting IPS related data structures
		2320	*/
		2321	DEFINE_SPINLOCK(mchdev_lock);
		2322
		2323	/* Global for IPS driver to get at the current i915 device. Protected by
		2324	* mchdev_lock. */
		2325	static struct drm_i915_private *i915_mch_dev;
		2326
		2327	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		2328	{
		2329	struct drm_i915_private *dev_priv = dev->dev_private;
		2330	u16 rgvswctl;
		2331
		2332	assert_spin_locked(&mchdev_lock);
		2333
		2334	rgvswctl = I915_READ16(MEMSWCTL);
		2335	if (rgvswctl & MEMCTL_CMD_STS) {
		2336	DRM_DEBUG("gpu busy, RCS change rejected\n");
		2337	return false; /* still busy with another command */
		2338	}
		2339
		2340	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		2341	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		2342	I915_WRITE16(MEMSWCTL, rgvswctl);
		2343	POSTING_READ16(MEMSWCTL);
		2344
		2345	rgvswctl \|= MEMCTL_CMD_STS;
		2346	I915_WRITE16(MEMSWCTL, rgvswctl);
		2347
		2348	return true;
		2349	}
		2350
		2351	static void ironlake_enable_drps(struct drm_device *dev)
		2352	{
		2353	struct drm_i915_private *dev_priv = dev->dev_private;
		2354	u32 rgvmodectl = I915_READ(MEMMODECTL);
		2355	u8 fmax, fmin, fstart, vstart;
		2356
		2357	spin_lock_irq(&mchdev_lock);
		2358
		2359	/* Enable temp reporting */
		2360	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		2361	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		2362
		2363	/* 100ms RC evaluation intervals */
		2364	I915_WRITE(RCUPEI, 100000);
		2365	I915_WRITE(RCDNEI, 100000);
		2366
		2367	/* Set max/min thresholds to 90ms and 80ms respectively */
		2368	I915_WRITE(RCBMAXAVG, 90000);
		2369	I915_WRITE(RCBMINAVG, 80000);
		2370
		2371	I915_WRITE(MEMIHYST, 1);
		2372
		2373	/* Set up min, max, and cur for interrupt handling */
		2374	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		2375	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		2376	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		2377	MEMMODE_FSTART_SHIFT;
		2378
		2379	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
		2380	PXVFREQ_PX_SHIFT;
		2381
		2382	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		2383	dev_priv->ips.fstart = fstart;
		2384
		2385	dev_priv->ips.max_delay = fstart;
		2386	dev_priv->ips.min_delay = fmin;
		2387	dev_priv->ips.cur_delay = fstart;
		2388
		2389	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		2390	fmax, fmin, fstart);
		2391
		2392	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		2393
		2394	/*
		2395	* Interrupts will be enabled in ironlake_irq_postinstall
		2396	*/
		2397
		2398	I915_WRITE(VIDSTART, vstart);
		2399	POSTING_READ(VIDSTART);
		2400
		2401	rgvmodectl \|= MEMMODE_SWMODE_EN;
		2402	I915_WRITE(MEMMODECTL, rgvmodectl);
		2403
		2404	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		2405	DRM_ERROR("stuck trying to change perf mode\n");
		2406	mdelay(1);
		2407
		2408	ironlake_set_drps(dev, fstart);
		2409
		2410	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
		2411	I915_READ(0x112e0);
		2412	dev_priv->ips.last_time1 = jiffies_to_msecs(GetTimerTicks());
		2413	dev_priv->ips.last_count2 = I915_READ(0x112f4);
		2414	// getrawmonotonic(&dev_priv->ips.last_time2);
		2415
		2416	spin_unlock_irq(&mchdev_lock);
		2417	}
		2418
		2419	static void ironlake_disable_drps(struct drm_device *dev)
		2420	{
		2421	struct drm_i915_private *dev_priv = dev->dev_private;
		2422	u16 rgvswctl;
		2423
		2424	spin_lock_irq(&mchdev_lock);
		2425
		2426	rgvswctl = I915_READ16(MEMSWCTL);
		2427
		2428	/* Ack interrupts, disable EFC interrupt */
		2429	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		2430	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		2431	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		2432	I915_WRITE(DEIIR, DE_PCU_EVENT);
		2433	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		2434
		2435	/* Go back to the starting frequency */
		2436	ironlake_set_drps(dev, dev_priv->ips.fstart);
		2437	mdelay(1);
		2438	rgvswctl \|= MEMCTL_CMD_STS;
		2439	I915_WRITE(MEMSWCTL, rgvswctl);
		2440	mdelay(1);
		2441
		2442	spin_unlock_irq(&mchdev_lock);
		2443	}
		2444
		2445	/* There's a funny hw issue where the hw returns all 0 when reading from
		2446	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		2447	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		2448	* all limits and the gpu stuck at whatever frequency it is at atm).
		2449	*/
		2450	static u32 gen6_rps_limits(struct drm_i915_private dev_priv, u8 val)
		2451	{
		2452	u32 limits;
		2453
		2454	limits = 0;
		2455
		2456	if (*val >= dev_priv->rps.max_delay)
		2457	*val = dev_priv->rps.max_delay;
		2458	limits \|= dev_priv->rps.max_delay << 24;
		2459
		2460	/* Only set the down limit when we've reached the lowest level to avoid
		2461	* getting more interrupts, otherwise leave this clear. This prevents a
		2462	* race in the hw when coming out of rc6: There's a tiny window where
		2463	* the hw runs at the minimal clock before selecting the desired
		2464	* frequency, if the down threshold expires in that window we will not
		2465	* receive a down interrupt. */
		2466	if (*val <= dev_priv->rps.min_delay) {
		2467	*val = dev_priv->rps.min_delay;
		2468	limits \|= dev_priv->rps.min_delay << 16;
		2469	}
		2470
		2471	return limits;
		2472	}
		2473
		2474	void gen6_set_rps(struct drm_device *dev, u8 val)
		2475	{
		2476	struct drm_i915_private *dev_priv = dev->dev_private;
		2477	u32 limits = gen6_rps_limits(dev_priv, &val);
		2478
3243	Serge	2479	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2480	WARN_ON(val > dev_priv->rps.max_delay);
		2481	WARN_ON(val < dev_priv->rps.min_delay);
		2482
		2483	if (val == dev_priv->rps.cur_delay)
		2484	return;
		2485
		2486	I915_WRITE(GEN6_RPNSWREQ,
		2487	GEN6_FREQUENCY(val) \|
		2488	GEN6_OFFSET(0) \|
		2489	GEN6_AGGRESSIVE_TURBO);
		2490
		2491	/* Make sure we continue to get interrupts
		2492	* until we hit the minimum or maximum frequencies.
		2493	*/
		2494	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
		2495
		2496	POSTING_READ(GEN6_RPNSWREQ);
		2497
		2498	dev_priv->rps.cur_delay = val;
		2499
		2500	trace_intel_gpu_freq_change(val * 50);
		2501	}
		2502
		2503	static void gen6_disable_rps(struct drm_device *dev)
		2504	{
		2505	struct drm_i915_private *dev_priv = dev->dev_private;
		2506
		2507	I915_WRITE(GEN6_RC_CONTROL, 0);
		2508	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		2509	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
		2510	I915_WRITE(GEN6_PMIER, 0);
		2511	/* Complete PM interrupt masking here doesn't race with the rps work
		2512	* item again unmasking PM interrupts because that is using a different
		2513	* register (PMIMR) to mask PM interrupts. The only risk is in leaving
		2514	* stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
		2515
		2516	spin_lock_irq(&dev_priv->rps.lock);
		2517	dev_priv->rps.pm_iir = 0;
		2518	spin_unlock_irq(&dev_priv->rps.lock);
		2519
		2520	I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
		2521	}
		2522
		2523	int intel_enable_rc6(const struct drm_device *dev)
		2524	{
		2525	/* Respect the kernel parameter if it is set */
		2526	if (i915_enable_rc6 >= 0)
		2527	return i915_enable_rc6;
		2528
3120	serge	2529	/* Disable RC6 on Ironlake */
		2530	if (INTEL_INFO(dev)->gen == 5)
		2531	return 0;
3031	serge	2532
		2533	if (IS_HASWELL(dev)) {
		2534	DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
		2535	return INTEL_RC6_ENABLE;
		2536	}
		2537
		2538	/* snb/ivb have more than one rc6 state. */
		2539	if (INTEL_INFO(dev)->gen == 6) {
		2540	DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
		2541	return INTEL_RC6_ENABLE;
		2542	}
		2543
		2544	DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
		2545	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
		2546	}
		2547
		2548	static void gen6_enable_rps(struct drm_device *dev)
		2549	{
		2550	struct drm_i915_private *dev_priv = dev->dev_private;
		2551	struct intel_ring_buffer *ring;
		2552	u32 rp_state_cap;
		2553	u32 gt_perf_status;
3243	Serge	2554	u32 rc6vids, pcu_mbox, rc6_mask = 0;
3031	serge	2555	u32 gtfifodbg;
		2556	int rc6_mode;
3243	Serge	2557	int i, ret;
3031	serge	2558
3243	Serge	2559	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2560
		2561	/* Here begins a magic sequence of register writes to enable
		2562	* auto-downclocking.
		2563	*
		2564	* Perhaps there might be some value in exposing these to
		2565	* userspace...
		2566	*/
		2567	I915_WRITE(GEN6_RC_STATE, 0);
		2568
		2569	/* Clear the DBG now so we don't confuse earlier errors */
		2570	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		2571	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		2572	I915_WRITE(GTFIFODBG, gtfifodbg);
		2573	}
		2574
		2575	gen6_gt_force_wake_get(dev_priv);
		2576
		2577	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		2578	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
		2579
		2580	/* In units of 100MHz */
		2581	dev_priv->rps.max_delay = rp_state_cap & 0xff;
		2582	dev_priv->rps.min_delay = (rp_state_cap & 0xff0000) >> 16;
		2583	dev_priv->rps.cur_delay = 0;
		2584
		2585	/* disable the counters and set deterministic thresholds */
		2586	I915_WRITE(GEN6_RC_CONTROL, 0);
		2587
		2588	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		2589	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		2590	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		2591	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		2592	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		2593
		2594	for_each_ring(ring, dev_priv, i)
		2595	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		2596
		2597	I915_WRITE(GEN6_RC_SLEEP, 0);
		2598	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
		2599	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
3480	Serge	2600	I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
3031	serge	2601	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		2602
		2603	/* Check if we are enabling RC6 */
		2604	rc6_mode = intel_enable_rc6(dev_priv->dev);
		2605	if (rc6_mode & INTEL_RC6_ENABLE)
		2606	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		2607
		2608	/* We don't use those on Haswell */
		2609	if (!IS_HASWELL(dev)) {
		2610	if (rc6_mode & INTEL_RC6p_ENABLE)
		2611	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		2612
		2613	if (rc6_mode & INTEL_RC6pp_ENABLE)
		2614	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		2615	}
		2616
		2617	DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
		2618	(rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		2619	(rc6_mask & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		2620	(rc6_mask & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
		2621
		2622	I915_WRITE(GEN6_RC_CONTROL,
		2623	rc6_mask \|
		2624	GEN6_RC_CTL_EI_MODE(1) \|
		2625	GEN6_RC_CTL_HW_ENABLE);
		2626
		2627	I915_WRITE(GEN6_RPNSWREQ,
		2628	GEN6_FREQUENCY(10) \|
		2629	GEN6_OFFSET(0) \|
		2630	GEN6_AGGRESSIVE_TURBO);
		2631	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		2632	GEN6_FREQUENCY(12));
		2633
		2634	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
		2635	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
		2636	dev_priv->rps.max_delay << 24 \|
		2637	dev_priv->rps.min_delay << 16);
		2638
		2639	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		2640	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		2641	I915_WRITE(GEN6_RP_UP_EI, 66000);
		2642	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		2643
		2644	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		2645	I915_WRITE(GEN6_RP_CONTROL,
		2646	GEN6_RP_MEDIA_TURBO \|
		2647	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		2648	GEN6_RP_MEDIA_IS_GFX \|
		2649	GEN6_RP_ENABLE \|
		2650	GEN6_RP_UP_BUSY_AVG \|
		2651	(IS_HASWELL(dev) ? GEN7_RP_DOWN_IDLE_AVG : GEN6_RP_DOWN_IDLE_CONT));
		2652
3243	Serge	2653	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
		2654	if (!ret) {
		2655	pcu_mbox = 0;
		2656	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
		2657	if (ret && pcu_mbox & (1<<31)) { /* OC supported */
3031	serge	2658	dev_priv->rps.max_delay = pcu_mbox & 0xff;
		2659	DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
		2660	}
3243	Serge	2661	} else {
		2662	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		2663	}
3031	serge	2664
		2665	gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
		2666
		2667	/* requires MSI enabled */
		2668	I915_WRITE(GEN6_PMIER, GEN6_PM_DEFERRED_EVENTS);
		2669	spin_lock_irq(&dev_priv->rps.lock);
		2670	WARN_ON(dev_priv->rps.pm_iir != 0);
		2671	I915_WRITE(GEN6_PMIMR, 0);
		2672	spin_unlock_irq(&dev_priv->rps.lock);
		2673	/* enable all PM interrupts */
		2674	I915_WRITE(GEN6_PMINTRMSK, 0);
		2675
3243	Serge	2676	rc6vids = 0;
		2677	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		2678	if (IS_GEN6(dev) && ret) {
		2679	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		2680	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		2681	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		2682	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		2683	rc6vids &= 0xffff00;
		2684	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		2685	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		2686	if (ret)
		2687	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		2688	}
		2689
3031	serge	2690	gen6_gt_force_wake_put(dev_priv);
		2691	}
		2692
		2693	#if 0
		2694	static void gen6_update_ring_freq(struct drm_device *dev)
		2695	{
		2696	struct drm_i915_private *dev_priv = dev->dev_private;
		2697	int min_freq = 15;
3243	Serge	2698	int gpu_freq;
		2699	unsigned int ia_freq, max_ia_freq;
3031	serge	2700	int scaling_factor = 180;
		2701
3243	Serge	2702	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2703
		2704	max_ia_freq = cpufreq_quick_get_max(0);
		2705	/*
		2706	* Default to measured freq if none found, PCU will ensure we don't go
		2707	* over
		2708	*/
		2709	if (!max_ia_freq)
		2710	max_ia_freq = tsc_khz;
		2711
		2712	/* Convert from kHz to MHz */
		2713	max_ia_freq /= 1000;
		2714
		2715	/*
		2716	* For each potential GPU frequency, load a ring frequency we'd like
		2717	* to use for memory access. We do this by specifying the IA frequency
		2718	* the PCU should use as a reference to determine the ring frequency.
		2719	*/
		2720	for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
		2721	gpu_freq--) {
		2722	int diff = dev_priv->rps.max_delay - gpu_freq;
		2723
		2724	/*
		2725	* For GPU frequencies less than 750MHz, just use the lowest
		2726	* ring freq.
		2727	*/
		2728	if (gpu_freq < min_freq)
		2729	ia_freq = 800;
		2730	else
		2731	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		2732	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3243	Serge	2733	ia_freq <<= GEN6_PCODE_FREQ_IA_RATIO_SHIFT;
3031	serge	2734
3243	Serge	2735	sandybridge_pcode_write(dev_priv,
		2736	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
		2737	ia_freq \| gpu_freq);
3031	serge	2738	}
		2739	}
		2740	#endif
		2741
		2742	void ironlake_teardown_rc6(struct drm_device *dev)
		2743	{
		2744	struct drm_i915_private *dev_priv = dev->dev_private;
		2745
3243	Serge	2746	if (dev_priv->ips.renderctx) {
		2747	i915_gem_object_unpin(dev_priv->ips.renderctx);
		2748	drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
		2749	dev_priv->ips.renderctx = NULL;
3031	serge	2750	}
		2751
3243	Serge	2752	if (dev_priv->ips.pwrctx) {
		2753	i915_gem_object_unpin(dev_priv->ips.pwrctx);
		2754	drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
		2755	dev_priv->ips.pwrctx = NULL;
3031	serge	2756	}
		2757	}
		2758
		2759	static void ironlake_disable_rc6(struct drm_device *dev)
		2760	{
		2761	struct drm_i915_private *dev_priv = dev->dev_private;
		2762
		2763	if (I915_READ(PWRCTXA)) {
		2764	/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
		2765	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) \| RCX_SW_EXIT);
		2766	wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
		2767	50);
		2768
		2769	I915_WRITE(PWRCTXA, 0);
		2770	POSTING_READ(PWRCTXA);
		2771
		2772	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		2773	POSTING_READ(RSTDBYCTL);
		2774	}
		2775	}
		2776
		2777	static int ironlake_setup_rc6(struct drm_device *dev)
		2778	{
		2779	struct drm_i915_private *dev_priv = dev->dev_private;
		2780
3243	Serge	2781	if (dev_priv->ips.renderctx == NULL)
		2782	dev_priv->ips.renderctx = intel_alloc_context_page(dev);
		2783	if (!dev_priv->ips.renderctx)
3031	serge	2784	return -ENOMEM;
		2785
3243	Serge	2786	if (dev_priv->ips.pwrctx == NULL)
		2787	dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
		2788	if (!dev_priv->ips.pwrctx) {
3031	serge	2789	ironlake_teardown_rc6(dev);
		2790	return -ENOMEM;
		2791	}
		2792
		2793	return 0;
		2794	}
		2795
		2796	static void ironlake_enable_rc6(struct drm_device *dev)
		2797	{
		2798	struct drm_i915_private *dev_priv = dev->dev_private;
		2799	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
3243	Serge	2800	bool was_interruptible;
3031	serge	2801	int ret;
		2802
		2803	/* rc6 disabled by default due to repeated reports of hanging during
		2804	* boot and resume.
		2805	*/
		2806	if (!intel_enable_rc6(dev))
		2807	return;
		2808
		2809	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2810
		2811	ret = ironlake_setup_rc6(dev);
		2812	if (ret)
		2813	return;
		2814
3243	Serge	2815	was_interruptible = dev_priv->mm.interruptible;
		2816	dev_priv->mm.interruptible = false;
		2817
3031	serge	2818	/*
		2819	* GPU can automatically power down the render unit if given a page
		2820	* to save state.
		2821	*/
		2822	ret = intel_ring_begin(ring, 6);
		2823	if (ret) {
		2824	ironlake_teardown_rc6(dev);
3243	Serge	2825	dev_priv->mm.interruptible = was_interruptible;
3031	serge	2826	return;
		2827	}
		2828
		2829	intel_ring_emit(ring, MI_SUSPEND_FLUSH \| MI_SUSPEND_FLUSH_EN);
		2830	intel_ring_emit(ring, MI_SET_CONTEXT);
3243	Serge	2831	intel_ring_emit(ring, dev_priv->ips.renderctx->gtt_offset \|
3031	serge	2832	MI_MM_SPACE_GTT \|
		2833	MI_SAVE_EXT_STATE_EN \|
		2834	MI_RESTORE_EXT_STATE_EN \|
		2835	MI_RESTORE_INHIBIT);
		2836	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
		2837	intel_ring_emit(ring, MI_NOOP);
		2838	intel_ring_emit(ring, MI_FLUSH);
		2839	intel_ring_advance(ring);
		2840
		2841	/*
		2842	* Wait for the command parser to advance past MI_SET_CONTEXT. The HW
		2843	* does an implicit flush, combined with MI_FLUSH above, it should be
		2844	* safe to assume that renderctx is valid
		2845	*/
3243	Serge	2846	ret = intel_ring_idle(ring);
		2847	dev_priv->mm.interruptible = was_interruptible;
3031	serge	2848	if (ret) {
		2849	DRM_ERROR("failed to enable ironlake power power savings\n");
		2850	ironlake_teardown_rc6(dev);
		2851	return;
		2852	}
		2853
3243	Serge	2854	I915_WRITE(PWRCTXA, dev_priv->ips.pwrctx->gtt_offset \| PWRCTX_EN);
3031	serge	2855	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		2856	}
		2857
		2858	static unsigned long intel_pxfreq(u32 vidfreq)
		2859	{
		2860	unsigned long freq;
		2861	int div = (vidfreq & 0x3f0000) >> 16;
		2862	int post = (vidfreq & 0x3000) >> 12;
		2863	int pre = (vidfreq & 0x7);
		2864
		2865	if (!pre)
		2866	return 0;
		2867
		2868	freq = ((div * 133333) / ((1<
		2869
		2870	return freq;
		2871	}
		2872
		2873	static const struct cparams {
		2874	u16 i;
		2875	u16 t;
		2876	u16 m;
		2877	u16 c;
		2878	} cparams[] = {
		2879	{ 1, 1333, 301, 28664 },
		2880	{ 1, 1066, 294, 24460 },
		2881	{ 1, 800, 294, 25192 },
		2882	{ 0, 1333, 276, 27605 },
		2883	{ 0, 1066, 276, 27605 },
		2884	{ 0, 800, 231, 23784 },
		2885	};
		2886
		2887	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		2888	{
		2889	u64 total_count, diff, ret;
		2890	u32 count1, count2, count3, m = 0, c = 0;
		2891	unsigned long now = jiffies_to_msecs(GetTimerTicks()), diff1;
		2892	int i;
		2893
		2894	assert_spin_locked(&mchdev_lock);
		2895
		2896	diff1 = now - dev_priv->ips.last_time1;
		2897
		2898	/* Prevent division-by-zero if we are asking too fast.
		2899	* Also, we don't get interesting results if we are polling
		2900	* faster than once in 10ms, so just return the saved value
		2901	* in such cases.
		2902	*/
		2903	if (diff1 <= 10)
		2904	return dev_priv->ips.chipset_power;
		2905
		2906	count1 = I915_READ(DMIEC);
		2907	count2 = I915_READ(DDREC);
		2908	count3 = I915_READ(CSIEC);
		2909
		2910	total_count = count1 + count2 + count3;
		2911
		2912	/* FIXME: handle per-counter overflow */
		2913	if (total_count < dev_priv->ips.last_count1) {
		2914	diff = ~0UL - dev_priv->ips.last_count1;
		2915	diff += total_count;
		2916	} else {
		2917	diff = total_count - dev_priv->ips.last_count1;
		2918	}
		2919
		2920	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		2921	if (cparams[i].i == dev_priv->ips.c_m &&
		2922	cparams[i].t == dev_priv->ips.r_t) {
		2923	m = cparams[i].m;
		2924	c = cparams[i].c;
		2925	break;
		2926	}
		2927	}
		2928
		2929	diff = div_u64(diff, diff1);
		2930	ret = ((m * diff) + c);
		2931	ret = div_u64(ret, 10);
		2932
		2933	dev_priv->ips.last_count1 = total_count;
		2934	dev_priv->ips.last_time1 = now;
		2935
		2936	dev_priv->ips.chipset_power = ret;
		2937
		2938	return ret;
		2939	}
		2940
		2941	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		2942	{
		2943	unsigned long val;
		2944
		2945	if (dev_priv->info->gen != 5)
		2946	return 0;
		2947
		2948	spin_lock_irq(&mchdev_lock);
		2949
		2950	val = __i915_chipset_val(dev_priv);
		2951
		2952	spin_unlock_irq(&mchdev_lock);
		2953
		2954	return val;
		2955	}
		2956
		2957	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		2958	{
		2959	unsigned long m, x, b;
		2960	u32 tsfs;
		2961
		2962	tsfs = I915_READ(TSFS);
		2963
		2964	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		2965	x = I915_READ8(TR1);
		2966
		2967	b = tsfs & TSFS_INTR_MASK;
		2968
		2969	return ((m * x) / 127) - b;
		2970	}
		2971
		2972	static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		2973	{
		2974	static const struct v_table {
		2975	u16 vd; /* in .1 mil */
		2976	u16 vm; /* in .1 mil */
		2977	} v_table[] = {
		2978	{ 0, 0, },
		2979	{ 375, 0, },
		2980	{ 500, 0, },
		2981	{ 625, 0, },
		2982	{ 750, 0, },
		2983	{ 875, 0, },
		2984	{ 1000, 0, },
		2985	{ 1125, 0, },
		2986	{ 4125, 3000, },
		2987	{ 4125, 3000, },
		2988	{ 4125, 3000, },
		2989	{ 4125, 3000, },
		2990	{ 4125, 3000, },
		2991	{ 4125, 3000, },
		2992	{ 4125, 3000, },
		2993	{ 4125, 3000, },
		2994	{ 4125, 3000, },
		2995	{ 4125, 3000, },
		2996	{ 4125, 3000, },
		2997	{ 4125, 3000, },
		2998	{ 4125, 3000, },
		2999	{ 4125, 3000, },
		3000	{ 4125, 3000, },
		3001	{ 4125, 3000, },
		3002	{ 4125, 3000, },
		3003	{ 4125, 3000, },
		3004	{ 4125, 3000, },
		3005	{ 4125, 3000, },
		3006	{ 4125, 3000, },
		3007	{ 4125, 3000, },
		3008	{ 4125, 3000, },
		3009	{ 4125, 3000, },
		3010	{ 4250, 3125, },
		3011	{ 4375, 3250, },
		3012	{ 4500, 3375, },
		3013	{ 4625, 3500, },
		3014	{ 4750, 3625, },
		3015	{ 4875, 3750, },
		3016	{ 5000, 3875, },
		3017	{ 5125, 4000, },
		3018	{ 5250, 4125, },
		3019	{ 5375, 4250, },
		3020	{ 5500, 4375, },
		3021	{ 5625, 4500, },
		3022	{ 5750, 4625, },
		3023	{ 5875, 4750, },
		3024	{ 6000, 4875, },
		3025	{ 6125, 5000, },
		3026	{ 6250, 5125, },
		3027	{ 6375, 5250, },
		3028	{ 6500, 5375, },
		3029	{ 6625, 5500, },
		3030	{ 6750, 5625, },
		3031	{ 6875, 5750, },
		3032	{ 7000, 5875, },
		3033	{ 7125, 6000, },
		3034	{ 7250, 6125, },
		3035	{ 7375, 6250, },
		3036	{ 7500, 6375, },
		3037	{ 7625, 6500, },
		3038	{ 7750, 6625, },
		3039	{ 7875, 6750, },
		3040	{ 8000, 6875, },
		3041	{ 8125, 7000, },
		3042	{ 8250, 7125, },
		3043	{ 8375, 7250, },
		3044	{ 8500, 7375, },
		3045	{ 8625, 7500, },
		3046	{ 8750, 7625, },
		3047	{ 8875, 7750, },
		3048	{ 9000, 7875, },
		3049	{ 9125, 8000, },
		3050	{ 9250, 8125, },
		3051	{ 9375, 8250, },
		3052	{ 9500, 8375, },
		3053	{ 9625, 8500, },
		3054	{ 9750, 8625, },
		3055	{ 9875, 8750, },
		3056	{ 10000, 8875, },
		3057	{ 10125, 9000, },
		3058	{ 10250, 9125, },
		3059	{ 10375, 9250, },
		3060	{ 10500, 9375, },
		3061	{ 10625, 9500, },
		3062	{ 10750, 9625, },
		3063	{ 10875, 9750, },
		3064	{ 11000, 9875, },
		3065	{ 11125, 10000, },
		3066	{ 11250, 10125, },
		3067	{ 11375, 10250, },
		3068	{ 11500, 10375, },
		3069	{ 11625, 10500, },
		3070	{ 11750, 10625, },
		3071	{ 11875, 10750, },
		3072	{ 12000, 10875, },
		3073	{ 12125, 11000, },
		3074	{ 12250, 11125, },
		3075	{ 12375, 11250, },
		3076	{ 12500, 11375, },
		3077	{ 12625, 11500, },
		3078	{ 12750, 11625, },
		3079	{ 12875, 11750, },
		3080	{ 13000, 11875, },
		3081	{ 13125, 12000, },
		3082	{ 13250, 12125, },
		3083	{ 13375, 12250, },
		3084	{ 13500, 12375, },
		3085	{ 13625, 12500, },
		3086	{ 13750, 12625, },
		3087	{ 13875, 12750, },
		3088	{ 14000, 12875, },
		3089	{ 14125, 13000, },
		3090	{ 14250, 13125, },
		3091	{ 14375, 13250, },
		3092	{ 14500, 13375, },
		3093	{ 14625, 13500, },
		3094	{ 14750, 13625, },
		3095	{ 14875, 13750, },
		3096	{ 15000, 13875, },
		3097	{ 15125, 14000, },
		3098	{ 15250, 14125, },
		3099	{ 15375, 14250, },
		3100	{ 15500, 14375, },
		3101	{ 15625, 14500, },
		3102	{ 15750, 14625, },
		3103	{ 15875, 14750, },
		3104	{ 16000, 14875, },
		3105	{ 16125, 15000, },
		3106	};
		3107	if (dev_priv->info->is_mobile)
		3108	return v_table[pxvid].vm;
		3109	else
		3110	return v_table[pxvid].vd;
		3111	}
		3112
		3113	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		3114	{
		3115	struct timespec now, diff1;
		3116	u64 diff;
		3117	unsigned long diffms;
		3118	u32 count;
		3119
		3120	assert_spin_locked(&mchdev_lock);
		3121
		3122	getrawmonotonic(&now);
		3123	diff1 = timespec_sub(now, dev_priv->ips.last_time2);
		3124
		3125	/* Don't divide by 0 */
		3126	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
		3127	if (!diffms)
		3128	return;
		3129
		3130	count = I915_READ(GFXEC);
		3131
		3132	if (count < dev_priv->ips.last_count2) {
		3133	diff = ~0UL - dev_priv->ips.last_count2;
		3134	diff += count;
		3135	} else {
		3136	diff = count - dev_priv->ips.last_count2;
		3137	}
		3138
		3139	dev_priv->ips.last_count2 = count;
		3140	dev_priv->ips.last_time2 = now;
		3141
		3142	/* More magic constants... */
		3143	diff = diff * 1181;
		3144	diff = div_u64(diff, diffms * 10);
		3145	dev_priv->ips.gfx_power = diff;
		3146	}
		3147
		3148	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		3149	{
		3150	if (dev_priv->info->gen != 5)
		3151	return;
		3152
		3153	spin_lock_irq(&mchdev_lock);
		3154
		3155	__i915_update_gfx_val(dev_priv);
		3156
		3157	spin_unlock_irq(&mchdev_lock);
		3158	}
		3159
		3160	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		3161	{
		3162	unsigned long t, corr, state1, corr2, state2;
		3163	u32 pxvid, ext_v;
		3164
		3165	assert_spin_locked(&mchdev_lock);
		3166
		3167	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
		3168	pxvid = (pxvid >> 24) & 0x7f;
		3169	ext_v = pvid_to_extvid(dev_priv, pxvid);
		3170
		3171	state1 = ext_v;
		3172
		3173	t = i915_mch_val(dev_priv);
		3174
		3175	/* Revel in the empirically derived constants */
		3176
		3177	/* Correction factor in 1/100000 units */
		3178	if (t > 80)
		3179	corr = ((t * 2349) + 135940);
		3180	else if (t >= 50)
		3181	corr = ((t * 964) + 29317);
		3182	else /* < 50 */
		3183	corr = ((t * 301) + 1004);
		3184
		3185	corr = corr * ((150142 * state1) / 10000 - 78642);
		3186	corr /= 100000;
		3187	corr2 = (corr * dev_priv->ips.corr);
		3188
		3189	state2 = (corr2 * state1) / 10000;
		3190	state2 /= 100; /* convert to mW */
		3191
		3192	__i915_update_gfx_val(dev_priv);
		3193
		3194	return dev_priv->ips.gfx_power + state2;
		3195	}
		3196
		3197	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		3198	{
		3199	unsigned long val;
		3200
		3201	if (dev_priv->info->gen != 5)
		3202	return 0;
		3203
		3204	spin_lock_irq(&mchdev_lock);
		3205
		3206	val = __i915_gfx_val(dev_priv);
		3207
		3208	spin_unlock_irq(&mchdev_lock);
		3209
		3210	return val;
		3211	}
		3212
		3213	/**
		3214	* i915_read_mch_val - return value for IPS use
		3215	*
		3216	* Calculate and return a value for the IPS driver to use when deciding whether
		3217	* we have thermal and power headroom to increase CPU or GPU power budget.
		3218	*/
		3219	unsigned long i915_read_mch_val(void)
		3220	{
		3221	struct drm_i915_private *dev_priv;
		3222	unsigned long chipset_val, graphics_val, ret = 0;
		3223
		3224	spin_lock_irq(&mchdev_lock);
		3225	if (!i915_mch_dev)
		3226	goto out_unlock;
		3227	dev_priv = i915_mch_dev;
		3228
		3229	chipset_val = __i915_chipset_val(dev_priv);
		3230	graphics_val = __i915_gfx_val(dev_priv);
		3231
		3232	ret = chipset_val + graphics_val;
		3233
		3234	out_unlock:
		3235	spin_unlock_irq(&mchdev_lock);
		3236
		3237	return ret;
		3238	}
		3239	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		3240
		3241	/**
		3242	* i915_gpu_raise - raise GPU frequency limit
		3243	*
		3244	* Raise the limit; IPS indicates we have thermal headroom.
		3245	*/
		3246	bool i915_gpu_raise(void)
		3247	{
		3248	struct drm_i915_private *dev_priv;
		3249	bool ret = true;
		3250
		3251	spin_lock_irq(&mchdev_lock);
		3252	if (!i915_mch_dev) {
		3253	ret = false;
		3254	goto out_unlock;
		3255	}
		3256	dev_priv = i915_mch_dev;
		3257
		3258	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		3259	dev_priv->ips.max_delay--;
		3260
		3261	out_unlock:
		3262	spin_unlock_irq(&mchdev_lock);
		3263
		3264	return ret;
		3265	}
		3266	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		3267
		3268	/**
		3269	* i915_gpu_lower - lower GPU frequency limit
		3270	*
		3271	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		3272	* frequency maximum.
		3273	*/
		3274	bool i915_gpu_lower(void)
		3275	{
		3276	struct drm_i915_private *dev_priv;
		3277	bool ret = true;
		3278
		3279	spin_lock_irq(&mchdev_lock);
		3280	if (!i915_mch_dev) {
		3281	ret = false;
		3282	goto out_unlock;
		3283	}
		3284	dev_priv = i915_mch_dev;
		3285
		3286	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		3287	dev_priv->ips.max_delay++;
		3288
		3289	out_unlock:
		3290	spin_unlock_irq(&mchdev_lock);
		3291
		3292	return ret;
		3293	}
		3294	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		3295
		3296	/**
		3297	* i915_gpu_busy - indicate GPU business to IPS
		3298	*
		3299	* Tell the IPS driver whether or not the GPU is busy.
		3300	*/
		3301	bool i915_gpu_busy(void)
		3302	{
		3303	struct drm_i915_private *dev_priv;
		3304	struct intel_ring_buffer *ring;
		3305	bool ret = false;
		3306	int i;
		3307
		3308	spin_lock_irq(&mchdev_lock);
		3309	if (!i915_mch_dev)
		3310	goto out_unlock;
		3311	dev_priv = i915_mch_dev;
		3312
		3313	for_each_ring(ring, dev_priv, i)
		3314	ret \|= !list_empty(&ring->request_list);
		3315
		3316	out_unlock:
		3317	spin_unlock_irq(&mchdev_lock);
		3318
		3319	return ret;
		3320	}
		3321	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		3322
		3323	/**
		3324	* i915_gpu_turbo_disable - disable graphics turbo
		3325	*
		3326	* Disable graphics turbo by resetting the max frequency and setting the
		3327	* current frequency to the default.
		3328	*/
		3329	bool i915_gpu_turbo_disable(void)
		3330	{
		3331	struct drm_i915_private *dev_priv;
		3332	bool ret = true;
		3333
		3334	spin_lock_irq(&mchdev_lock);
		3335	if (!i915_mch_dev) {
		3336	ret = false;
		3337	goto out_unlock;
		3338	}
		3339	dev_priv = i915_mch_dev;
		3340
		3341	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		3342
		3343	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		3344	ret = false;
		3345
		3346	out_unlock:
		3347	spin_unlock_irq(&mchdev_lock);
		3348
		3349	return ret;
		3350	}
		3351	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		3352
		3353	/**
		3354	* Tells the intel_ips driver that the i915 driver is now loaded, if
		3355	* IPS got loaded first.
		3356	*
		3357	* This awkward dance is so that neither module has to depend on the
		3358	* other in order for IPS to do the appropriate communication of
		3359	* GPU turbo limits to i915.
		3360	*/
		3361	static void
		3362	ips_ping_for_i915_load(void)
		3363	{
		3364	void (*link)(void);
		3365
		3366	// link = symbol_get(ips_link_to_i915_driver);
		3367	// if (link) {
		3368	// link();
		3369	// symbol_put(ips_link_to_i915_driver);
		3370	// }
		3371	}
		3372
		3373	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		3374	{
		3375	/* We only register the i915 ips part with intel-ips once everything is
		3376	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		3377	spin_lock_irq(&mchdev_lock);
		3378	i915_mch_dev = dev_priv;
		3379	spin_unlock_irq(&mchdev_lock);
		3380
		3381	ips_ping_for_i915_load();
		3382	}
		3383
		3384	void intel_gpu_ips_teardown(void)
		3385	{
		3386	spin_lock_irq(&mchdev_lock);
		3387	i915_mch_dev = NULL;
		3388	spin_unlock_irq(&mchdev_lock);
		3389	}
		3390	static void intel_init_emon(struct drm_device *dev)
		3391	{
		3392	struct drm_i915_private *dev_priv = dev->dev_private;
		3393	u32 lcfuse;
		3394	u8 pxw[16];
		3395	int i;
		3396
		3397	/* Disable to program */
		3398	I915_WRITE(ECR, 0);
		3399	POSTING_READ(ECR);
		3400
		3401	/* Program energy weights for various events */
		3402	I915_WRITE(SDEW, 0x15040d00);
		3403	I915_WRITE(CSIEW0, 0x007f0000);
		3404	I915_WRITE(CSIEW1, 0x1e220004);
		3405	I915_WRITE(CSIEW2, 0x04000004);
		3406
		3407	for (i = 0; i < 5; i++)
		3408	I915_WRITE(PEW + (i * 4), 0);
		3409	for (i = 0; i < 3; i++)
		3410	I915_WRITE(DEW + (i * 4), 0);
		3411
		3412	/* Program P-state weights to account for frequency power adjustment */
		3413	for (i = 0; i < 16; i++) {
		3414	u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
		3415	unsigned long freq = intel_pxfreq(pxvidfreq);
		3416	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		3417	PXVFREQ_PX_SHIFT;
		3418	unsigned long val;
		3419
		3420	val = vid * vid;
		3421	val *= (freq / 1000);
		3422	val *= 255;
		3423	val /= (127127900);
		3424	if (val > 0xff)
		3425	DRM_ERROR("bad pxval: %ld\n", val);
		3426	pxw[i] = val;
		3427	}
		3428	/* Render standby states get 0 weight */
		3429	pxw[14] = 0;
		3430	pxw[15] = 0;
		3431
		3432	for (i = 0; i < 4; i++) {
		3433	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		3434	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
		3435	I915_WRITE(PXW + (i * 4), val);
		3436	}
		3437
		3438	/* Adjust magic regs to magic values (more experimental results) */
		3439	I915_WRITE(OGW0, 0);
		3440	I915_WRITE(OGW1, 0);
		3441	I915_WRITE(EG0, 0x00007f00);
		3442	I915_WRITE(EG1, 0x0000000e);
		3443	I915_WRITE(EG2, 0x000e0000);
		3444	I915_WRITE(EG3, 0x68000300);
		3445	I915_WRITE(EG4, 0x42000000);
		3446	I915_WRITE(EG5, 0x00140031);
		3447	I915_WRITE(EG6, 0);
		3448	I915_WRITE(EG7, 0);
		3449
		3450	for (i = 0; i < 8; i++)
		3451	I915_WRITE(PXWL + (i * 4), 0);
		3452
		3453	/* Enable PMON + select events */
		3454	I915_WRITE(ECR, 0x80000019);
		3455
		3456	lcfuse = I915_READ(LCFUSE02);
		3457
		3458	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		3459	}
		3460
		3461	void intel_disable_gt_powersave(struct drm_device *dev)
		3462	{
3243	Serge	3463	struct drm_i915_private *dev_priv = dev->dev_private;
		3464
3031	serge	3465	if (IS_IRONLAKE_M(dev)) {
		3466	ironlake_disable_drps(dev);
		3467	ironlake_disable_rc6(dev);
		3468	} else if (INTEL_INFO(dev)->gen >= 6 && !IS_VALLEYVIEW(dev)) {
		3469	gen6_disable_rps(dev);
3480	Serge	3470	mutex_unlock(&dev_priv->rps.hw_lock);
3031	serge	3471	}
		3472	}
		3473
		3474	void intel_enable_gt_powersave(struct drm_device *dev)
		3475	{
3243	Serge	3476	struct drm_i915_private *dev_priv = dev->dev_private;
		3477
3031	serge	3478	if (IS_IRONLAKE_M(dev)) {
		3479	ironlake_enable_drps(dev);
		3480	ironlake_enable_rc6(dev);
		3481	intel_init_emon(dev);
		3482	} else if ((IS_GEN6(dev) \|\| IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) {
3243	Serge	3483	/*
		3484	* PCU communication is slow and this doesn't need to be
		3485	* done at any specific time, so do this out of our fast path
		3486	* to make resume and init faster.
		3487	*/
		3488	// schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		3489	// round_jiffies_up_relative(HZ));
3031	serge	3490	}
		3491	}
		3492
3243	Serge	3493	static void ibx_init_clock_gating(struct drm_device *dev)
		3494	{
		3495	struct drm_i915_private *dev_priv = dev->dev_private;
		3496
		3497	/*
		3498	* On Ibex Peak and Cougar Point, we need to disable clock
		3499	* gating for the panel power sequencer or it will fail to
		3500	* start up when no ports are active.
		3501	*/
		3502	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		3503	}
		3504
3031	serge	3505	static void ironlake_init_clock_gating(struct drm_device *dev)
		3506	{
		3507	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	3508	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	3509
		3510	/* Required for FBC */
3243	Serge	3511	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		3512	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		3513	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	3514
		3515	I915_WRITE(PCH_3DCGDIS0,
		3516	MARIUNIT_CLOCK_GATE_DISABLE \|
		3517	SVSMUNIT_CLOCK_GATE_DISABLE);
		3518	I915_WRITE(PCH_3DCGDIS1,
		3519	VFMUNIT_CLOCK_GATE_DISABLE);
		3520
		3521	/*
		3522	* According to the spec the following bits should be set in
		3523	* order to enable memory self-refresh
		3524	* The bit 22/21 of 0x42004
		3525	* The bit 5 of 0x42020
		3526	* The bit 15 of 0x45000
		3527	*/
		3528	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3529	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3530	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	3531	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	3532	I915_WRITE(DISP_ARB_CTL,
		3533	(I915_READ(DISP_ARB_CTL) \|
		3534	DISP_FBC_WM_DIS));
		3535	I915_WRITE(WM3_LP_ILK, 0);
		3536	I915_WRITE(WM2_LP_ILK, 0);
		3537	I915_WRITE(WM1_LP_ILK, 0);
		3538
		3539	/*
		3540	* Based on the document from hardware guys the following bits
		3541	* should be set unconditionally in order to enable FBC.
		3542	* The bit 22 of 0x42000
		3543	* The bit 22 of 0x42004
		3544	* The bit 7,8,9 of 0x42020.
		3545	*/
		3546	if (IS_IRONLAKE_M(dev)) {
		3547	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		3548	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		3549	ILK_FBCQ_DIS);
		3550	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3551	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3552	ILK_DPARB_GATE);
		3553	}
		3554
3243	Serge	3555	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		3556
3031	serge	3557	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3558	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3559	ILK_ELPIN_409_SELECT);
		3560	I915_WRITE(_3D_CHICKEN2,
		3561	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		3562	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	3563
		3564	/* WaDisableRenderCachePipelinedFlush */
		3565	I915_WRITE(CACHE_MODE_0,
		3566	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		3567
		3568	ibx_init_clock_gating(dev);
3031	serge	3569	}
		3570
3243	Serge	3571	static void cpt_init_clock_gating(struct drm_device *dev)
		3572	{
		3573	struct drm_i915_private *dev_priv = dev->dev_private;
		3574	int pipe;
		3575
		3576	/*
		3577	* On Ibex Peak and Cougar Point, we need to disable clock
		3578	* gating for the panel power sequencer or it will fail to
		3579	* start up when no ports are active.
		3580	*/
		3581	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		3582	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		3583	DPLS_EDP_PPS_FIX_DIS);
		3584	/* The below fixes the weird display corruption, a few pixels shifted
		3585	* downward, on (only) LVDS of some HP laptops with IVY.
		3586	*/
		3587	for_each_pipe(pipe)
		3588	I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_CHICKEN2_TIMING_OVERRIDE);
		3589	/* WADP0ClockGatingDisable */
		3590	for_each_pipe(pipe) {
		3591	I915_WRITE(TRANS_CHICKEN1(pipe),
		3592	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		3593	}
		3594	}
		3595
3480	Serge	3596	static void gen6_check_mch_setup(struct drm_device *dev)
		3597	{
		3598	struct drm_i915_private *dev_priv = dev->dev_private;
		3599	uint32_t tmp;
		3600
		3601	tmp = I915_READ(MCH_SSKPD);
		3602	if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL) {
		3603	DRM_INFO("Wrong MCH_SSKPD value: 0x%08x\n", tmp);
		3604	DRM_INFO("This can cause pipe underruns and display issues.\n");
		3605	DRM_INFO("Please upgrade your BIOS to fix this.\n");
		3606	}
		3607	}
		3608
3031	serge	3609	static void gen6_init_clock_gating(struct drm_device *dev)
		3610	{
		3611	struct drm_i915_private *dev_priv = dev->dev_private;
		3612	int pipe;
3243	Serge	3613	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	3614
3243	Serge	3615	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	3616
		3617	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3618	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3619	ILK_ELPIN_409_SELECT);
		3620
3243	Serge	3621	/* WaDisableHiZPlanesWhenMSAAEnabled */
		3622	I915_WRITE(_3D_CHICKEN,
		3623	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		3624
		3625	/* WaSetupGtModeTdRowDispatch */
		3626	if (IS_SNB_GT1(dev))
		3627	I915_WRITE(GEN6_GT_MODE,
		3628	_MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
		3629
3031	serge	3630	I915_WRITE(WM3_LP_ILK, 0);
		3631	I915_WRITE(WM2_LP_ILK, 0);
		3632	I915_WRITE(WM1_LP_ILK, 0);
		3633
		3634	I915_WRITE(CACHE_MODE_0,
		3635	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		3636
		3637	I915_WRITE(GEN6_UCGCTL1,
		3638	I915_READ(GEN6_UCGCTL1) \|
		3639	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		3640	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		3641
		3642	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3643	* gating disable must be set. Failure to set it results in
		3644	* flickering pixels due to Z write ordering failures after
		3645	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3646	* Sanctuary and Tropics, and apparently anything else with
		3647	* alpha test or pixel discard.
		3648	*
		3649	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3650	* but we didn't debug actual testcases to find it out.
		3651	*
		3652	* Also apply WaDisableVDSUnitClockGating and
		3653	* WaDisableRCPBUnitClockGating.
		3654	*/
		3655	I915_WRITE(GEN6_UCGCTL2,
		3656	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		3657	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		3658	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3659
		3660	/* Bspec says we need to always set all mask bits. */
		3661	I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) \|
		3662	_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
		3663
		3664	/*
		3665	* According to the spec the following bits should be
		3666	* set in order to enable memory self-refresh and fbc:
		3667	* The bit21 and bit22 of 0x42000
		3668	* The bit21 and bit22 of 0x42004
		3669	* The bit5 and bit7 of 0x42020
		3670	* The bit14 of 0x70180
		3671	* The bit14 of 0x71180
		3672	*/
		3673	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		3674	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		3675	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		3676	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3677	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3678	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	3679	I915_WRITE(ILK_DSPCLK_GATE_D,
		3680	I915_READ(ILK_DSPCLK_GATE_D) \|
		3681	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		3682	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	3683
3243	Serge	3684	/* WaMbcDriverBootEnable */
3031	serge	3685	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3686	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3687
		3688	for_each_pipe(pipe) {
		3689	I915_WRITE(DSPCNTR(pipe),
		3690	I915_READ(DSPCNTR(pipe)) \|
		3691	DISPPLANE_TRICKLE_FEED_DISABLE);
		3692	intel_flush_display_plane(dev_priv, pipe);
		3693	}
		3694
		3695	/* The default value should be 0x200 according to docs, but the two
		3696	* platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
		3697	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
		3698	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
3243	Serge	3699
		3700	cpt_init_clock_gating(dev);
3480	Serge	3701
		3702	gen6_check_mch_setup(dev);
3031	serge	3703	}
		3704
		3705	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		3706	{
		3707	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		3708
		3709	reg &= ~GEN7_FF_SCHED_MASK;
		3710	reg \|= GEN7_FF_TS_SCHED_HW;
		3711	reg \|= GEN7_FF_VS_SCHED_HW;
		3712	reg \|= GEN7_FF_DS_SCHED_HW;
		3713
3480	Serge	3714	/* WaVSRefCountFullforceMissDisable */
		3715	if (IS_HASWELL(dev_priv->dev))
		3716	reg &= ~GEN7_FF_VS_REF_CNT_FFME;
		3717
3031	serge	3718	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		3719	}
		3720
3243	Serge	3721	static void lpt_init_clock_gating(struct drm_device *dev)
		3722	{
		3723	struct drm_i915_private *dev_priv = dev->dev_private;
		3724
		3725	/*
		3726	* TODO: this bit should only be enabled when really needed, then
		3727	* disabled when not needed anymore in order to save power.
		3728	*/
		3729	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
		3730	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		3731	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		3732	PCH_LP_PARTITION_LEVEL_DISABLE);
		3733	}
		3734
3031	serge	3735	static void haswell_init_clock_gating(struct drm_device *dev)
		3736	{
		3737	struct drm_i915_private *dev_priv = dev->dev_private;
		3738	int pipe;
		3739
		3740	I915_WRITE(WM3_LP_ILK, 0);
		3741	I915_WRITE(WM2_LP_ILK, 0);
		3742	I915_WRITE(WM1_LP_ILK, 0);
		3743
		3744	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3745	* This implements the WaDisableRCZUnitClockGating workaround.
		3746	*/
		3747	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
		3748
		3749	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3750	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3751	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3752
		3753	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3754	I915_WRITE(GEN7_L3CNTLREG1,
		3755	GEN7_WA_FOR_GEN7_L3_CONTROL);
		3756	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		3757	GEN7_WA_L3_CHICKEN_MODE);
		3758
		3759	/* This is required by WaCatErrorRejectionIssue */
		3760	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3761	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3762	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3763
		3764	for_each_pipe(pipe) {
		3765	I915_WRITE(DSPCNTR(pipe),
		3766	I915_READ(DSPCNTR(pipe)) \|
		3767	DISPPLANE_TRICKLE_FEED_DISABLE);
		3768	intel_flush_display_plane(dev_priv, pipe);
		3769	}
		3770
		3771	gen7_setup_fixed_func_scheduler(dev_priv);
		3772
		3773	/* WaDisable4x2SubspanOptimization */
		3774	I915_WRITE(CACHE_MODE_1,
		3775	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3776
3243	Serge	3777	/* WaMbcDriverBootEnable */
		3778	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3779	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3780
3031	serge	3781	/* XXX: This is a workaround for early silicon revisions and should be
		3782	* removed later.
		3783	*/
		3784	I915_WRITE(WM_DBG,
		3785	I915_READ(WM_DBG) \|
		3786	WM_DBG_DISALLOW_MULTIPLE_LP \|
		3787	WM_DBG_DISALLOW_SPRITE \|
		3788	WM_DBG_DISALLOW_MAXFIFO);
		3789
3243	Serge	3790	lpt_init_clock_gating(dev);
3031	serge	3791	}
		3792
		3793	static void ivybridge_init_clock_gating(struct drm_device *dev)
		3794	{
		3795	struct drm_i915_private *dev_priv = dev->dev_private;
		3796	int pipe;
		3797	uint32_t snpcr;
		3798
		3799	I915_WRITE(WM3_LP_ILK, 0);
		3800	I915_WRITE(WM2_LP_ILK, 0);
		3801	I915_WRITE(WM1_LP_ILK, 0);
		3802
3243	Serge	3803	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	3804
3243	Serge	3805	/* WaDisableEarlyCull */
		3806	I915_WRITE(_3D_CHICKEN3,
		3807	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		3808
		3809	/* WaDisableBackToBackFlipFix */
3031	serge	3810	I915_WRITE(IVB_CHICKEN3,
		3811	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3812	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3813
3243	Serge	3814	/* WaDisablePSDDualDispatchEnable */
		3815	if (IS_IVB_GT1(dev))
		3816	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		3817	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3818	else
		3819	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
		3820	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3821
3031	serge	3822	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3823	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3824	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3825
		3826	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3827	I915_WRITE(GEN7_L3CNTLREG1,
		3828	GEN7_WA_FOR_GEN7_L3_CONTROL);
		3829	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		3830	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	3831	if (IS_IVB_GT1(dev))
		3832	I915_WRITE(GEN7_ROW_CHICKEN2,
		3833	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		3834	else
		3835	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		3836	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3031	serge	3837
3243	Serge	3838
		3839	/* WaForceL3Serialization */
		3840	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3841	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3842
3031	serge	3843	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3844	* gating disable must be set. Failure to set it results in
		3845	* flickering pixels due to Z write ordering failures after
		3846	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3847	* Sanctuary and Tropics, and apparently anything else with
		3848	* alpha test or pixel discard.
		3849	*
		3850	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3851	* but we didn't debug actual testcases to find it out.
		3852	*
		3853	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3854	* This implements the WaDisableRCZUnitClockGating workaround.
		3855	*/
		3856	I915_WRITE(GEN6_UCGCTL2,
		3857	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		3858	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3859
		3860	/* This is required by WaCatErrorRejectionIssue */
		3861	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3862	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3863	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3864
		3865	for_each_pipe(pipe) {
		3866	I915_WRITE(DSPCNTR(pipe),
		3867	I915_READ(DSPCNTR(pipe)) \|
		3868	DISPPLANE_TRICKLE_FEED_DISABLE);
		3869	intel_flush_display_plane(dev_priv, pipe);
		3870	}
		3871
3243	Serge	3872	/* WaMbcDriverBootEnable */
3031	serge	3873	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3874	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3875
		3876	gen7_setup_fixed_func_scheduler(dev_priv);
		3877
		3878	/* WaDisable4x2SubspanOptimization */
		3879	I915_WRITE(CACHE_MODE_1,
		3880	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3881
		3882	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		3883	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		3884	snpcr \|= GEN6_MBC_SNPCR_MED;
		3885	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	3886
		3887	cpt_init_clock_gating(dev);
3480	Serge	3888
		3889	gen6_check_mch_setup(dev);
3031	serge	3890	}
		3891
		3892	static void valleyview_init_clock_gating(struct drm_device *dev)
		3893	{
		3894	struct drm_i915_private *dev_priv = dev->dev_private;
		3895	int pipe;
		3896
		3897	I915_WRITE(WM3_LP_ILK, 0);
		3898	I915_WRITE(WM2_LP_ILK, 0);
		3899	I915_WRITE(WM1_LP_ILK, 0);
		3900
3243	Serge	3901	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	3902
3243	Serge	3903	/* WaDisableEarlyCull */
		3904	I915_WRITE(_3D_CHICKEN3,
		3905	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		3906
		3907	/* WaDisableBackToBackFlipFix */
3031	serge	3908	I915_WRITE(IVB_CHICKEN3,
		3909	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3910	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3911
3243	Serge	3912	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		3913	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3914
3031	serge	3915	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3916	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3917	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3918
		3919	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
3243	Serge	3920	I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) \| GEN7_L3AGDIS);
3031	serge	3921	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
		3922
3243	Serge	3923	/* WaForceL3Serialization */
		3924	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3925	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3926
		3927	/* WaDisableDopClockGating */
		3928	I915_WRITE(GEN7_ROW_CHICKEN2,
		3929	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		3930
		3931	/* WaForceL3Serialization */
		3932	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3933	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3934
3031	serge	3935	/* This is required by WaCatErrorRejectionIssue */
		3936	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3937	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3938	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3939
3243	Serge	3940	/* WaMbcDriverBootEnable */
3031	serge	3941	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3942	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3943
		3944
		3945	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3946	* gating disable must be set. Failure to set it results in
		3947	* flickering pixels due to Z write ordering failures after
		3948	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3949	* Sanctuary and Tropics, and apparently anything else with
		3950	* alpha test or pixel discard.
		3951	*
		3952	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3953	* but we didn't debug actual testcases to find it out.
		3954	*
		3955	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3956	* This implements the WaDisableRCZUnitClockGating workaround.
		3957	*
		3958	* Also apply WaDisableVDSUnitClockGating and
		3959	* WaDisableRCPBUnitClockGating.
		3960	*/
		3961	I915_WRITE(GEN6_UCGCTL2,
		3962	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		3963	GEN7_TDLUNIT_CLOCK_GATE_DISABLE \|
		3964	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		3965	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		3966	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3967
		3968	I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
		3969
		3970	for_each_pipe(pipe) {
		3971	I915_WRITE(DSPCNTR(pipe),
		3972	I915_READ(DSPCNTR(pipe)) \|
		3973	DISPPLANE_TRICKLE_FEED_DISABLE);
		3974	intel_flush_display_plane(dev_priv, pipe);
		3975	}
		3976
		3977	I915_WRITE(CACHE_MODE_1,
		3978	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3979
		3980	/*
		3981	* On ValleyView, the GUnit needs to signal the GT
		3982	* when flip and other events complete. So enable
		3983	* all the GUnit->GT interrupts here
		3984	*/
		3985	I915_WRITE(VLV_DPFLIPSTAT, PIPEB_LINE_COMPARE_INT_EN \|
		3986	PIPEB_HLINE_INT_EN \| PIPEB_VBLANK_INT_EN \|
		3987	SPRITED_FLIPDONE_INT_EN \| SPRITEC_FLIPDONE_INT_EN \|
		3988	PLANEB_FLIPDONE_INT_EN \| PIPEA_LINE_COMPARE_INT_EN \|
		3989	PIPEA_HLINE_INT_EN \| PIPEA_VBLANK_INT_EN \|
		3990	SPRITEB_FLIPDONE_INT_EN \| SPRITEA_FLIPDONE_INT_EN \|
		3991	PLANEA_FLIPDONE_INT_EN);
3243	Serge	3992
		3993	/*
		3994	* WaDisableVLVClockGating_VBIIssue
		3995	* Disable clock gating on th GCFG unit to prevent a delay
		3996	* in the reporting of vblank events.
		3997	*/
		3998	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
3031	serge	3999	}
		4000
		4001	static void g4x_init_clock_gating(struct drm_device *dev)
		4002	{
		4003	struct drm_i915_private *dev_priv = dev->dev_private;
		4004	uint32_t dspclk_gate;
		4005
		4006	I915_WRITE(RENCLK_GATE_D1, 0);
		4007	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		4008	GS_UNIT_CLOCK_GATE_DISABLE \|
		4009	CL_UNIT_CLOCK_GATE_DISABLE);
		4010	I915_WRITE(RAMCLK_GATE_D, 0);
		4011	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		4012	OVRUNIT_CLOCK_GATE_DISABLE \|
		4013	OVCUNIT_CLOCK_GATE_DISABLE;
		4014	if (IS_GM45(dev))
		4015	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		4016	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	4017
		4018	/* WaDisableRenderCachePipelinedFlush */
		4019	I915_WRITE(CACHE_MODE_0,
		4020	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
3031	serge	4021	}
		4022
		4023	static void crestline_init_clock_gating(struct drm_device *dev)
		4024	{
		4025	struct drm_i915_private *dev_priv = dev->dev_private;
		4026
		4027	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		4028	I915_WRITE(RENCLK_GATE_D2, 0);
		4029	I915_WRITE(DSPCLK_GATE_D, 0);
		4030	I915_WRITE(RAMCLK_GATE_D, 0);
		4031	I915_WRITE16(DEUC, 0);
		4032	}
		4033
		4034	static void broadwater_init_clock_gating(struct drm_device *dev)
		4035	{
		4036	struct drm_i915_private *dev_priv = dev->dev_private;
		4037
		4038	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		4039	I965_RCC_CLOCK_GATE_DISABLE \|
		4040	I965_RCPB_CLOCK_GATE_DISABLE \|
		4041	I965_ISC_CLOCK_GATE_DISABLE \|
		4042	I965_FBC_CLOCK_GATE_DISABLE);
		4043	I915_WRITE(RENCLK_GATE_D2, 0);
		4044	}
		4045
		4046	static void gen3_init_clock_gating(struct drm_device *dev)
		4047	{
		4048	struct drm_i915_private *dev_priv = dev->dev_private;
		4049	u32 dstate = I915_READ(D_STATE);
		4050
		4051	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		4052	DSTATE_DOT_CLOCK_GATING;
		4053	I915_WRITE(D_STATE, dstate);
		4054
		4055	if (IS_PINEVIEW(dev))
		4056	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		4057
		4058	/* IIR "flip pending" means done if this bit is set */
		4059	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
		4060	}
		4061
		4062	static void i85x_init_clock_gating(struct drm_device *dev)
		4063	{
		4064	struct drm_i915_private *dev_priv = dev->dev_private;
		4065
		4066	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
		4067	}
		4068
		4069	static void i830_init_clock_gating(struct drm_device *dev)
		4070	{
		4071	struct drm_i915_private *dev_priv = dev->dev_private;
		4072
		4073	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
		4074	}
		4075
		4076	void intel_init_clock_gating(struct drm_device *dev)
		4077	{
		4078	struct drm_i915_private *dev_priv = dev->dev_private;
		4079
		4080	dev_priv->display.init_clock_gating(dev);
		4081	}
		4082
3480	Serge	4083	void intel_set_power_well(struct drm_device *dev, bool enable)
3031	serge	4084	{
		4085	struct drm_i915_private *dev_priv = dev->dev_private;
3480	Serge	4086	bool is_enabled, enable_requested;
		4087	uint32_t tmp;
3031	serge	4088
		4089	if (!IS_HASWELL(dev))
		4090	return;
		4091
3480	Serge	4092	tmp = I915_READ(HSW_PWR_WELL_DRIVER);
		4093	is_enabled = tmp & HSW_PWR_WELL_STATE;
		4094	enable_requested = tmp & HSW_PWR_WELL_ENABLE;
3031	serge	4095
3480	Serge	4096	if (enable) {
		4097	if (!enable_requested)
		4098	I915_WRITE(HSW_PWR_WELL_DRIVER, HSW_PWR_WELL_ENABLE);
3031	serge	4099
3480	Serge	4100	if (!is_enabled) {
		4101	DRM_DEBUG_KMS("Enabling power well\n");
		4102	if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
		4103	HSW_PWR_WELL_STATE), 20))
		4104	DRM_ERROR("Timeout enabling power well\n");
3031	serge	4105	}
3480	Serge	4106	} else {
		4107	if (enable_requested) {
		4108	I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
		4109	DRM_DEBUG_KMS("Requesting to disable the power well\n");
3031	serge	4110	}
3480	Serge	4111	}
		4112	}
3031	serge	4113
3480	Serge	4114	/*
		4115	* Starting with Haswell, we have a "Power Down Well" that can be turned off
		4116	* when not needed anymore. We have 4 registers that can request the power well
		4117	* to be enabled, and it will only be disabled if none of the registers is
		4118	* requesting it to be enabled.
		4119	*/
		4120	void intel_init_power_well(struct drm_device *dev)
		4121	{
		4122	struct drm_i915_private *dev_priv = dev->dev_private;
		4123
		4124	if (!IS_HASWELL(dev))
		4125	return;
		4126
		4127	/* For now, we need the power well to be always enabled. */
		4128	intel_set_power_well(dev, true);
		4129
		4130	/* We're taking over the BIOS, so clear any requests made by it since
		4131	* the driver is in charge now. */
		4132	if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE)
		4133	I915_WRITE(HSW_PWR_WELL_BIOS, 0);
3031	serge	4134	}
		4135
		4136	/* Set up chip specific power management-related functions */
		4137	void intel_init_pm(struct drm_device *dev)
		4138	{
		4139	struct drm_i915_private *dev_priv = dev->dev_private;
		4140
		4141	if (I915_HAS_FBC(dev)) {
		4142	if (HAS_PCH_SPLIT(dev)) {
		4143	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
		4144	dev_priv->display.enable_fbc = ironlake_enable_fbc;
		4145	dev_priv->display.disable_fbc = ironlake_disable_fbc;
		4146	} else if (IS_GM45(dev)) {
		4147	dev_priv->display.fbc_enabled = g4x_fbc_enabled;
		4148	dev_priv->display.enable_fbc = g4x_enable_fbc;
		4149	dev_priv->display.disable_fbc = g4x_disable_fbc;
		4150	} else if (IS_CRESTLINE(dev)) {
		4151	dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
		4152	dev_priv->display.enable_fbc = i8xx_enable_fbc;
		4153	dev_priv->display.disable_fbc = i8xx_disable_fbc;
		4154	}
		4155	/* 855GM needs testing */
		4156	}
		4157
		4158	/* For cxsr */
		4159	if (IS_PINEVIEW(dev))
		4160	i915_pineview_get_mem_freq(dev);
		4161	else if (IS_GEN5(dev))
		4162	i915_ironlake_get_mem_freq(dev);
		4163
		4164	/* For FIFO watermark updates */
		4165	if (HAS_PCH_SPLIT(dev)) {
		4166	if (IS_GEN5(dev)) {
		4167	if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
		4168	dev_priv->display.update_wm = ironlake_update_wm;
		4169	else {
		4170	DRM_DEBUG_KMS("Failed to get proper latency. "
		4171	"Disable CxSR\n");
		4172	dev_priv->display.update_wm = NULL;
		4173	}
		4174	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		4175	} else if (IS_GEN6(dev)) {
		4176	if (SNB_READ_WM0_LATENCY()) {
		4177	dev_priv->display.update_wm = sandybridge_update_wm;
		4178	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4179	} else {
		4180	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4181	"Disable CxSR\n");
		4182	dev_priv->display.update_wm = NULL;
		4183	}
		4184	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
		4185	} else if (IS_IVYBRIDGE(dev)) {
		4186	/* FIXME: detect B0+ stepping and use auto training */
		4187	if (SNB_READ_WM0_LATENCY()) {
3243	Serge	4188	dev_priv->display.update_wm = ivybridge_update_wm;
3031	serge	4189	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4190	} else {
		4191	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4192	"Disable CxSR\n");
		4193	dev_priv->display.update_wm = NULL;
		4194	}
		4195	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
		4196	} else if (IS_HASWELL(dev)) {
		4197	if (SNB_READ_WM0_LATENCY()) {
		4198	dev_priv->display.update_wm = sandybridge_update_wm;
		4199	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4200	dev_priv->display.update_linetime_wm = haswell_update_linetime_wm;
		4201	} else {
		4202	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4203	"Disable CxSR\n");
		4204	dev_priv->display.update_wm = NULL;
		4205	}
		4206	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
		4207	} else
		4208	dev_priv->display.update_wm = NULL;
		4209	} else if (IS_VALLEYVIEW(dev)) {
		4210	dev_priv->display.update_wm = valleyview_update_wm;
		4211	dev_priv->display.init_clock_gating =
		4212	valleyview_init_clock_gating;
		4213	} else if (IS_PINEVIEW(dev)) {
		4214	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		4215	dev_priv->is_ddr3,
		4216	dev_priv->fsb_freq,
		4217	dev_priv->mem_freq)) {
		4218	DRM_INFO("failed to find known CxSR latency "
		4219	"(found ddr%s fsb freq %d, mem freq %d), "
		4220	"disabling CxSR\n",
		4221	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		4222	dev_priv->fsb_freq, dev_priv->mem_freq);
		4223	/* Disable CxSR and never update its watermark again */
		4224	pineview_disable_cxsr(dev);
		4225	dev_priv->display.update_wm = NULL;
		4226	} else
		4227	dev_priv->display.update_wm = pineview_update_wm;
		4228	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		4229	} else if (IS_G4X(dev)) {
		4230	dev_priv->display.update_wm = g4x_update_wm;
		4231	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		4232	} else if (IS_GEN4(dev)) {
		4233	dev_priv->display.update_wm = i965_update_wm;
		4234	if (IS_CRESTLINE(dev))
		4235	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		4236	else if (IS_BROADWATER(dev))
		4237	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		4238	} else if (IS_GEN3(dev)) {
		4239	dev_priv->display.update_wm = i9xx_update_wm;
		4240	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		4241	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		4242	} else if (IS_I865G(dev)) {
		4243	dev_priv->display.update_wm = i830_update_wm;
		4244	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		4245	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		4246	} else if (IS_I85X(dev)) {
		4247	dev_priv->display.update_wm = i9xx_update_wm;
		4248	dev_priv->display.get_fifo_size = i85x_get_fifo_size;
		4249	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		4250	} else {
		4251	dev_priv->display.update_wm = i830_update_wm;
		4252	dev_priv->display.init_clock_gating = i830_init_clock_gating;
		4253	if (IS_845G(dev))
		4254	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		4255	else
		4256	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		4257	}
		4258	}
		4259
		4260	static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
		4261	{
		4262	u32 gt_thread_status_mask;
		4263
		4264	if (IS_HASWELL(dev_priv->dev))
		4265	gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
		4266	else
		4267	gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
		4268
		4269	/* w/a for a sporadic read returning 0 by waiting for the GT
		4270	* thread to wake up.
		4271	*/
		4272	if (wait_for_atomic_us((I915_READ_NOTRACE(GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
		4273	DRM_ERROR("GT thread status wait timed out\n");
		4274	}
		4275
3243	Serge	4276	static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
		4277	{
		4278	I915_WRITE_NOTRACE(FORCEWAKE, 0);
		4279	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
		4280	}
		4281
3031	serge	4282	static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
		4283	{
		4284	u32 forcewake_ack;
		4285
		4286	if (IS_HASWELL(dev_priv->dev))
		4287	forcewake_ack = FORCEWAKE_ACK_HSW;
		4288	else
		4289	forcewake_ack = FORCEWAKE_ACK;
		4290
		4291	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1) == 0,
		4292	FORCEWAKE_ACK_TIMEOUT_MS))
		4293	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4294
3243	Serge	4295	I915_WRITE_NOTRACE(FORCEWAKE, FORCEWAKE_KERNEL);
3031	serge	4296	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
		4297
		4298	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
		4299	FORCEWAKE_ACK_TIMEOUT_MS))
		4300	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4301
		4302	__gen6_gt_wait_for_thread_c0(dev_priv);
		4303	}
		4304
3243	Serge	4305	static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
		4306	{
		4307	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
		4308	/* something from same cacheline, but !FORCEWAKE_MT */
		4309	POSTING_READ(ECOBUS);
		4310	}
		4311
3031	serge	4312	static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
		4313	{
		4314	u32 forcewake_ack;
		4315
		4316	if (IS_HASWELL(dev_priv->dev))
		4317	forcewake_ack = FORCEWAKE_ACK_HSW;
		4318	else
		4319	forcewake_ack = FORCEWAKE_MT_ACK;
		4320
		4321	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1) == 0,
		4322	FORCEWAKE_ACK_TIMEOUT_MS))
		4323	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4324
3243	Serge	4325	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
		4326	/* something from same cacheline, but !FORCEWAKE_MT */
		4327	POSTING_READ(ECOBUS);
3031	serge	4328
		4329	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
		4330	FORCEWAKE_ACK_TIMEOUT_MS))
		4331	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4332
		4333	__gen6_gt_wait_for_thread_c0(dev_priv);
		4334	}
		4335
		4336	/*
		4337	* Generally this is called implicitly by the register read function. However,
		4338	* if some sequence requires the GT to not power down then this function should
		4339	* be called at the beginning of the sequence followed by a call to
		4340	* gen6_gt_force_wake_put() at the end of the sequence.
		4341	*/
		4342	void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
		4343	{
		4344	unsigned long irqflags;
		4345
		4346	spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
		4347	if (dev_priv->forcewake_count++ == 0)
		4348	dev_priv->gt.force_wake_get(dev_priv);
		4349	spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
		4350	}
		4351
		4352	void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
		4353	{
		4354	u32 gtfifodbg;
		4355	gtfifodbg = I915_READ_NOTRACE(GTFIFODBG);
		4356	if (WARN(gtfifodbg & GT_FIFO_CPU_ERROR_MASK,
		4357	"MMIO read or write has been dropped %x\n", gtfifodbg))
		4358	I915_WRITE_NOTRACE(GTFIFODBG, GT_FIFO_CPU_ERROR_MASK);
		4359	}
		4360
		4361	static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
		4362	{
		4363	I915_WRITE_NOTRACE(FORCEWAKE, 0);
3243	Serge	4364	/* something from same cacheline, but !FORCEWAKE */
		4365	POSTING_READ(ECOBUS);
3031	serge	4366	gen6_gt_check_fifodbg(dev_priv);
		4367	}
		4368
		4369	static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
		4370	{
3243	Serge	4371	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
		4372	/* something from same cacheline, but !FORCEWAKE_MT */
		4373	POSTING_READ(ECOBUS);
3031	serge	4374	gen6_gt_check_fifodbg(dev_priv);
		4375	}
		4376
		4377	/*
		4378	* see gen6_gt_force_wake_get()
		4379	*/
		4380	void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
		4381	{
		4382	unsigned long irqflags;
		4383
		4384	spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
		4385	if (--dev_priv->forcewake_count == 0)
		4386	dev_priv->gt.force_wake_put(dev_priv);
		4387	spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
		4388	}
		4389
		4390	int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
		4391	{
		4392	int ret = 0;
		4393
		4394	if (dev_priv->gt_fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
		4395	int loop = 500;
		4396	u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
		4397	while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
		4398	udelay(10);
		4399	fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
		4400	}
		4401	if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
		4402	++ret;
		4403	dev_priv->gt_fifo_count = fifo;
		4404	}
		4405	dev_priv->gt_fifo_count--;
		4406
		4407	return ret;
		4408	}
		4409
3243	Serge	4410	static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
		4411	{
		4412	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(0xffff));
		4413	/* something from same cacheline, but !FORCEWAKE_VLV */
		4414	POSTING_READ(FORCEWAKE_ACK_VLV);
		4415	}
		4416
3031	serge	4417	static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
		4418	{
		4419	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1) == 0,
		4420	FORCEWAKE_ACK_TIMEOUT_MS))
		4421	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4422
3243	Serge	4423	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
3031	serge	4424
		4425	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1),
		4426	FORCEWAKE_ACK_TIMEOUT_MS))
		4427	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4428
		4429	__gen6_gt_wait_for_thread_c0(dev_priv);
		4430	}
		4431
		4432	static void vlv_force_wake_put(struct drm_i915_private *dev_priv)
		4433	{
3243	Serge	4434	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
		4435	/* something from same cacheline, but !FORCEWAKE_VLV */
		4436	POSTING_READ(FORCEWAKE_ACK_VLV);
3031	serge	4437	gen6_gt_check_fifodbg(dev_priv);
		4438	}
		4439
3243	Serge	4440	void intel_gt_reset(struct drm_device *dev)
		4441	{
		4442	struct drm_i915_private *dev_priv = dev->dev_private;
		4443
		4444	if (IS_VALLEYVIEW(dev)) {
		4445	vlv_force_wake_reset(dev_priv);
		4446	} else if (INTEL_INFO(dev)->gen >= 6) {
		4447	__gen6_gt_force_wake_reset(dev_priv);
		4448	if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev))
		4449	__gen6_gt_force_wake_mt_reset(dev_priv);
		4450	}
		4451	}
		4452
3031	serge	4453	void intel_gt_init(struct drm_device *dev)
		4454	{
		4455	struct drm_i915_private *dev_priv = dev->dev_private;
		4456
		4457	spin_lock_init(&dev_priv->gt_lock);
		4458
3243	Serge	4459	intel_gt_reset(dev);
		4460
3031	serge	4461	if (IS_VALLEYVIEW(dev)) {
		4462	dev_priv->gt.force_wake_get = vlv_force_wake_get;
		4463	dev_priv->gt.force_wake_put = vlv_force_wake_put;
3243	Serge	4464	} else if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev)) {
		4465	dev_priv->gt.force_wake_get = __gen6_gt_force_wake_mt_get;
		4466	dev_priv->gt.force_wake_put = __gen6_gt_force_wake_mt_put;
		4467	} else if (IS_GEN6(dev)) {
3031	serge	4468	dev_priv->gt.force_wake_get = __gen6_gt_force_wake_get;
		4469	dev_priv->gt.force_wake_put = __gen6_gt_force_wake_put;
3243	Serge	4470	}
		4471	}
3031	serge	4472
3243	Serge	4473	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u8 mbox, u32 val)
		4474	{
		4475	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4476
3243	Serge	4477	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		4478	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		4479	return -EAGAIN;
		4480	}
3031	serge	4481
3243	Serge	4482	I915_WRITE(GEN6_PCODE_DATA, *val);
		4483	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		4484
		4485	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		4486	500)) {
		4487	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		4488	return -ETIMEDOUT;
3031	serge	4489	}
3243	Serge	4490
		4491	*val = I915_READ(GEN6_PCODE_DATA);
		4492	I915_WRITE(GEN6_PCODE_DATA, 0);
		4493
		4494	return 0;
		4495	}
		4496
		4497	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
		4498	{
		4499	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4500
		4501	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		4502	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		4503	return -EAGAIN;
3031	serge	4504	}
3243	Serge	4505
		4506	I915_WRITE(GEN6_PCODE_DATA, val);
		4507	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		4508
		4509	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		4510	500)) {
		4511	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		4512	return -ETIMEDOUT;
3031	serge	4513	}
3243	Serge	4514
		4515	I915_WRITE(GEN6_PCODE_DATA, 0);
		4516
		4517	return 0;
3031	serge	4518	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 3480