WebSVN – Kolibri OS – Blame – /drivers/video/drm/i915/intel_pm.c

Rev	Author	Line No.	Line
3031	serge	1	/*
		2	* Copyright © 2012 Intel Corporation
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice (including the next
		12	* paragraph) shall be included in all copies or substantial portions of the
		13	* Software.
		14	*
		15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		18	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
		19	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		20	* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
		21	* IN THE SOFTWARE.
		22	*
		23	* Authors:
		24	* Eugeni Dodonov
		25	*
		26	*/
		27
		28	#define iowrite32(v, addr) writel((v), (addr))
		29	#define ioread32(addr) readl(addr)
		30
		31	//#include
		32	#include "i915_drv.h"
		33	#include "intel_drv.h"
		34	#include
		35	//#include "../../../platform/x86/intel_ips.h"
		36	#include
		37
		38	#define FORCEWAKE_ACK_TIMEOUT_MS 2
		39
		40	#define assert_spin_locked(x)
		41
		42	void getrawmonotonic(struct timespec *ts);
		43	void set_normalized_timespec(struct timespec *ts, time_t sec, long nsec);
		44
		45	static inline struct timespec timespec_sub(struct timespec lhs,
		46	struct timespec rhs)
		47	{
		48	struct timespec ts_delta;
		49	set_normalized_timespec(&ts_delta, lhs.tv_sec - rhs.tv_sec,
		50	lhs.tv_nsec - rhs.tv_nsec);
		51	return ts_delta;
		52	}
		53
		54
		55	/* FBC, or Frame Buffer Compression, is a technique employed to compress the
		56	* framebuffer contents in-memory, aiming at reducing the required bandwidth
		57	* during in-memory transfers and, therefore, reduce the power packet.
		58	*
		59	* The benefits of FBC are mostly visible with solid backgrounds and
		60	* variation-less patterns.
		61	*
		62	* FBC-related functionality can be enabled by the means of the
		63	* i915.i915_enable_fbc parameter
		64	*/
		65
3243	Serge	66	static bool intel_crtc_active(struct drm_crtc *crtc)
		67	{
		68	/* Be paranoid as we can arrive here with only partial
		69	* state retrieved from the hardware during setup.
		70	*/
		71	return to_intel_crtc(crtc)->active && crtc->fb && crtc->mode.clock;
		72	}
		73
3031	serge	74	static void i8xx_disable_fbc(struct drm_device *dev)
		75	{
		76	struct drm_i915_private *dev_priv = dev->dev_private;
		77	u32 fbc_ctl;
		78
		79	/* Disable compression */
		80	fbc_ctl = I915_READ(FBC_CONTROL);
		81	if ((fbc_ctl & FBC_CTL_EN) == 0)
		82	return;
		83
		84	fbc_ctl &= ~FBC_CTL_EN;
		85	I915_WRITE(FBC_CONTROL, fbc_ctl);
		86
		87	/* Wait for compressing bit to clear */
		88	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
		89	DRM_DEBUG_KMS("FBC idle timed out\n");
		90	return;
		91	}
		92
		93	DRM_DEBUG_KMS("disabled FBC\n");
		94	}
		95
		96	static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		97	{
		98	struct drm_device *dev = crtc->dev;
		99	struct drm_i915_private *dev_priv = dev->dev_private;
		100	struct drm_framebuffer *fb = crtc->fb;
		101	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		102	struct drm_i915_gem_object *obj = intel_fb->obj;
		103	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		104	int cfb_pitch;
		105	int plane, i;
		106	u32 fbc_ctl, fbc_ctl2;
		107
		108	cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
		109	if (fb->pitches[0] < cfb_pitch)
		110	cfb_pitch = fb->pitches[0];
		111
		112	/* FBC_CTL wants 64B units */
		113	cfb_pitch = (cfb_pitch / 64) - 1;
		114	plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
		115
		116	/* Clear old tags */
		117	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
		118	I915_WRITE(FBC_TAG + (i * 4), 0);
		119
		120	/* Set it up... */
		121	fbc_ctl2 = FBC_CTL_FENCE_DBL \| FBC_CTL_IDLE_IMM \| FBC_CTL_CPU_FENCE;
		122	fbc_ctl2 \|= plane;
		123	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
		124	I915_WRITE(FBC_FENCE_OFF, crtc->y);
		125
		126	/* enable it... */
		127	fbc_ctl = FBC_CTL_EN \| FBC_CTL_PERIODIC;
		128	if (IS_I945GM(dev))
		129	fbc_ctl \|= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
		130	fbc_ctl \|= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
		131	fbc_ctl \|= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
		132	fbc_ctl \|= obj->fence_reg;
		133	I915_WRITE(FBC_CONTROL, fbc_ctl);
		134
		135	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
		136	cfb_pitch, crtc->y, intel_crtc->plane);
		137	}
		138
		139	static bool i8xx_fbc_enabled(struct drm_device *dev)
		140	{
		141	struct drm_i915_private *dev_priv = dev->dev_private;
		142
		143	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
		144	}
		145
		146	static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		147	{
		148	struct drm_device *dev = crtc->dev;
		149	struct drm_i915_private *dev_priv = dev->dev_private;
		150	struct drm_framebuffer *fb = crtc->fb;
		151	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		152	struct drm_i915_gem_object *obj = intel_fb->obj;
		153	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		154	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		155	unsigned long stall_watermark = 200;
		156	u32 dpfc_ctl;
		157
		158	dpfc_ctl = plane \| DPFC_SR_EN \| DPFC_CTL_LIMIT_1X;
		159	dpfc_ctl \|= DPFC_CTL_FENCE_EN \| obj->fence_reg;
		160	I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
		161
		162	I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		163	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		164	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		165	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
		166
		167	/* enable it... */
		168	I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) \| DPFC_CTL_EN);
		169
		170	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		171	}
		172
		173	static void g4x_disable_fbc(struct drm_device *dev)
		174	{
		175	struct drm_i915_private *dev_priv = dev->dev_private;
		176	u32 dpfc_ctl;
		177
		178	/* Disable compression */
		179	dpfc_ctl = I915_READ(DPFC_CONTROL);
		180	if (dpfc_ctl & DPFC_CTL_EN) {
		181	dpfc_ctl &= ~DPFC_CTL_EN;
		182	I915_WRITE(DPFC_CONTROL, dpfc_ctl);
		183
		184	DRM_DEBUG_KMS("disabled FBC\n");
		185	}
		186	}
		187
		188	static bool g4x_fbc_enabled(struct drm_device *dev)
		189	{
		190	struct drm_i915_private *dev_priv = dev->dev_private;
		191
		192	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
		193	}
		194
		195	static void sandybridge_blit_fbc_update(struct drm_device *dev)
		196	{
		197	struct drm_i915_private *dev_priv = dev->dev_private;
		198	u32 blt_ecoskpd;
		199
		200	/* Make sure blitter notifies FBC of writes */
		201	gen6_gt_force_wake_get(dev_priv);
		202	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
		203	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY <<
		204	GEN6_BLITTER_LOCK_SHIFT;
		205	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		206	blt_ecoskpd \|= GEN6_BLITTER_FBC_NOTIFY;
		207	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		208	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
		209	GEN6_BLITTER_LOCK_SHIFT);
		210	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
		211	POSTING_READ(GEN6_BLITTER_ECOSKPD);
		212	gen6_gt_force_wake_put(dev_priv);
		213	}
		214
		215	static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		216	{
		217	struct drm_device *dev = crtc->dev;
		218	struct drm_i915_private *dev_priv = dev->dev_private;
		219	struct drm_framebuffer *fb = crtc->fb;
		220	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
		221	struct drm_i915_gem_object *obj = intel_fb->obj;
		222	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
		223	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
		224	unsigned long stall_watermark = 200;
		225	u32 dpfc_ctl;
		226
		227	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		228	dpfc_ctl &= DPFC_RESERVED;
		229	dpfc_ctl \|= (plane \| DPFC_CTL_LIMIT_1X);
		230	/* Set persistent mode for front-buffer rendering, ala X. */
		231	dpfc_ctl \|= DPFC_CTL_PERSISTENT_MODE;
		232	dpfc_ctl \|= (DPFC_CTL_FENCE_EN \| obj->fence_reg);
		233	I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
		234
		235	I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN \|
		236	(stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) \|
		237	(interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
		238	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
		239	I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset \| ILK_FBC_RT_VALID);
		240	/* enable it... */
		241	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl \| DPFC_CTL_EN);
		242
		243	if (IS_GEN6(dev)) {
		244	I915_WRITE(SNB_DPFC_CTL_SA,
		245	SNB_CPU_FENCE_ENABLE \| obj->fence_reg);
		246	I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
		247	sandybridge_blit_fbc_update(dev);
		248	}
		249
		250	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
		251	}
		252
		253	static void ironlake_disable_fbc(struct drm_device *dev)
		254	{
		255	struct drm_i915_private *dev_priv = dev->dev_private;
		256	u32 dpfc_ctl;
		257
		258	/* Disable compression */
		259	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
		260	if (dpfc_ctl & DPFC_CTL_EN) {
		261	dpfc_ctl &= ~DPFC_CTL_EN;
		262	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
		263
		264	DRM_DEBUG_KMS("disabled FBC\n");
		265	}
		266	}
		267
		268	static bool ironlake_fbc_enabled(struct drm_device *dev)
		269	{
		270	struct drm_i915_private *dev_priv = dev->dev_private;
		271
		272	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
		273	}
		274
		275	bool intel_fbc_enabled(struct drm_device *dev)
		276	{
		277	struct drm_i915_private *dev_priv = dev->dev_private;
		278
		279	if (!dev_priv->display.fbc_enabled)
		280	return false;
		281
		282	return dev_priv->display.fbc_enabled(dev);
		283	}
		284
		285	#if 0
		286	static void intel_fbc_work_fn(struct work_struct *__work)
		287	{
		288	struct intel_fbc_work *work =
		289	container_of(to_delayed_work(__work),
		290	struct intel_fbc_work, work);
		291	struct drm_device *dev = work->crtc->dev;
		292	struct drm_i915_private *dev_priv = dev->dev_private;
		293
		294	mutex_lock(&dev->struct_mutex);
		295	if (work == dev_priv->fbc_work) {
		296	/* Double check that we haven't switched fb without cancelling
		297	* the prior work.
		298	*/
		299	if (work->crtc->fb == work->fb) {
		300	dev_priv->display.enable_fbc(work->crtc,
		301	work->interval);
		302
		303	dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
		304	dev_priv->cfb_fb = work->crtc->fb->base.id;
		305	dev_priv->cfb_y = work->crtc->y;
		306	}
		307
		308	dev_priv->fbc_work = NULL;
		309	}
		310	mutex_unlock(&dev->struct_mutex);
		311
		312	kfree(work);
		313	}
		314
		315	static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
		316	{
		317	if (dev_priv->fbc_work == NULL)
		318	return;
		319
		320	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
		321
		322	/* Synchronisation is provided by struct_mutex and checking of
		323	* dev_priv->fbc_work, so we can perform the cancellation
		324	* entirely asynchronously.
		325	*/
		326	if (cancel_delayed_work(&dev_priv->fbc_work->work))
		327	/* tasklet was killed before being run, clean up */
		328	kfree(dev_priv->fbc_work);
		329
		330	/* Mark the work as no longer wanted so that if it does
		331	* wake-up (because the work was already running and waiting
		332	* for our mutex), it will discover that is no longer
		333	* necessary to run.
		334	*/
		335	dev_priv->fbc_work = NULL;
		336	}
		337	#endif
		338
		339	void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
		340	{
		341	struct intel_fbc_work *work;
		342	struct drm_device *dev = crtc->dev;
		343	struct drm_i915_private *dev_priv = dev->dev_private;
		344
		345	// if (!dev_priv->display.enable_fbc)
		346	return;
		347	#if 0
		348	intel_cancel_fbc_work(dev_priv);
		349
		350	work = kzalloc(sizeof *work, GFP_KERNEL);
		351	if (work == NULL) {
		352	dev_priv->display.enable_fbc(crtc, interval);
		353	return;
		354	}
		355
		356	work->crtc = crtc;
		357	work->fb = crtc->fb;
		358	work->interval = interval;
		359	INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
		360
		361	dev_priv->fbc_work = work;
		362
		363	DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
		364
		365	/* Delay the actual enabling to let pageflipping cease and the
		366	* display to settle before starting the compression. Note that
		367	* this delay also serves a second purpose: it allows for a
		368	* vblank to pass after disabling the FBC before we attempt
		369	* to modify the control registers.
		370	*
		371	* A more complicated solution would involve tracking vblanks
		372	* following the termination of the page-flipping sequence
		373	* and indeed performing the enable as a co-routine and not
		374	* waiting synchronously upon the vblank.
		375	*/
		376	schedule_delayed_work(&work->work, msecs_to_jiffies(50));
		377	#endif
		378
		379	}
		380
		381	void intel_disable_fbc(struct drm_device *dev)
		382	{
		383	struct drm_i915_private *dev_priv = dev->dev_private;
		384
		385	// intel_cancel_fbc_work(dev_priv);
		386
		387	// if (!dev_priv->display.disable_fbc)
		388	// return;
		389
		390	// dev_priv->display.disable_fbc(dev);
		391	dev_priv->cfb_plane = -1;
		392	}
		393
		394	/**
		395	* intel_update_fbc - enable/disable FBC as needed
		396	* @dev: the drm_device
		397	*
		398	* Set up the framebuffer compression hardware at mode set time. We
		399	* enable it if possible:
		400	* - plane A only (on pre-965)
		401	* - no pixel mulitply/line duplication
		402	* - no alpha buffer discard
		403	* - no dual wide
		404	* - framebuffer <= 2048 in width, 1536 in height
		405	*
		406	* We can't assume that any compression will take place (worst case),
		407	* so the compressed buffer has to be the same size as the uncompressed
		408	* one. It also must reside (along with the line length buffer) in
		409	* stolen memory.
		410	*
		411	* We need to enable/disable FBC on a global basis.
		412	*/
		413	void intel_update_fbc(struct drm_device *dev)
		414	{
		415	struct drm_i915_private *dev_priv = dev->dev_private;
		416	struct drm_crtc crtc = NULL, tmp_crtc;
		417	struct intel_crtc *intel_crtc;
		418	struct drm_framebuffer *fb;
		419	struct intel_framebuffer *intel_fb;
		420	struct drm_i915_gem_object *obj;
		421	int enable_fbc;
		422
		423	if (!i915_powersave)
		424	return;
		425
		426	if (!I915_HAS_FBC(dev))
		427	return;
		428
		429	/*
		430	* If FBC is already on, we just have to verify that we can
		431	* keep it that way...
		432	* Need to disable if:
		433	* - more than one pipe is active
		434	* - changing FBC params (stride, fence, mode)
		435	* - new fb is too large to fit in compressed buffer
		436	* - going to an unsupported config (interlace, pixel multiply, etc.)
		437	*/
		438	list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	439	if (intel_crtc_active(tmp_crtc) &&
		440	!to_intel_crtc(tmp_crtc)->primary_disabled) {
3031	serge	441	if (crtc) {
		442	DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
		443	dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
		444	goto out_disable;
		445	}
		446	crtc = tmp_crtc;
		447	}
		448	}
		449
		450	if (!crtc \|\| crtc->fb == NULL) {
		451	DRM_DEBUG_KMS("no output, disabling\n");
		452	dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
		453	goto out_disable;
		454	}
		455
		456	intel_crtc = to_intel_crtc(crtc);
		457	fb = crtc->fb;
		458	intel_fb = to_intel_framebuffer(fb);
		459	obj = intel_fb->obj;
		460
		461	enable_fbc = i915_enable_fbc;
		462	if (enable_fbc < 0) {
		463	DRM_DEBUG_KMS("fbc set to per-chip default\n");
		464	enable_fbc = 1;
		465	if (INTEL_INFO(dev)->gen <= 6)
		466	enable_fbc = 0;
		467	}
		468	if (!enable_fbc) {
		469	DRM_DEBUG_KMS("fbc disabled per module param\n");
		470	dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
		471	goto out_disable;
		472	}
		473	if (intel_fb->obj->base.size > dev_priv->cfb_size) {
		474	DRM_DEBUG_KMS("framebuffer too large, disabling "
		475	"compression\n");
		476	dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
		477	goto out_disable;
		478	}
		479	if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) \|\|
		480	(crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
		481	DRM_DEBUG_KMS("mode incompatible with compression, "
		482	"disabling\n");
		483	dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
		484	goto out_disable;
		485	}
		486	if ((crtc->mode.hdisplay > 2048) \|\|
		487	(crtc->mode.vdisplay > 1536)) {
		488	DRM_DEBUG_KMS("mode too large for compression, disabling\n");
		489	dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
		490	goto out_disable;
		491	}
		492	if ((IS_I915GM(dev) \|\| IS_I945GM(dev)) && intel_crtc->plane != 0) {
		493	DRM_DEBUG_KMS("plane not 0, disabling compression\n");
		494	dev_priv->no_fbc_reason = FBC_BAD_PLANE;
		495	goto out_disable;
		496	}
		497
		498	/* The use of a CPU fence is mandatory in order to detect writes
		499	* by the CPU to the scanout and trigger updates to the FBC.
		500	*/
		501	if (obj->tiling_mode != I915_TILING_X \|\|
		502	obj->fence_reg == I915_FENCE_REG_NONE) {
		503	DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
		504	dev_priv->no_fbc_reason = FBC_NOT_TILED;
		505	goto out_disable;
		506	}
		507
		508	/* If the kernel debugger is active, always disable compression */
		509	if (in_dbg_master())
		510	goto out_disable;
		511
		512	/* If the scanout has not changed, don't modify the FBC settings.
		513	* Note that we make the fundamental assumption that the fb->obj
		514	* cannot be unpinned (and have its GTT offset and fence revoked)
		515	* without first being decoupled from the scanout and FBC disabled.
		516	*/
		517	if (dev_priv->cfb_plane == intel_crtc->plane &&
		518	dev_priv->cfb_fb == fb->base.id &&
		519	dev_priv->cfb_y == crtc->y)
		520	return;
		521
		522	if (intel_fbc_enabled(dev)) {
		523	/* We update FBC along two paths, after changing fb/crtc
		524	* configuration (modeswitching) and after page-flipping
		525	* finishes. For the latter, we know that not only did
		526	* we disable the FBC at the start of the page-flip
		527	* sequence, but also more than one vblank has passed.
		528	*
		529	* For the former case of modeswitching, it is possible
		530	* to switch between two FBC valid configurations
		531	* instantaneously so we do need to disable the FBC
		532	* before we can modify its control registers. We also
		533	* have to wait for the next vblank for that to take
		534	* effect. However, since we delay enabling FBC we can
		535	* assume that a vblank has passed since disabling and
		536	* that we can safely alter the registers in the deferred
		537	* callback.
		538	*
		539	* In the scenario that we go from a valid to invalid
		540	* and then back to valid FBC configuration we have
		541	* no strict enforcement that a vblank occurred since
		542	* disabling the FBC. However, along all current pipe
		543	* disabling paths we do need to wait for a vblank at
		544	* some point. And we wait before enabling FBC anyway.
		545	*/
		546	DRM_DEBUG_KMS("disabling active FBC for update\n");
		547	intel_disable_fbc(dev);
		548	}
		549
		550	intel_enable_fbc(crtc, 500);
		551	return;
		552
		553	out_disable:
		554	/* Multiple disables should be harmless */
		555	if (intel_fbc_enabled(dev)) {
		556	DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
		557	intel_disable_fbc(dev);
		558	}
		559	}
		560
		561	static void i915_pineview_get_mem_freq(struct drm_device *dev)
		562	{
		563	drm_i915_private_t *dev_priv = dev->dev_private;
		564	u32 tmp;
		565
		566	tmp = I915_READ(CLKCFG);
		567
		568	switch (tmp & CLKCFG_FSB_MASK) {
		569	case CLKCFG_FSB_533:
		570	dev_priv->fsb_freq = 533; /* 1334 /
		571	break;
		572	case CLKCFG_FSB_800:
		573	dev_priv->fsb_freq = 800; /* 2004 /
		574	break;
		575	case CLKCFG_FSB_667:
		576	dev_priv->fsb_freq = 667; /* 1674 /
		577	break;
		578	case CLKCFG_FSB_400:
		579	dev_priv->fsb_freq = 400; /* 1004 /
		580	break;
		581	}
		582
		583	switch (tmp & CLKCFG_MEM_MASK) {
		584	case CLKCFG_MEM_533:
		585	dev_priv->mem_freq = 533;
		586	break;
		587	case CLKCFG_MEM_667:
		588	dev_priv->mem_freq = 667;
		589	break;
		590	case CLKCFG_MEM_800:
		591	dev_priv->mem_freq = 800;
		592	break;
		593	}
		594
		595	/* detect pineview DDR3 setting */
		596	tmp = I915_READ(CSHRDDR3CTL);
		597	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
		598	}
		599
		600	static void i915_ironlake_get_mem_freq(struct drm_device *dev)
		601	{
		602	drm_i915_private_t *dev_priv = dev->dev_private;
		603	u16 ddrpll, csipll;
		604
		605	ddrpll = I915_READ16(DDRMPLL1);
		606	csipll = I915_READ16(CSIPLL0);
		607
		608	switch (ddrpll & 0xff) {
		609	case 0xc:
		610	dev_priv->mem_freq = 800;
		611	break;
		612	case 0x10:
		613	dev_priv->mem_freq = 1066;
		614	break;
		615	case 0x14:
		616	dev_priv->mem_freq = 1333;
		617	break;
		618	case 0x18:
		619	dev_priv->mem_freq = 1600;
		620	break;
		621	default:
		622	DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
		623	ddrpll & 0xff);
		624	dev_priv->mem_freq = 0;
		625	break;
		626	}
		627
		628	dev_priv->ips.r_t = dev_priv->mem_freq;
		629
		630	switch (csipll & 0x3ff) {
		631	case 0x00c:
		632	dev_priv->fsb_freq = 3200;
		633	break;
		634	case 0x00e:
		635	dev_priv->fsb_freq = 3733;
		636	break;
		637	case 0x010:
		638	dev_priv->fsb_freq = 4266;
		639	break;
		640	case 0x012:
		641	dev_priv->fsb_freq = 4800;
		642	break;
		643	case 0x014:
		644	dev_priv->fsb_freq = 5333;
		645	break;
		646	case 0x016:
		647	dev_priv->fsb_freq = 5866;
		648	break;
		649	case 0x018:
		650	dev_priv->fsb_freq = 6400;
		651	break;
		652	default:
		653	DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
		654	csipll & 0x3ff);
		655	dev_priv->fsb_freq = 0;
		656	break;
		657	}
		658
		659	if (dev_priv->fsb_freq == 3200) {
		660	dev_priv->ips.c_m = 0;
		661	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
		662	dev_priv->ips.c_m = 1;
		663	} else {
		664	dev_priv->ips.c_m = 2;
		665	}
		666	}
		667
		668	static const struct cxsr_latency cxsr_latency_table[] = {
		669	{1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
		670	{1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
		671	{1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
		672	{1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
		673	{1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
		674
		675	{1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
		676	{1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
		677	{1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
		678	{1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
		679	{1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
		680
		681	{1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
		682	{1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
		683	{1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
		684	{1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
		685	{1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
		686
		687	{0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
		688	{0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
		689	{0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
		690	{0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
		691	{0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
		692
		693	{0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
		694	{0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
		695	{0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
		696	{0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
		697	{0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
		698
		699	{0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
		700	{0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
		701	{0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
		702	{0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
		703	{0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
		704	};
		705
		706	static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
		707	int is_ddr3,
		708	int fsb,
		709	int mem)
		710	{
		711	const struct cxsr_latency *latency;
		712	int i;
		713
		714	if (fsb == 0 \|\| mem == 0)
		715	return NULL;
		716
		717	for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
		718	latency = &cxsr_latency_table[i];
		719	if (is_desktop == latency->is_desktop &&
		720	is_ddr3 == latency->is_ddr3 &&
		721	fsb == latency->fsb_freq && mem == latency->mem_freq)
		722	return latency;
		723	}
		724
		725	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		726
		727	return NULL;
		728	}
		729
		730	static void pineview_disable_cxsr(struct drm_device *dev)
		731	{
		732	struct drm_i915_private *dev_priv = dev->dev_private;
		733
		734	/* deactivate cxsr */
		735	I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
		736	}
		737
		738	/*
		739	* Latency for FIFO fetches is dependent on several factors:
		740	* - memory configuration (speed, channels)
		741	* - chipset
		742	* - current MCH state
		743	* It can be fairly high in some situations, so here we assume a fairly
		744	* pessimal value. It's a tradeoff between extra memory fetches (if we
		745	* set this value too high, the FIFO will fetch frequently to stay full)
		746	* and power consumption (set it too low to save power and we might see
		747	* FIFO underruns and display "flicker").
		748	*
		749	* A value of 5us seems to be a good balance; safe for very low end
		750	* platforms but not overly aggressive on lower latency configs.
		751	*/
		752	static const int latency_ns = 5000;
		753
		754	static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
		755	{
		756	struct drm_i915_private *dev_priv = dev->dev_private;
		757	uint32_t dsparb = I915_READ(DSPARB);
		758	int size;
		759
		760	size = dsparb & 0x7f;
		761	if (plane)
		762	size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
		763
		764	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		765	plane ? "B" : "A", size);
		766
		767	return size;
		768	}
		769
		770	static int i85x_get_fifo_size(struct drm_device *dev, int plane)
		771	{
		772	struct drm_i915_private *dev_priv = dev->dev_private;
		773	uint32_t dsparb = I915_READ(DSPARB);
		774	int size;
		775
		776	size = dsparb & 0x1ff;
		777	if (plane)
		778	size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
		779	size >>= 1; /* Convert to cachelines */
		780
		781	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		782	plane ? "B" : "A", size);
		783
		784	return size;
		785	}
		786
		787	static int i845_get_fifo_size(struct drm_device *dev, int plane)
		788	{
		789	struct drm_i915_private *dev_priv = dev->dev_private;
		790	uint32_t dsparb = I915_READ(DSPARB);
		791	int size;
		792
		793	size = dsparb & 0x7f;
		794	size >>= 2; /* Convert to cachelines */
		795
		796	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		797	plane ? "B" : "A",
		798	size);
		799
		800	return size;
		801	}
		802
		803	static int i830_get_fifo_size(struct drm_device *dev, int plane)
		804	{
		805	struct drm_i915_private *dev_priv = dev->dev_private;
		806	uint32_t dsparb = I915_READ(DSPARB);
		807	int size;
		808
		809	size = dsparb & 0x7f;
		810	size >>= 1; /* Convert to cachelines */
		811
		812	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
		813	plane ? "B" : "A", size);
		814
		815	return size;
		816	}
		817
		818	/* Pineview has different values for various configs */
		819	static const struct intel_watermark_params pineview_display_wm = {
		820	PINEVIEW_DISPLAY_FIFO,
		821	PINEVIEW_MAX_WM,
		822	PINEVIEW_DFT_WM,
		823	PINEVIEW_GUARD_WM,
		824	PINEVIEW_FIFO_LINE_SIZE
		825	};
		826	static const struct intel_watermark_params pineview_display_hplloff_wm = {
		827	PINEVIEW_DISPLAY_FIFO,
		828	PINEVIEW_MAX_WM,
		829	PINEVIEW_DFT_HPLLOFF_WM,
		830	PINEVIEW_GUARD_WM,
		831	PINEVIEW_FIFO_LINE_SIZE
		832	};
		833	static const struct intel_watermark_params pineview_cursor_wm = {
		834	PINEVIEW_CURSOR_FIFO,
		835	PINEVIEW_CURSOR_MAX_WM,
		836	PINEVIEW_CURSOR_DFT_WM,
		837	PINEVIEW_CURSOR_GUARD_WM,
		838	PINEVIEW_FIFO_LINE_SIZE,
		839	};
		840	static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
		841	PINEVIEW_CURSOR_FIFO,
		842	PINEVIEW_CURSOR_MAX_WM,
		843	PINEVIEW_CURSOR_DFT_WM,
		844	PINEVIEW_CURSOR_GUARD_WM,
		845	PINEVIEW_FIFO_LINE_SIZE
		846	};
		847	static const struct intel_watermark_params g4x_wm_info = {
		848	G4X_FIFO_SIZE,
		849	G4X_MAX_WM,
		850	G4X_MAX_WM,
		851	2,
		852	G4X_FIFO_LINE_SIZE,
		853	};
		854	static const struct intel_watermark_params g4x_cursor_wm_info = {
		855	I965_CURSOR_FIFO,
		856	I965_CURSOR_MAX_WM,
		857	I965_CURSOR_DFT_WM,
		858	2,
		859	G4X_FIFO_LINE_SIZE,
		860	};
		861	static const struct intel_watermark_params valleyview_wm_info = {
		862	VALLEYVIEW_FIFO_SIZE,
		863	VALLEYVIEW_MAX_WM,
		864	VALLEYVIEW_MAX_WM,
		865	2,
		866	G4X_FIFO_LINE_SIZE,
		867	};
		868	static const struct intel_watermark_params valleyview_cursor_wm_info = {
		869	I965_CURSOR_FIFO,
		870	VALLEYVIEW_CURSOR_MAX_WM,
		871	I965_CURSOR_DFT_WM,
		872	2,
		873	G4X_FIFO_LINE_SIZE,
		874	};
		875	static const struct intel_watermark_params i965_cursor_wm_info = {
		876	I965_CURSOR_FIFO,
		877	I965_CURSOR_MAX_WM,
		878	I965_CURSOR_DFT_WM,
		879	2,
		880	I915_FIFO_LINE_SIZE,
		881	};
		882	static const struct intel_watermark_params i945_wm_info = {
		883	I945_FIFO_SIZE,
		884	I915_MAX_WM,
		885	1,
		886	2,
		887	I915_FIFO_LINE_SIZE
		888	};
		889	static const struct intel_watermark_params i915_wm_info = {
		890	I915_FIFO_SIZE,
		891	I915_MAX_WM,
		892	1,
		893	2,
		894	I915_FIFO_LINE_SIZE
		895	};
		896	static const struct intel_watermark_params i855_wm_info = {
		897	I855GM_FIFO_SIZE,
		898	I915_MAX_WM,
		899	1,
		900	2,
		901	I830_FIFO_LINE_SIZE
		902	};
		903	static const struct intel_watermark_params i830_wm_info = {
		904	I830_FIFO_SIZE,
		905	I915_MAX_WM,
		906	1,
		907	2,
		908	I830_FIFO_LINE_SIZE
		909	};
		910
		911	static const struct intel_watermark_params ironlake_display_wm_info = {
		912	ILK_DISPLAY_FIFO,
		913	ILK_DISPLAY_MAXWM,
		914	ILK_DISPLAY_DFTWM,
		915	2,
		916	ILK_FIFO_LINE_SIZE
		917	};
		918	static const struct intel_watermark_params ironlake_cursor_wm_info = {
		919	ILK_CURSOR_FIFO,
		920	ILK_CURSOR_MAXWM,
		921	ILK_CURSOR_DFTWM,
		922	2,
		923	ILK_FIFO_LINE_SIZE
		924	};
		925	static const struct intel_watermark_params ironlake_display_srwm_info = {
		926	ILK_DISPLAY_SR_FIFO,
		927	ILK_DISPLAY_MAX_SRWM,
		928	ILK_DISPLAY_DFT_SRWM,
		929	2,
		930	ILK_FIFO_LINE_SIZE
		931	};
		932	static const struct intel_watermark_params ironlake_cursor_srwm_info = {
		933	ILK_CURSOR_SR_FIFO,
		934	ILK_CURSOR_MAX_SRWM,
		935	ILK_CURSOR_DFT_SRWM,
		936	2,
		937	ILK_FIFO_LINE_SIZE
		938	};
		939
		940	static const struct intel_watermark_params sandybridge_display_wm_info = {
		941	SNB_DISPLAY_FIFO,
		942	SNB_DISPLAY_MAXWM,
		943	SNB_DISPLAY_DFTWM,
		944	2,
		945	SNB_FIFO_LINE_SIZE
		946	};
		947	static const struct intel_watermark_params sandybridge_cursor_wm_info = {
		948	SNB_CURSOR_FIFO,
		949	SNB_CURSOR_MAXWM,
		950	SNB_CURSOR_DFTWM,
		951	2,
		952	SNB_FIFO_LINE_SIZE
		953	};
		954	static const struct intel_watermark_params sandybridge_display_srwm_info = {
		955	SNB_DISPLAY_SR_FIFO,
		956	SNB_DISPLAY_MAX_SRWM,
		957	SNB_DISPLAY_DFT_SRWM,
		958	2,
		959	SNB_FIFO_LINE_SIZE
		960	};
		961	static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
		962	SNB_CURSOR_SR_FIFO,
		963	SNB_CURSOR_MAX_SRWM,
		964	SNB_CURSOR_DFT_SRWM,
		965	2,
		966	SNB_FIFO_LINE_SIZE
		967	};
		968
		969
		970	/**
		971	* intel_calculate_wm - calculate watermark level
		972	* @clock_in_khz: pixel clock
		973	* @wm: chip FIFO params
		974	* @pixel_size: display pixel size
		975	* @latency_ns: memory latency for the platform
		976	*
		977	* Calculate the watermark level (the level at which the display plane will
		978	* start fetching from memory again). Each chip has a different display
		979	* FIFO size and allocation, so the caller needs to figure that out and pass
		980	* in the correct intel_watermark_params structure.
		981	*
		982	* As the pixel clock runs, the FIFO will be drained at a rate that depends
		983	* on the pixel size. When it reaches the watermark level, it'll start
		984	* fetching FIFO line sized based chunks from memory until the FIFO fills
		985	* past the watermark point. If the FIFO drains completely, a FIFO underrun
		986	* will occur, and a display engine hang could result.
		987	*/
		988	static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
		989	const struct intel_watermark_params *wm,
		990	int fifo_size,
		991	int pixel_size,
		992	unsigned long latency_ns)
		993	{
		994	long entries_required, wm_size;
		995
		996	/*
		997	* Note: we need to make sure we don't overflow for various clock &
		998	* latency values.
		999	* clocks go from a few thousand to several hundred thousand.
		1000	* latency is usually a few thousand
		1001	*/
		1002	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
		1003	1000;
		1004	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
		1005
		1006	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
		1007
		1008	wm_size = fifo_size - (entries_required + wm->guard_size);
		1009
		1010	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
		1011
		1012	/* Don't promote wm_size to unsigned... */
		1013	if (wm_size > (long)wm->max_wm)
		1014	wm_size = wm->max_wm;
		1015	if (wm_size <= 0)
		1016	wm_size = wm->default_wm;
		1017	return wm_size;
		1018	}
		1019
		1020	static struct drm_crtc single_enabled_crtc(struct drm_device dev)
		1021	{
		1022	struct drm_crtc crtc, enabled = NULL;
		1023
		1024	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3243	Serge	1025	if (intel_crtc_active(crtc)) {
3031	serge	1026	if (enabled)
		1027	return NULL;
		1028	enabled = crtc;
		1029	}
		1030	}
		1031
		1032	return enabled;
		1033	}
		1034
		1035	static void pineview_update_wm(struct drm_device *dev)
		1036	{
		1037	struct drm_i915_private *dev_priv = dev->dev_private;
		1038	struct drm_crtc *crtc;
		1039	const struct cxsr_latency *latency;
		1040	u32 reg;
		1041	unsigned long wm;
		1042
		1043	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
		1044	dev_priv->fsb_freq, dev_priv->mem_freq);
		1045	if (!latency) {
		1046	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
		1047	pineview_disable_cxsr(dev);
		1048	return;
		1049	}
		1050
		1051	crtc = single_enabled_crtc(dev);
		1052	if (crtc) {
		1053	int clock = crtc->mode.clock;
		1054	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1055
		1056	/* Display SR */
		1057	wm = intel_calculate_wm(clock, &pineview_display_wm,
		1058	pineview_display_wm.fifo_size,
		1059	pixel_size, latency->display_sr);
		1060	reg = I915_READ(DSPFW1);
		1061	reg &= ~DSPFW_SR_MASK;
		1062	reg \|= wm << DSPFW_SR_SHIFT;
		1063	I915_WRITE(DSPFW1, reg);
		1064	DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
		1065
		1066	/* cursor SR */
		1067	wm = intel_calculate_wm(clock, &pineview_cursor_wm,
		1068	pineview_display_wm.fifo_size,
		1069	pixel_size, latency->cursor_sr);
		1070	reg = I915_READ(DSPFW3);
		1071	reg &= ~DSPFW_CURSOR_SR_MASK;
		1072	reg \|= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
		1073	I915_WRITE(DSPFW3, reg);
		1074
		1075	/* Display HPLL off SR */
		1076	wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
		1077	pineview_display_hplloff_wm.fifo_size,
		1078	pixel_size, latency->display_hpll_disable);
		1079	reg = I915_READ(DSPFW3);
		1080	reg &= ~DSPFW_HPLL_SR_MASK;
		1081	reg \|= wm & DSPFW_HPLL_SR_MASK;
		1082	I915_WRITE(DSPFW3, reg);
		1083
		1084	/* cursor HPLL off SR */
		1085	wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
		1086	pineview_display_hplloff_wm.fifo_size,
		1087	pixel_size, latency->cursor_hpll_disable);
		1088	reg = I915_READ(DSPFW3);
		1089	reg &= ~DSPFW_HPLL_CURSOR_MASK;
		1090	reg \|= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
		1091	I915_WRITE(DSPFW3, reg);
		1092	DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
		1093
		1094	/* activate cxsr */
		1095	I915_WRITE(DSPFW3,
		1096	I915_READ(DSPFW3) \| PINEVIEW_SELF_REFRESH_EN);
		1097	DRM_DEBUG_KMS("Self-refresh is enabled\n");
		1098	} else {
		1099	pineview_disable_cxsr(dev);
		1100	DRM_DEBUG_KMS("Self-refresh is disabled\n");
		1101	}
		1102	}
		1103
		1104	static bool g4x_compute_wm0(struct drm_device *dev,
		1105	int plane,
		1106	const struct intel_watermark_params *display,
		1107	int display_latency_ns,
		1108	const struct intel_watermark_params *cursor,
		1109	int cursor_latency_ns,
		1110	int *plane_wm,
		1111	int *cursor_wm)
		1112	{
		1113	struct drm_crtc *crtc;
		1114	int htotal, hdisplay, clock, pixel_size;
		1115	int line_time_us, line_count;
		1116	int entries, tlb_miss;
		1117
		1118	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1119	if (!intel_crtc_active(crtc)) {
3031	serge	1120	*cursor_wm = cursor->guard_size;
		1121	*plane_wm = display->guard_size;
		1122	return false;
		1123	}
		1124
		1125	htotal = crtc->mode.htotal;
		1126	hdisplay = crtc->mode.hdisplay;
		1127	clock = crtc->mode.clock;
		1128	pixel_size = crtc->fb->bits_per_pixel / 8;
		1129
		1130	/* Use the small buffer method to calculate plane watermark */
		1131	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		1132	tlb_miss = display->fifo_sizedisplay->cacheline_size - hdisplay 8;
		1133	if (tlb_miss > 0)
		1134	entries += tlb_miss;
		1135	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		1136	*plane_wm = entries + display->guard_size;
		1137	if (*plane_wm > (int)display->max_wm)
		1138	*plane_wm = display->max_wm;
		1139
		1140	/* Use the large buffer method to calculate cursor watermark */
		1141	line_time_us = ((htotal * 1000) / clock);
		1142	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
		1143	entries = line_count * 64 * pixel_size;
		1144	tlb_miss = cursor->fifo_sizecursor->cacheline_size - hdisplay 8;
		1145	if (tlb_miss > 0)
		1146	entries += tlb_miss;
		1147	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1148	*cursor_wm = entries + cursor->guard_size;
		1149	if (*cursor_wm > (int)cursor->max_wm)
		1150	*cursor_wm = (int)cursor->max_wm;
		1151
		1152	return true;
		1153	}
		1154
		1155	/*
		1156	* Check the wm result.
		1157	*
		1158	* If any calculated watermark values is larger than the maximum value that
		1159	* can be programmed into the associated watermark register, that watermark
		1160	* must be disabled.
		1161	*/
		1162	static bool g4x_check_srwm(struct drm_device *dev,
		1163	int display_wm, int cursor_wm,
		1164	const struct intel_watermark_params *display,
		1165	const struct intel_watermark_params *cursor)
		1166	{
		1167	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
		1168	display_wm, cursor_wm);
		1169
		1170	if (display_wm > display->max_wm) {
		1171	DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
		1172	display_wm, display->max_wm);
		1173	return false;
		1174	}
		1175
		1176	if (cursor_wm > cursor->max_wm) {
		1177	DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
		1178	cursor_wm, cursor->max_wm);
		1179	return false;
		1180	}
		1181
		1182	if (!(display_wm \|\| cursor_wm)) {
		1183	DRM_DEBUG_KMS("SR latency is 0, disabling\n");
		1184	return false;
		1185	}
		1186
		1187	return true;
		1188	}
		1189
		1190	static bool g4x_compute_srwm(struct drm_device *dev,
		1191	int plane,
		1192	int latency_ns,
		1193	const struct intel_watermark_params *display,
		1194	const struct intel_watermark_params *cursor,
		1195	int display_wm, int cursor_wm)
		1196	{
		1197	struct drm_crtc *crtc;
		1198	int hdisplay, htotal, pixel_size, clock;
		1199	unsigned long line_time_us;
		1200	int line_count, line_size;
		1201	int small, large;
		1202	int entries;
		1203
		1204	if (!latency_ns) {
		1205	display_wm = cursor_wm = 0;
		1206	return false;
		1207	}
		1208
		1209	crtc = intel_get_crtc_for_plane(dev, plane);
		1210	hdisplay = crtc->mode.hdisplay;
		1211	htotal = crtc->mode.htotal;
		1212	clock = crtc->mode.clock;
		1213	pixel_size = crtc->fb->bits_per_pixel / 8;
		1214
		1215	line_time_us = (htotal * 1000) / clock;
		1216	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1217	line_size = hdisplay * pixel_size;
		1218
		1219	/* Use the minimum of the small and large buffer method for primary */
		1220	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1221	large = line_count * line_size;
		1222
		1223	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1224	*display_wm = entries + display->guard_size;
		1225
		1226	/* calculate the self-refresh watermark for display cursor */
		1227	entries = line_count * pixel_size * 64;
		1228	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1229	*cursor_wm = entries + cursor->guard_size;
		1230
		1231	return g4x_check_srwm(dev,
		1232	display_wm, cursor_wm,
		1233	display, cursor);
		1234	}
		1235
		1236	static bool vlv_compute_drain_latency(struct drm_device *dev,
		1237	int plane,
		1238	int *plane_prec_mult,
		1239	int *plane_dl,
		1240	int *cursor_prec_mult,
		1241	int *cursor_dl)
		1242	{
		1243	struct drm_crtc *crtc;
		1244	int clock, pixel_size;
		1245	int entries;
		1246
		1247	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	1248	if (!intel_crtc_active(crtc))
3031	serge	1249	return false;
		1250
		1251	clock = crtc->mode.clock; /* VESA DOT Clock */
		1252	pixel_size = crtc->fb->bits_per_pixel / 8; /* BPP */
		1253
		1254	entries = (clock / 1000) * pixel_size;
		1255	*plane_prec_mult = (entries > 256) ?
		1256	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1257	plane_dl = (64 (plane_prec_mult) 4) / ((clock / 1000) *
		1258	pixel_size);
		1259
		1260	entries = (clock / 1000) * 4; /* BPP is always 4 for cursor */
		1261	*cursor_prec_mult = (entries > 256) ?
		1262	DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
		1263	cursor_dl = (64 (cursor_prec_mult) 4) / ((clock / 1000) * 4);
		1264
		1265	return true;
		1266	}
		1267
		1268	/*
		1269	* Update drain latency registers of memory arbiter
		1270	*
		1271	* Valleyview SoC has a new memory arbiter and needs drain latency registers
		1272	* to be programmed. Each plane has a drain latency multiplier and a drain
		1273	* latency value.
		1274	*/
		1275
		1276	static void vlv_update_drain_latency(struct drm_device *dev)
		1277	{
		1278	struct drm_i915_private *dev_priv = dev->dev_private;
		1279	int planea_prec, planea_dl, planeb_prec, planeb_dl;
		1280	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
		1281	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
		1282	either 16 or 32 */
		1283
		1284	/* For plane A, Cursor A */
		1285	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
		1286	&cursor_prec_mult, &cursora_dl)) {
		1287	cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1288	DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
		1289	planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1290	DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
		1291
		1292	I915_WRITE(VLV_DDL1, cursora_prec \|
		1293	(cursora_dl << DDL_CURSORA_SHIFT) \|
		1294	planea_prec \| planea_dl);
		1295	}
		1296
		1297	/* For plane B, Cursor B */
		1298	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
		1299	&cursor_prec_mult, &cursorb_dl)) {
		1300	cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1301	DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
		1302	planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
		1303	DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
		1304
		1305	I915_WRITE(VLV_DDL2, cursorb_prec \|
		1306	(cursorb_dl << DDL_CURSORB_SHIFT) \|
		1307	planeb_prec \| planeb_dl);
		1308	}
		1309	}
		1310
		1311	#define single_plane_enabled(mask) is_power_of_2(mask)
		1312
		1313	static void valleyview_update_wm(struct drm_device *dev)
		1314	{
		1315	static const int sr_latency_ns = 12000;
		1316	struct drm_i915_private *dev_priv = dev->dev_private;
		1317	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1318	int plane_sr, cursor_sr;
3243	Serge	1319	int ignore_plane_sr, ignore_cursor_sr;
3031	serge	1320	unsigned int enabled = 0;
		1321
		1322	vlv_update_drain_latency(dev);
		1323
		1324	if (g4x_compute_wm0(dev, 0,
		1325	&valleyview_wm_info, latency_ns,
		1326	&valleyview_cursor_wm_info, latency_ns,
		1327	&planea_wm, &cursora_wm))
		1328	enabled \|= 1;
		1329
		1330	if (g4x_compute_wm0(dev, 1,
		1331	&valleyview_wm_info, latency_ns,
		1332	&valleyview_cursor_wm_info, latency_ns,
		1333	&planeb_wm, &cursorb_wm))
		1334	enabled \|= 2;
		1335
		1336	if (single_plane_enabled(enabled) &&
		1337	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1338	sr_latency_ns,
		1339	&valleyview_wm_info,
		1340	&valleyview_cursor_wm_info,
3243	Serge	1341	&plane_sr, &ignore_cursor_sr) &&
		1342	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1343	2*sr_latency_ns,
		1344	&valleyview_wm_info,
		1345	&valleyview_cursor_wm_info,
		1346	&ignore_plane_sr, &cursor_sr)) {
3031	serge	1347	I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
3243	Serge	1348	} else {
3031	serge	1349	I915_WRITE(FW_BLC_SELF_VLV,
		1350	I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
3243	Serge	1351	plane_sr = cursor_sr = 0;
		1352	}
3031	serge	1353
		1354	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1355	planea_wm, cursora_wm,
		1356	planeb_wm, cursorb_wm,
		1357	plane_sr, cursor_sr);
		1358
		1359	I915_WRITE(DSPFW1,
		1360	(plane_sr << DSPFW_SR_SHIFT) \|
		1361	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1362	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1363	planea_wm);
		1364	I915_WRITE(DSPFW2,
3243	Serge	1365	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1366	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1367	I915_WRITE(DSPFW3,
3243	Serge	1368	(I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) \|
		1369	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
3031	serge	1370	}
		1371
		1372	static void g4x_update_wm(struct drm_device *dev)
		1373	{
		1374	static const int sr_latency_ns = 12000;
		1375	struct drm_i915_private *dev_priv = dev->dev_private;
		1376	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
		1377	int plane_sr, cursor_sr;
		1378	unsigned int enabled = 0;
		1379
		1380	if (g4x_compute_wm0(dev, 0,
		1381	&g4x_wm_info, latency_ns,
		1382	&g4x_cursor_wm_info, latency_ns,
		1383	&planea_wm, &cursora_wm))
		1384	enabled \|= 1;
		1385
		1386	if (g4x_compute_wm0(dev, 1,
		1387	&g4x_wm_info, latency_ns,
		1388	&g4x_cursor_wm_info, latency_ns,
		1389	&planeb_wm, &cursorb_wm))
		1390	enabled \|= 2;
		1391
		1392	if (single_plane_enabled(enabled) &&
		1393	g4x_compute_srwm(dev, ffs(enabled) - 1,
		1394	sr_latency_ns,
		1395	&g4x_wm_info,
		1396	&g4x_cursor_wm_info,
3243	Serge	1397	&plane_sr, &cursor_sr)) {
3031	serge	1398	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
3243	Serge	1399	} else {
3031	serge	1400	I915_WRITE(FW_BLC_SELF,
		1401	I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
3243	Serge	1402	plane_sr = cursor_sr = 0;
		1403	}
3031	serge	1404
		1405	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
		1406	planea_wm, cursora_wm,
		1407	planeb_wm, cursorb_wm,
		1408	plane_sr, cursor_sr);
		1409
		1410	I915_WRITE(DSPFW1,
		1411	(plane_sr << DSPFW_SR_SHIFT) \|
		1412	(cursorb_wm << DSPFW_CURSORB_SHIFT) \|
		1413	(planeb_wm << DSPFW_PLANEB_SHIFT) \|
		1414	planea_wm);
		1415	I915_WRITE(DSPFW2,
3243	Serge	1416	(I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) \|
3031	serge	1417	(cursora_wm << DSPFW_CURSORA_SHIFT));
		1418	/* HPLL off in SR has some issues on G4x... disable it */
		1419	I915_WRITE(DSPFW3,
3243	Serge	1420	(I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN \| DSPFW_CURSOR_SR_MASK)) \|
3031	serge	1421	(cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1422	}
		1423
		1424	static void i965_update_wm(struct drm_device *dev)
		1425	{
		1426	struct drm_i915_private *dev_priv = dev->dev_private;
		1427	struct drm_crtc *crtc;
		1428	int srwm = 1;
		1429	int cursor_sr = 16;
		1430
		1431	/* Calc sr entries for one plane configs */
		1432	crtc = single_enabled_crtc(dev);
		1433	if (crtc) {
		1434	/* self-refresh has much higher latency */
		1435	static const int sr_latency_ns = 12000;
		1436	int clock = crtc->mode.clock;
		1437	int htotal = crtc->mode.htotal;
		1438	int hdisplay = crtc->mode.hdisplay;
		1439	int pixel_size = crtc->fb->bits_per_pixel / 8;
		1440	unsigned long line_time_us;
		1441	int entries;
		1442
		1443	line_time_us = ((htotal * 1000) / clock);
		1444
		1445	/* Use ns/us then divide to preserve precision */
		1446	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1447	pixel_size * hdisplay;
		1448	entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
		1449	srwm = I965_FIFO_SIZE - entries;
		1450	if (srwm < 0)
		1451	srwm = 1;
		1452	srwm &= 0x1ff;
		1453	DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
		1454	entries, srwm);
		1455
		1456	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1457	pixel_size * 64;
		1458	entries = DIV_ROUND_UP(entries,
		1459	i965_cursor_wm_info.cacheline_size);
		1460	cursor_sr = i965_cursor_wm_info.fifo_size -
		1461	(entries + i965_cursor_wm_info.guard_size);
		1462
		1463	if (cursor_sr > i965_cursor_wm_info.max_wm)
		1464	cursor_sr = i965_cursor_wm_info.max_wm;
		1465
		1466	DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
		1467	"cursor %d\n", srwm, cursor_sr);
		1468
		1469	if (IS_CRESTLINE(dev))
		1470	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
		1471	} else {
		1472	/* Turn off self refresh if both pipes are enabled */
		1473	if (IS_CRESTLINE(dev))
		1474	I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
		1475	& ~FW_BLC_SELF_EN);
		1476	}
		1477
		1478	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
		1479	srwm);
		1480
		1481	/* 965 has limitations... */
		1482	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) \|
		1483	(8 << 16) \| (8 << 8) \| (8 << 0));
		1484	I915_WRITE(DSPFW2, (8 << 8) \| (8 << 0));
		1485	/* update cursor SR watermark */
		1486	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
		1487	}
		1488
		1489	static void i9xx_update_wm(struct drm_device *dev)
		1490	{
		1491	struct drm_i915_private *dev_priv = dev->dev_private;
		1492	const struct intel_watermark_params *wm_info;
		1493	uint32_t fwater_lo;
		1494	uint32_t fwater_hi;
		1495	int cwm, srwm = 1;
		1496	int fifo_size;
		1497	int planea_wm, planeb_wm;
		1498	struct drm_crtc crtc, enabled = NULL;
		1499
		1500	if (IS_I945GM(dev))
		1501	wm_info = &i945_wm_info;
		1502	else if (!IS_GEN2(dev))
		1503	wm_info = &i915_wm_info;
		1504	else
		1505	wm_info = &i855_wm_info;
		1506
		1507	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
		1508	crtc = intel_get_crtc_for_plane(dev, 0);
3243	Serge	1509	if (intel_crtc_active(crtc)) {
		1510	int cpp = crtc->fb->bits_per_pixel / 8;
		1511	if (IS_GEN2(dev))
		1512	cpp = 4;
		1513
3031	serge	1514	planea_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1515	wm_info, fifo_size, cpp,
3031	serge	1516	latency_ns);
		1517	enabled = crtc;
		1518	} else
		1519	planea_wm = fifo_size - wm_info->guard_size;
		1520
		1521	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
		1522	crtc = intel_get_crtc_for_plane(dev, 1);
3243	Serge	1523	if (intel_crtc_active(crtc)) {
		1524	int cpp = crtc->fb->bits_per_pixel / 8;
		1525	if (IS_GEN2(dev))
		1526	cpp = 4;
		1527
3031	serge	1528	planeb_wm = intel_calculate_wm(crtc->mode.clock,
3243	Serge	1529	wm_info, fifo_size, cpp,
3031	serge	1530	latency_ns);
		1531	if (enabled == NULL)
		1532	enabled = crtc;
		1533	else
		1534	enabled = NULL;
		1535	} else
		1536	planeb_wm = fifo_size - wm_info->guard_size;
		1537
		1538	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
		1539
		1540	/*
		1541	* Overlay gets an aggressive default since video jitter is bad.
		1542	*/
		1543	cwm = 2;
		1544
		1545	/* Play safe and disable self-refresh before adjusting watermarks. */
		1546	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1547	I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK \| 0);
		1548	else if (IS_I915GM(dev))
		1549	I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
		1550
		1551	/* Calc sr entries for one plane configs */
		1552	if (HAS_FW_BLC(dev) && enabled) {
		1553	/* self-refresh has much higher latency */
		1554	static const int sr_latency_ns = 6000;
		1555	int clock = enabled->mode.clock;
		1556	int htotal = enabled->mode.htotal;
		1557	int hdisplay = enabled->mode.hdisplay;
		1558	int pixel_size = enabled->fb->bits_per_pixel / 8;
		1559	unsigned long line_time_us;
		1560	int entries;
		1561
		1562	line_time_us = (htotal * 1000) / clock;
		1563
		1564	/* Use ns/us then divide to preserve precision */
		1565	entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
		1566	pixel_size * hdisplay;
		1567	entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
		1568	DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
		1569	srwm = wm_info->fifo_size - entries;
		1570	if (srwm < 0)
		1571	srwm = 1;
		1572
		1573	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1574	I915_WRITE(FW_BLC_SELF,
		1575	FW_BLC_SELF_FIFO_MASK \| (srwm & 0xff));
		1576	else if (IS_I915GM(dev))
		1577	I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
		1578	}
		1579
		1580	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
		1581	planea_wm, planeb_wm, cwm, srwm);
		1582
		1583	fwater_lo = ((planeb_wm & 0x3f) << 16) \| (planea_wm & 0x3f);
		1584	fwater_hi = (cwm & 0x1f);
		1585
		1586	/* Set request length to 8 cachelines per fetch */
		1587	fwater_lo = fwater_lo \| (1 << 24) \| (1 << 8);
		1588	fwater_hi = fwater_hi \| (1 << 8);
		1589
		1590	I915_WRITE(FW_BLC, fwater_lo);
		1591	I915_WRITE(FW_BLC2, fwater_hi);
		1592
		1593	if (HAS_FW_BLC(dev)) {
		1594	if (enabled) {
		1595	if (IS_I945G(dev) \|\| IS_I945GM(dev))
		1596	I915_WRITE(FW_BLC_SELF,
		1597	FW_BLC_SELF_EN_MASK \| FW_BLC_SELF_EN);
		1598	else if (IS_I915GM(dev))
		1599	I915_WRITE(INSTPM, I915_READ(INSTPM) \| INSTPM_SELF_EN);
		1600	DRM_DEBUG_KMS("memory self refresh enabled\n");
		1601	} else
		1602	DRM_DEBUG_KMS("memory self refresh disabled\n");
		1603	}
		1604	}
		1605
		1606	static void i830_update_wm(struct drm_device *dev)
		1607	{
		1608	struct drm_i915_private *dev_priv = dev->dev_private;
		1609	struct drm_crtc *crtc;
		1610	uint32_t fwater_lo;
		1611	int planea_wm;
		1612
		1613	crtc = single_enabled_crtc(dev);
		1614	if (crtc == NULL)
		1615	return;
		1616
		1617	planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
		1618	dev_priv->display.get_fifo_size(dev, 0),
3243	Serge	1619	4, latency_ns);
3031	serge	1620	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
		1621	fwater_lo \|= (3<<8) \| planea_wm;
		1622
		1623	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
		1624
		1625	I915_WRITE(FW_BLC, fwater_lo);
		1626	}
		1627
		1628	#define ILK_LP0_PLANE_LATENCY 700
		1629	#define ILK_LP0_CURSOR_LATENCY 1300
		1630
		1631	/*
		1632	* Check the wm result.
		1633	*
		1634	* If any calculated watermark values is larger than the maximum value that
		1635	* can be programmed into the associated watermark register, that watermark
		1636	* must be disabled.
		1637	*/
		1638	static bool ironlake_check_srwm(struct drm_device *dev, int level,
		1639	int fbc_wm, int display_wm, int cursor_wm,
		1640	const struct intel_watermark_params *display,
		1641	const struct intel_watermark_params *cursor)
		1642	{
		1643	struct drm_i915_private *dev_priv = dev->dev_private;
		1644
		1645	DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
		1646	" cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
		1647
		1648	if (fbc_wm > SNB_FBC_MAX_SRWM) {
		1649	DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
		1650	fbc_wm, SNB_FBC_MAX_SRWM, level);
		1651
		1652	/* fbc has it's own way to disable FBC WM */
		1653	I915_WRITE(DISP_ARB_CTL,
		1654	I915_READ(DISP_ARB_CTL) \| DISP_FBC_WM_DIS);
		1655	return false;
		1656	}
		1657
		1658	if (display_wm > display->max_wm) {
		1659	DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
		1660	display_wm, SNB_DISPLAY_MAX_SRWM, level);
		1661	return false;
		1662	}
		1663
		1664	if (cursor_wm > cursor->max_wm) {
		1665	DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
		1666	cursor_wm, SNB_CURSOR_MAX_SRWM, level);
		1667	return false;
		1668	}
		1669
		1670	if (!(fbc_wm \|\| display_wm \|\| cursor_wm)) {
		1671	DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
		1672	return false;
		1673	}
		1674
		1675	return true;
		1676	}
		1677
		1678	/*
		1679	* Compute watermark values of WM[1-3],
		1680	*/
		1681	static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
		1682	int latency_ns,
		1683	const struct intel_watermark_params *display,
		1684	const struct intel_watermark_params *cursor,
		1685	int fbc_wm, int display_wm, int *cursor_wm)
		1686	{
		1687	struct drm_crtc *crtc;
		1688	unsigned long line_time_us;
		1689	int hdisplay, htotal, pixel_size, clock;
		1690	int line_count, line_size;
		1691	int small, large;
		1692	int entries;
		1693
		1694	if (!latency_ns) {
		1695	fbc_wm = display_wm = *cursor_wm = 0;
		1696	return false;
		1697	}
		1698
		1699	crtc = intel_get_crtc_for_plane(dev, plane);
		1700	hdisplay = crtc->mode.hdisplay;
		1701	htotal = crtc->mode.htotal;
		1702	clock = crtc->mode.clock;
		1703	pixel_size = crtc->fb->bits_per_pixel / 8;
		1704
		1705	line_time_us = (htotal * 1000) / clock;
		1706	line_count = (latency_ns / line_time_us + 1000) / 1000;
		1707	line_size = hdisplay * pixel_size;
		1708
		1709	/* Use the minimum of the small and large buffer method for primary */
		1710	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		1711	large = line_count * line_size;
		1712
		1713	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		1714	*display_wm = entries + display->guard_size;
		1715
		1716	/*
		1717	* Spec says:
		1718	* FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
		1719	*/
		1720	fbc_wm = DIV_ROUND_UP(display_wm * 64, line_size) + 2;
		1721
		1722	/* calculate the self-refresh watermark for display cursor */
		1723	entries = line_count * pixel_size * 64;
		1724	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
		1725	*cursor_wm = entries + cursor->guard_size;
		1726
		1727	return ironlake_check_srwm(dev, level,
		1728	fbc_wm, display_wm, *cursor_wm,
		1729	display, cursor);
		1730	}
		1731
		1732	static void ironlake_update_wm(struct drm_device *dev)
		1733	{
		1734	struct drm_i915_private *dev_priv = dev->dev_private;
		1735	int fbc_wm, plane_wm, cursor_wm;
		1736	unsigned int enabled;
		1737
		1738	enabled = 0;
		1739	if (g4x_compute_wm0(dev, 0,
		1740	&ironlake_display_wm_info,
		1741	ILK_LP0_PLANE_LATENCY,
		1742	&ironlake_cursor_wm_info,
		1743	ILK_LP0_CURSOR_LATENCY,
		1744	&plane_wm, &cursor_wm)) {
		1745	I915_WRITE(WM0_PIPEA_ILK,
		1746	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1747	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1748	" plane %d, " "cursor: %d\n",
		1749	plane_wm, cursor_wm);
		1750	enabled \|= 1;
		1751	}
		1752
		1753	if (g4x_compute_wm0(dev, 1,
		1754	&ironlake_display_wm_info,
		1755	ILK_LP0_PLANE_LATENCY,
		1756	&ironlake_cursor_wm_info,
		1757	ILK_LP0_CURSOR_LATENCY,
		1758	&plane_wm, &cursor_wm)) {
		1759	I915_WRITE(WM0_PIPEB_ILK,
		1760	(plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm);
		1761	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1762	" plane %d, cursor: %d\n",
		1763	plane_wm, cursor_wm);
		1764	enabled \|= 2;
		1765	}
		1766
		1767	/*
		1768	* Calculate and update the self-refresh watermark only when one
		1769	* display plane is used.
		1770	*/
		1771	I915_WRITE(WM3_LP_ILK, 0);
		1772	I915_WRITE(WM2_LP_ILK, 0);
		1773	I915_WRITE(WM1_LP_ILK, 0);
		1774
		1775	if (!single_plane_enabled(enabled))
		1776	return;
		1777	enabled = ffs(enabled) - 1;
		1778
		1779	/* WM1 */
		1780	if (!ironlake_compute_srwm(dev, 1, enabled,
		1781	ILK_READ_WM1_LATENCY() * 500,
		1782	&ironlake_display_srwm_info,
		1783	&ironlake_cursor_srwm_info,
		1784	&fbc_wm, &plane_wm, &cursor_wm))
		1785	return;
		1786
		1787	I915_WRITE(WM1_LP_ILK,
		1788	WM1_LP_SR_EN \|
		1789	(ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1790	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1791	(plane_wm << WM1_LP_SR_SHIFT) \|
		1792	cursor_wm);
		1793
		1794	/* WM2 */
		1795	if (!ironlake_compute_srwm(dev, 2, enabled,
		1796	ILK_READ_WM2_LATENCY() * 500,
		1797	&ironlake_display_srwm_info,
		1798	&ironlake_cursor_srwm_info,
		1799	&fbc_wm, &plane_wm, &cursor_wm))
		1800	return;
		1801
		1802	I915_WRITE(WM2_LP_ILK,
		1803	WM2_LP_EN \|
		1804	(ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1805	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1806	(plane_wm << WM1_LP_SR_SHIFT) \|
		1807	cursor_wm);
		1808
		1809	/*
		1810	* WM3 is unsupported on ILK, probably because we don't have latency
		1811	* data for that power state
		1812	*/
		1813	}
		1814
		1815	static void sandybridge_update_wm(struct drm_device *dev)
		1816	{
		1817	struct drm_i915_private *dev_priv = dev->dev_private;
		1818	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		1819	u32 val;
		1820	int fbc_wm, plane_wm, cursor_wm;
		1821	unsigned int enabled;
		1822
		1823	enabled = 0;
		1824	if (g4x_compute_wm0(dev, 0,
		1825	&sandybridge_display_wm_info, latency,
		1826	&sandybridge_cursor_wm_info, latency,
		1827	&plane_wm, &cursor_wm)) {
		1828	val = I915_READ(WM0_PIPEA_ILK);
		1829	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1830	I915_WRITE(WM0_PIPEA_ILK, val \|
		1831	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1832	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1833	" plane %d, " "cursor: %d\n",
		1834	plane_wm, cursor_wm);
		1835	enabled \|= 1;
		1836	}
		1837
		1838	if (g4x_compute_wm0(dev, 1,
		1839	&sandybridge_display_wm_info, latency,
		1840	&sandybridge_cursor_wm_info, latency,
		1841	&plane_wm, &cursor_wm)) {
		1842	val = I915_READ(WM0_PIPEB_ILK);
		1843	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1844	I915_WRITE(WM0_PIPEB_ILK, val \|
		1845	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1846	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1847	" plane %d, cursor: %d\n",
		1848	plane_wm, cursor_wm);
		1849	enabled \|= 2;
		1850	}
		1851
3243	Serge	1852	/*
		1853	* Calculate and update the self-refresh watermark only when one
		1854	* display plane is used.
		1855	*
		1856	* SNB support 3 levels of watermark.
		1857	*
		1858	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1859	* and disabled in the descending order
		1860	*
		1861	*/
		1862	I915_WRITE(WM3_LP_ILK, 0);
		1863	I915_WRITE(WM2_LP_ILK, 0);
		1864	I915_WRITE(WM1_LP_ILK, 0);
		1865
		1866	if (!single_plane_enabled(enabled) \|\|
		1867	dev_priv->sprite_scaling_enabled)
		1868	return;
		1869	enabled = ffs(enabled) - 1;
		1870
		1871	/* WM1 */
		1872	if (!ironlake_compute_srwm(dev, 1, enabled,
		1873	SNB_READ_WM1_LATENCY() * 500,
		1874	&sandybridge_display_srwm_info,
		1875	&sandybridge_cursor_srwm_info,
		1876	&fbc_wm, &plane_wm, &cursor_wm))
		1877	return;
		1878
		1879	I915_WRITE(WM1_LP_ILK,
		1880	WM1_LP_SR_EN \|
		1881	(SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1882	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1883	(plane_wm << WM1_LP_SR_SHIFT) \|
		1884	cursor_wm);
		1885
		1886	/* WM2 */
		1887	if (!ironlake_compute_srwm(dev, 2, enabled,
		1888	SNB_READ_WM2_LATENCY() * 500,
		1889	&sandybridge_display_srwm_info,
		1890	&sandybridge_cursor_srwm_info,
		1891	&fbc_wm, &plane_wm, &cursor_wm))
		1892	return;
		1893
		1894	I915_WRITE(WM2_LP_ILK,
		1895	WM2_LP_EN \|
		1896	(SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1897	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1898	(plane_wm << WM1_LP_SR_SHIFT) \|
		1899	cursor_wm);
		1900
		1901	/* WM3 */
		1902	if (!ironlake_compute_srwm(dev, 3, enabled,
		1903	SNB_READ_WM3_LATENCY() * 500,
		1904	&sandybridge_display_srwm_info,
		1905	&sandybridge_cursor_srwm_info,
		1906	&fbc_wm, &plane_wm, &cursor_wm))
		1907	return;
		1908
		1909	I915_WRITE(WM3_LP_ILK,
		1910	WM3_LP_EN \|
		1911	(SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1912	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		1913	(plane_wm << WM1_LP_SR_SHIFT) \|
		1914	cursor_wm);
		1915	}
		1916
		1917	static void ivybridge_update_wm(struct drm_device *dev)
		1918	{
		1919	struct drm_i915_private *dev_priv = dev->dev_private;
		1920	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		1921	u32 val;
		1922	int fbc_wm, plane_wm, cursor_wm;
		1923	int ignore_fbc_wm, ignore_plane_wm, ignore_cursor_wm;
		1924	unsigned int enabled;
		1925
		1926	enabled = 0;
		1927	if (g4x_compute_wm0(dev, 0,
3031	serge	1928	&sandybridge_display_wm_info, latency,
		1929	&sandybridge_cursor_wm_info, latency,
		1930	&plane_wm, &cursor_wm)) {
3243	Serge	1931	val = I915_READ(WM0_PIPEA_ILK);
		1932	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1933	I915_WRITE(WM0_PIPEA_ILK, val \|
		1934	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1935	DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
		1936	" plane %d, " "cursor: %d\n",
		1937	plane_wm, cursor_wm);
		1938	enabled \|= 1;
		1939	}
		1940
		1941	if (g4x_compute_wm0(dev, 1,
		1942	&sandybridge_display_wm_info, latency,
		1943	&sandybridge_cursor_wm_info, latency,
		1944	&plane_wm, &cursor_wm)) {
		1945	val = I915_READ(WM0_PIPEB_ILK);
		1946	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1947	I915_WRITE(WM0_PIPEB_ILK, val \|
		1948	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1949	DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
		1950	" plane %d, cursor: %d\n",
		1951	plane_wm, cursor_wm);
		1952	enabled \|= 2;
		1953	}
		1954
		1955	if (g4x_compute_wm0(dev, 2,
		1956	&sandybridge_display_wm_info, latency,
		1957	&sandybridge_cursor_wm_info, latency,
		1958	&plane_wm, &cursor_wm)) {
3031	serge	1959	val = I915_READ(WM0_PIPEC_IVB);
		1960	val &= ~(WM0_PIPE_PLANE_MASK \| WM0_PIPE_CURSOR_MASK);
		1961	I915_WRITE(WM0_PIPEC_IVB, val \|
		1962	((plane_wm << WM0_PIPE_PLANE_SHIFT) \| cursor_wm));
		1963	DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
		1964	" plane %d, cursor: %d\n",
		1965	plane_wm, cursor_wm);
		1966	enabled \|= 3;
		1967	}
		1968
		1969	/*
		1970	* Calculate and update the self-refresh watermark only when one
		1971	* display plane is used.
		1972	*
		1973	* SNB support 3 levels of watermark.
		1974	*
		1975	* WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
		1976	* and disabled in the descending order
		1977	*
		1978	*/
		1979	I915_WRITE(WM3_LP_ILK, 0);
		1980	I915_WRITE(WM2_LP_ILK, 0);
		1981	I915_WRITE(WM1_LP_ILK, 0);
		1982
		1983	if (!single_plane_enabled(enabled) \|\|
		1984	dev_priv->sprite_scaling_enabled)
		1985	return;
		1986	enabled = ffs(enabled) - 1;
		1987
		1988	/* WM1 */
		1989	if (!ironlake_compute_srwm(dev, 1, enabled,
		1990	SNB_READ_WM1_LATENCY() * 500,
		1991	&sandybridge_display_srwm_info,
		1992	&sandybridge_cursor_srwm_info,
		1993	&fbc_wm, &plane_wm, &cursor_wm))
		1994	return;
		1995
		1996	I915_WRITE(WM1_LP_ILK,
		1997	WM1_LP_SR_EN \|
		1998	(SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		1999	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2000	(plane_wm << WM1_LP_SR_SHIFT) \|
		2001	cursor_wm);
		2002
		2003	/* WM2 */
		2004	if (!ironlake_compute_srwm(dev, 2, enabled,
		2005	SNB_READ_WM2_LATENCY() * 500,
		2006	&sandybridge_display_srwm_info,
		2007	&sandybridge_cursor_srwm_info,
		2008	&fbc_wm, &plane_wm, &cursor_wm))
		2009	return;
		2010
		2011	I915_WRITE(WM2_LP_ILK,
		2012	WM2_LP_EN \|
		2013	(SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2014	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2015	(plane_wm << WM1_LP_SR_SHIFT) \|
		2016	cursor_wm);
		2017
3243	Serge	2018	/* WM3, note we have to correct the cursor latency */
3031	serge	2019	if (!ironlake_compute_srwm(dev, 3, enabled,
		2020	SNB_READ_WM3_LATENCY() * 500,
		2021	&sandybridge_display_srwm_info,
		2022	&sandybridge_cursor_srwm_info,
3243	Serge	2023	&fbc_wm, &plane_wm, &ignore_cursor_wm) \|\|
		2024	!ironlake_compute_srwm(dev, 3, enabled,
		2025	2 * SNB_READ_WM3_LATENCY() * 500,
		2026	&sandybridge_display_srwm_info,
		2027	&sandybridge_cursor_srwm_info,
		2028	&ignore_fbc_wm, &ignore_plane_wm, &cursor_wm))
3031	serge	2029	return;
		2030
		2031	I915_WRITE(WM3_LP_ILK,
		2032	WM3_LP_EN \|
		2033	(SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) \|
		2034	(fbc_wm << WM1_LP_FBC_SHIFT) \|
		2035	(plane_wm << WM1_LP_SR_SHIFT) \|
		2036	cursor_wm);
		2037	}
		2038
		2039	static void
		2040	haswell_update_linetime_wm(struct drm_device *dev, int pipe,
		2041	struct drm_display_mode *mode)
		2042	{
		2043	struct drm_i915_private *dev_priv = dev->dev_private;
		2044	u32 temp;
		2045
		2046	temp = I915_READ(PIPE_WM_LINETIME(pipe));
		2047	temp &= ~PIPE_WM_LINETIME_MASK;
		2048
		2049	/* The WM are computed with base on how long it takes to fill a single
		2050	* row at the given clock rate, multiplied by 8.
		2051	* */
		2052	temp \|= PIPE_WM_LINETIME_TIME(
		2053	((mode->crtc_hdisplay * 1000) / mode->clock) * 8);
		2054
		2055	/* IPS watermarks are only used by pipe A, and are ignored by
		2056	* pipes B and C. They are calculated similarly to the common
		2057	* linetime values, except that we are using CD clock frequency
		2058	* in MHz instead of pixel rate for the division.
		2059	*
		2060	* This is a placeholder for the IPS watermark calculation code.
		2061	*/
		2062
		2063	I915_WRITE(PIPE_WM_LINETIME(pipe), temp);
		2064	}
		2065
		2066	static bool
		2067	sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
		2068	uint32_t sprite_width, int pixel_size,
		2069	const struct intel_watermark_params *display,
		2070	int display_latency_ns, int *sprite_wm)
		2071	{
		2072	struct drm_crtc *crtc;
		2073	int clock;
		2074	int entries, tlb_miss;
		2075
		2076	crtc = intel_get_crtc_for_plane(dev, plane);
3243	Serge	2077	if (!intel_crtc_active(crtc)) {
3031	serge	2078	*sprite_wm = display->guard_size;
		2079	return false;
		2080	}
		2081
		2082	clock = crtc->mode.clock;
		2083
		2084	/* Use the small buffer method to calculate the sprite watermark */
		2085	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
		2086	tlb_miss = display->fifo_size*display->cacheline_size -
		2087	sprite_width * 8;
		2088	if (tlb_miss > 0)
		2089	entries += tlb_miss;
		2090	entries = DIV_ROUND_UP(entries, display->cacheline_size);
		2091	*sprite_wm = entries + display->guard_size;
		2092	if (*sprite_wm > (int)display->max_wm)
		2093	*sprite_wm = display->max_wm;
		2094
		2095	return true;
		2096	}
		2097
		2098	static bool
		2099	sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
		2100	uint32_t sprite_width, int pixel_size,
		2101	const struct intel_watermark_params *display,
		2102	int latency_ns, int *sprite_wm)
		2103	{
		2104	struct drm_crtc *crtc;
		2105	unsigned long line_time_us;
		2106	int clock;
		2107	int line_count, line_size;
		2108	int small, large;
		2109	int entries;
		2110
		2111	if (!latency_ns) {
		2112	*sprite_wm = 0;
		2113	return false;
		2114	}
		2115
		2116	crtc = intel_get_crtc_for_plane(dev, plane);
		2117	clock = crtc->mode.clock;
		2118	if (!clock) {
		2119	*sprite_wm = 0;
		2120	return false;
		2121	}
		2122
		2123	line_time_us = (sprite_width * 1000) / clock;
		2124	if (!line_time_us) {
		2125	*sprite_wm = 0;
		2126	return false;
		2127	}
		2128
		2129	line_count = (latency_ns / line_time_us + 1000) / 1000;
		2130	line_size = sprite_width * pixel_size;
		2131
		2132	/* Use the minimum of the small and large buffer method for primary */
		2133	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
		2134	large = line_count * line_size;
		2135
		2136	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
		2137	*sprite_wm = entries + display->guard_size;
		2138
		2139	return *sprite_wm > 0x3ff ? false : true;
		2140	}
		2141
		2142	static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
		2143	uint32_t sprite_width, int pixel_size)
		2144	{
		2145	struct drm_i915_private *dev_priv = dev->dev_private;
		2146	int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */
		2147	u32 val;
		2148	int sprite_wm, reg;
		2149	int ret;
		2150
		2151	switch (pipe) {
		2152	case 0:
		2153	reg = WM0_PIPEA_ILK;
		2154	break;
		2155	case 1:
		2156	reg = WM0_PIPEB_ILK;
		2157	break;
		2158	case 2:
		2159	reg = WM0_PIPEC_IVB;
		2160	break;
		2161	default:
		2162	return; /* bad pipe */
		2163	}
		2164
		2165	ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
		2166	&sandybridge_display_wm_info,
		2167	latency, &sprite_wm);
		2168	if (!ret) {
		2169	DRM_DEBUG_KMS("failed to compute sprite wm for pipe %d\n",
		2170	pipe);
		2171	return;
		2172	}
		2173
		2174	val = I915_READ(reg);
		2175	val &= ~WM0_PIPE_SPRITE_MASK;
		2176	I915_WRITE(reg, val \| (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
		2177	DRM_DEBUG_KMS("sprite watermarks For pipe %d - %d\n", pipe, sprite_wm);
		2178
		2179
		2180	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2181	pixel_size,
		2182	&sandybridge_display_srwm_info,
		2183	SNB_READ_WM1_LATENCY() * 500,
		2184	&sprite_wm);
		2185	if (!ret) {
		2186	DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %d\n",
		2187	pipe);
		2188	return;
		2189	}
		2190	I915_WRITE(WM1S_LP_ILK, sprite_wm);
		2191
		2192	/* Only IVB has two more LP watermarks for sprite */
		2193	if (!IS_IVYBRIDGE(dev))
		2194	return;
		2195
		2196	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2197	pixel_size,
		2198	&sandybridge_display_srwm_info,
		2199	SNB_READ_WM2_LATENCY() * 500,
		2200	&sprite_wm);
		2201	if (!ret) {
		2202	DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %d\n",
		2203	pipe);
		2204	return;
		2205	}
		2206	I915_WRITE(WM2S_LP_IVB, sprite_wm);
		2207
		2208	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
		2209	pixel_size,
		2210	&sandybridge_display_srwm_info,
		2211	SNB_READ_WM3_LATENCY() * 500,
		2212	&sprite_wm);
		2213	if (!ret) {
		2214	DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %d\n",
		2215	pipe);
		2216	return;
		2217	}
		2218	I915_WRITE(WM3S_LP_IVB, sprite_wm);
		2219	}
		2220
		2221	/**
		2222	* intel_update_watermarks - update FIFO watermark values based on current modes
		2223	*
		2224	* Calculate watermark values for the various WM regs based on current mode
		2225	* and plane configuration.
		2226	*
		2227	* There are several cases to deal with here:
		2228	* - normal (i.e. non-self-refresh)
		2229	* - self-refresh (SR) mode
		2230	* - lines are large relative to FIFO size (buffer can hold up to 2)
		2231	* - lines are small relative to FIFO size (buffer can hold more than 2
		2232	* lines), so need to account for TLB latency
		2233	*
		2234	* The normal calculation is:
		2235	* watermark = dotclock * bytes per pixel * latency
		2236	* where latency is platform & configuration dependent (we assume pessimal
		2237	* values here).
		2238	*
		2239	* The SR calculation is:
		2240	* watermark = (trunc(latency/line time)+1) * surface width *
		2241	* bytes per pixel
		2242	* where
		2243	* line time = htotal / dotclock
		2244	* surface width = hdisplay for normal plane and 64 for cursor
		2245	* and latency is assumed to be high, as above.
		2246	*
		2247	* The final value programmed to the register should always be rounded up,
		2248	* and include an extra 2 entries to account for clock crossings.
		2249	*
		2250	* We don't use the sprite, so we can ignore that. And on Crestline we have
		2251	* to set the non-SR watermarks to 8.
		2252	*/
		2253	void intel_update_watermarks(struct drm_device *dev)
		2254	{
		2255	struct drm_i915_private *dev_priv = dev->dev_private;
		2256
		2257	if (dev_priv->display.update_wm)
		2258	dev_priv->display.update_wm(dev);
		2259	}
		2260
		2261	void intel_update_linetime_watermarks(struct drm_device *dev,
		2262	int pipe, struct drm_display_mode *mode)
		2263	{
		2264	struct drm_i915_private *dev_priv = dev->dev_private;
		2265
		2266	if (dev_priv->display.update_linetime_wm)
		2267	dev_priv->display.update_linetime_wm(dev, pipe, mode);
		2268	}
		2269
		2270	void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
		2271	uint32_t sprite_width, int pixel_size)
		2272	{
		2273	struct drm_i915_private *dev_priv = dev->dev_private;
		2274
		2275	if (dev_priv->display.update_sprite_wm)
		2276	dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
		2277	pixel_size);
		2278	}
		2279
		2280	static struct drm_i915_gem_object *
		2281	intel_alloc_context_page(struct drm_device *dev)
		2282	{
		2283	struct drm_i915_gem_object *ctx;
		2284	int ret;
		2285
		2286	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2287
		2288	ctx = i915_gem_alloc_object(dev, 4096);
		2289	if (!ctx) {
		2290	DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
		2291	return NULL;
		2292	}
		2293
		2294	ret = i915_gem_object_pin(ctx, 4096, true, false);
		2295	if (ret) {
		2296	DRM_ERROR("failed to pin power context: %d\n", ret);
		2297	goto err_unref;
		2298	}
		2299
		2300	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
		2301	if (ret) {
		2302	DRM_ERROR("failed to set-domain on power context: %d\n", ret);
		2303	goto err_unpin;
		2304	}
		2305
		2306	return ctx;
		2307
		2308	err_unpin:
		2309	i915_gem_object_unpin(ctx);
		2310	err_unref:
		2311	drm_gem_object_unreference(&ctx->base);
		2312	mutex_unlock(&dev->struct_mutex);
		2313	return NULL;
		2314	}
		2315
		2316	/**
		2317	* Lock protecting IPS related data structures
		2318	*/
		2319	DEFINE_SPINLOCK(mchdev_lock);
		2320
		2321	/* Global for IPS driver to get at the current i915 device. Protected by
		2322	* mchdev_lock. */
		2323	static struct drm_i915_private *i915_mch_dev;
		2324
		2325	bool ironlake_set_drps(struct drm_device *dev, u8 val)
		2326	{
		2327	struct drm_i915_private *dev_priv = dev->dev_private;
		2328	u16 rgvswctl;
		2329
		2330	assert_spin_locked(&mchdev_lock);
		2331
		2332	rgvswctl = I915_READ16(MEMSWCTL);
		2333	if (rgvswctl & MEMCTL_CMD_STS) {
		2334	DRM_DEBUG("gpu busy, RCS change rejected\n");
		2335	return false; /* still busy with another command */
		2336	}
		2337
		2338	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) \|
		2339	(val << MEMCTL_FREQ_SHIFT) \| MEMCTL_SFCAVM;
		2340	I915_WRITE16(MEMSWCTL, rgvswctl);
		2341	POSTING_READ16(MEMSWCTL);
		2342
		2343	rgvswctl \|= MEMCTL_CMD_STS;
		2344	I915_WRITE16(MEMSWCTL, rgvswctl);
		2345
		2346	return true;
		2347	}
		2348
		2349	static void ironlake_enable_drps(struct drm_device *dev)
		2350	{
		2351	struct drm_i915_private *dev_priv = dev->dev_private;
		2352	u32 rgvmodectl = I915_READ(MEMMODECTL);
		2353	u8 fmax, fmin, fstart, vstart;
		2354
		2355	spin_lock_irq(&mchdev_lock);
		2356
		2357	/* Enable temp reporting */
		2358	I915_WRITE16(PMMISC, I915_READ(PMMISC) \| MCPPCE_EN);
		2359	I915_WRITE16(TSC1, I915_READ(TSC1) \| TSE);
		2360
		2361	/* 100ms RC evaluation intervals */
		2362	I915_WRITE(RCUPEI, 100000);
		2363	I915_WRITE(RCDNEI, 100000);
		2364
		2365	/* Set max/min thresholds to 90ms and 80ms respectively */
		2366	I915_WRITE(RCBMAXAVG, 90000);
		2367	I915_WRITE(RCBMINAVG, 80000);
		2368
		2369	I915_WRITE(MEMIHYST, 1);
		2370
		2371	/* Set up min, max, and cur for interrupt handling */
		2372	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
		2373	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
		2374	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
		2375	MEMMODE_FSTART_SHIFT;
		2376
		2377	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
		2378	PXVFREQ_PX_SHIFT;
		2379
		2380	dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
		2381	dev_priv->ips.fstart = fstart;
		2382
		2383	dev_priv->ips.max_delay = fstart;
		2384	dev_priv->ips.min_delay = fmin;
		2385	dev_priv->ips.cur_delay = fstart;
		2386
		2387	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
		2388	fmax, fmin, fstart);
		2389
		2390	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN \| MEMINT_EVAL_CHG_EN);
		2391
		2392	/*
		2393	* Interrupts will be enabled in ironlake_irq_postinstall
		2394	*/
		2395
		2396	I915_WRITE(VIDSTART, vstart);
		2397	POSTING_READ(VIDSTART);
		2398
		2399	rgvmodectl \|= MEMMODE_SWMODE_EN;
		2400	I915_WRITE(MEMMODECTL, rgvmodectl);
		2401
		2402	if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
		2403	DRM_ERROR("stuck trying to change perf mode\n");
		2404	mdelay(1);
		2405
		2406	ironlake_set_drps(dev, fstart);
		2407
		2408	dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
		2409	I915_READ(0x112e0);
		2410	dev_priv->ips.last_time1 = jiffies_to_msecs(GetTimerTicks());
		2411	dev_priv->ips.last_count2 = I915_READ(0x112f4);
		2412	// getrawmonotonic(&dev_priv->ips.last_time2);
		2413
		2414	spin_unlock_irq(&mchdev_lock);
		2415	}
		2416
		2417	static void ironlake_disable_drps(struct drm_device *dev)
		2418	{
		2419	struct drm_i915_private *dev_priv = dev->dev_private;
		2420	u16 rgvswctl;
		2421
		2422	spin_lock_irq(&mchdev_lock);
		2423
		2424	rgvswctl = I915_READ16(MEMSWCTL);
		2425
		2426	/* Ack interrupts, disable EFC interrupt */
		2427	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
		2428	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
		2429	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
		2430	I915_WRITE(DEIIR, DE_PCU_EVENT);
		2431	I915_WRITE(DEIMR, I915_READ(DEIMR) \| DE_PCU_EVENT);
		2432
		2433	/* Go back to the starting frequency */
		2434	ironlake_set_drps(dev, dev_priv->ips.fstart);
		2435	mdelay(1);
		2436	rgvswctl \|= MEMCTL_CMD_STS;
		2437	I915_WRITE(MEMSWCTL, rgvswctl);
		2438	mdelay(1);
		2439
		2440	spin_unlock_irq(&mchdev_lock);
		2441	}
		2442
		2443	/* There's a funny hw issue where the hw returns all 0 when reading from
		2444	* GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
		2445	* ourselves, instead of doing a rmw cycle (which might result in us clearing
		2446	* all limits and the gpu stuck at whatever frequency it is at atm).
		2447	*/
		2448	static u32 gen6_rps_limits(struct drm_i915_private dev_priv, u8 val)
		2449	{
		2450	u32 limits;
		2451
		2452	limits = 0;
		2453
		2454	if (*val >= dev_priv->rps.max_delay)
		2455	*val = dev_priv->rps.max_delay;
		2456	limits \|= dev_priv->rps.max_delay << 24;
		2457
		2458	/* Only set the down limit when we've reached the lowest level to avoid
		2459	* getting more interrupts, otherwise leave this clear. This prevents a
		2460	* race in the hw when coming out of rc6: There's a tiny window where
		2461	* the hw runs at the minimal clock before selecting the desired
		2462	* frequency, if the down threshold expires in that window we will not
		2463	* receive a down interrupt. */
		2464	if (*val <= dev_priv->rps.min_delay) {
		2465	*val = dev_priv->rps.min_delay;
		2466	limits \|= dev_priv->rps.min_delay << 16;
		2467	}
		2468
		2469	return limits;
		2470	}
		2471
		2472	void gen6_set_rps(struct drm_device *dev, u8 val)
		2473	{
		2474	struct drm_i915_private *dev_priv = dev->dev_private;
		2475	u32 limits = gen6_rps_limits(dev_priv, &val);
		2476
3243	Serge	2477	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2478	WARN_ON(val > dev_priv->rps.max_delay);
		2479	WARN_ON(val < dev_priv->rps.min_delay);
		2480
		2481	if (val == dev_priv->rps.cur_delay)
		2482	return;
		2483
		2484	I915_WRITE(GEN6_RPNSWREQ,
		2485	GEN6_FREQUENCY(val) \|
		2486	GEN6_OFFSET(0) \|
		2487	GEN6_AGGRESSIVE_TURBO);
		2488
		2489	/* Make sure we continue to get interrupts
		2490	* until we hit the minimum or maximum frequencies.
		2491	*/
		2492	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
		2493
		2494	POSTING_READ(GEN6_RPNSWREQ);
		2495
		2496	dev_priv->rps.cur_delay = val;
		2497
		2498	trace_intel_gpu_freq_change(val * 50);
		2499	}
		2500
		2501	static void gen6_disable_rps(struct drm_device *dev)
		2502	{
		2503	struct drm_i915_private *dev_priv = dev->dev_private;
		2504
		2505	I915_WRITE(GEN6_RC_CONTROL, 0);
		2506	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
		2507	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
		2508	I915_WRITE(GEN6_PMIER, 0);
		2509	/* Complete PM interrupt masking here doesn't race with the rps work
		2510	* item again unmasking PM interrupts because that is using a different
		2511	* register (PMIMR) to mask PM interrupts. The only risk is in leaving
		2512	* stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
		2513
		2514	spin_lock_irq(&dev_priv->rps.lock);
		2515	dev_priv->rps.pm_iir = 0;
		2516	spin_unlock_irq(&dev_priv->rps.lock);
		2517
		2518	I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
		2519	}
		2520
		2521	int intel_enable_rc6(const struct drm_device *dev)
		2522	{
		2523	/* Respect the kernel parameter if it is set */
		2524	if (i915_enable_rc6 >= 0)
		2525	return i915_enable_rc6;
		2526
3120	serge	2527	/* Disable RC6 on Ironlake */
		2528	if (INTEL_INFO(dev)->gen == 5)
		2529	return 0;
3031	serge	2530
		2531	if (IS_HASWELL(dev)) {
		2532	DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
		2533	return INTEL_RC6_ENABLE;
		2534	}
		2535
		2536	/* snb/ivb have more than one rc6 state. */
		2537	if (INTEL_INFO(dev)->gen == 6) {
		2538	DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
		2539	return INTEL_RC6_ENABLE;
		2540	}
		2541
		2542	DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
		2543	return (INTEL_RC6_ENABLE \| INTEL_RC6p_ENABLE);
		2544	}
		2545
		2546	static void gen6_enable_rps(struct drm_device *dev)
		2547	{
		2548	struct drm_i915_private *dev_priv = dev->dev_private;
		2549	struct intel_ring_buffer *ring;
		2550	u32 rp_state_cap;
		2551	u32 gt_perf_status;
3243	Serge	2552	u32 rc6vids, pcu_mbox, rc6_mask = 0;
3031	serge	2553	u32 gtfifodbg;
		2554	int rc6_mode;
3243	Serge	2555	int i, ret;
3031	serge	2556
3243	Serge	2557	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2558
		2559	/* Here begins a magic sequence of register writes to enable
		2560	* auto-downclocking.
		2561	*
		2562	* Perhaps there might be some value in exposing these to
		2563	* userspace...
		2564	*/
		2565	I915_WRITE(GEN6_RC_STATE, 0);
		2566
		2567	/* Clear the DBG now so we don't confuse earlier errors */
		2568	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
		2569	DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
		2570	I915_WRITE(GTFIFODBG, gtfifodbg);
		2571	}
		2572
		2573	gen6_gt_force_wake_get(dev_priv);
		2574
		2575	rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
		2576	gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
		2577
		2578	/* In units of 100MHz */
		2579	dev_priv->rps.max_delay = rp_state_cap & 0xff;
		2580	dev_priv->rps.min_delay = (rp_state_cap & 0xff0000) >> 16;
		2581	dev_priv->rps.cur_delay = 0;
		2582
		2583	/* disable the counters and set deterministic thresholds */
		2584	I915_WRITE(GEN6_RC_CONTROL, 0);
		2585
		2586	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
		2587	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 \| 30);
		2588	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
		2589	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
		2590	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
		2591
		2592	for_each_ring(ring, dev_priv, i)
		2593	I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
		2594
		2595	I915_WRITE(GEN6_RC_SLEEP, 0);
		2596	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
		2597	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
		2598	I915_WRITE(GEN6_RC6p_THRESHOLD, 100000);
		2599	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
		2600
		2601	/* Check if we are enabling RC6 */
		2602	rc6_mode = intel_enable_rc6(dev_priv->dev);
		2603	if (rc6_mode & INTEL_RC6_ENABLE)
		2604	rc6_mask \|= GEN6_RC_CTL_RC6_ENABLE;
		2605
		2606	/* We don't use those on Haswell */
		2607	if (!IS_HASWELL(dev)) {
		2608	if (rc6_mode & INTEL_RC6p_ENABLE)
		2609	rc6_mask \|= GEN6_RC_CTL_RC6p_ENABLE;
		2610
		2611	if (rc6_mode & INTEL_RC6pp_ENABLE)
		2612	rc6_mask \|= GEN6_RC_CTL_RC6pp_ENABLE;
		2613	}
		2614
		2615	DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
		2616	(rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
		2617	(rc6_mask & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
		2618	(rc6_mask & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
		2619
		2620	I915_WRITE(GEN6_RC_CONTROL,
		2621	rc6_mask \|
		2622	GEN6_RC_CTL_EI_MODE(1) \|
		2623	GEN6_RC_CTL_HW_ENABLE);
		2624
		2625	I915_WRITE(GEN6_RPNSWREQ,
		2626	GEN6_FREQUENCY(10) \|
		2627	GEN6_OFFSET(0) \|
		2628	GEN6_AGGRESSIVE_TURBO);
		2629	I915_WRITE(GEN6_RC_VIDEO_FREQ,
		2630	GEN6_FREQUENCY(12));
		2631
		2632	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
		2633	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
		2634	dev_priv->rps.max_delay << 24 \|
		2635	dev_priv->rps.min_delay << 16);
		2636
		2637	I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
		2638	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
		2639	I915_WRITE(GEN6_RP_UP_EI, 66000);
		2640	I915_WRITE(GEN6_RP_DOWN_EI, 350000);
		2641
		2642	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
		2643	I915_WRITE(GEN6_RP_CONTROL,
		2644	GEN6_RP_MEDIA_TURBO \|
		2645	GEN6_RP_MEDIA_HW_NORMAL_MODE \|
		2646	GEN6_RP_MEDIA_IS_GFX \|
		2647	GEN6_RP_ENABLE \|
		2648	GEN6_RP_UP_BUSY_AVG \|
		2649	(IS_HASWELL(dev) ? GEN7_RP_DOWN_IDLE_AVG : GEN6_RP_DOWN_IDLE_CONT));
		2650
3243	Serge	2651	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
		2652	if (!ret) {
		2653	pcu_mbox = 0;
		2654	ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
		2655	if (ret && pcu_mbox & (1<<31)) { /* OC supported */
3031	serge	2656	dev_priv->rps.max_delay = pcu_mbox & 0xff;
		2657	DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
		2658	}
3243	Serge	2659	} else {
		2660	DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
		2661	}
3031	serge	2662
		2663	gen6_set_rps(dev_priv->dev, (gt_perf_status & 0xff00) >> 8);
		2664
		2665	/* requires MSI enabled */
		2666	I915_WRITE(GEN6_PMIER, GEN6_PM_DEFERRED_EVENTS);
		2667	spin_lock_irq(&dev_priv->rps.lock);
		2668	WARN_ON(dev_priv->rps.pm_iir != 0);
		2669	I915_WRITE(GEN6_PMIMR, 0);
		2670	spin_unlock_irq(&dev_priv->rps.lock);
		2671	/* enable all PM interrupts */
		2672	I915_WRITE(GEN6_PMINTRMSK, 0);
		2673
3243	Serge	2674	rc6vids = 0;
		2675	ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
		2676	if (IS_GEN6(dev) && ret) {
		2677	DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
		2678	} else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
		2679	DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
		2680	GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
		2681	rc6vids &= 0xffff00;
		2682	rc6vids \|= GEN6_ENCODE_RC6_VID(450);
		2683	ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
		2684	if (ret)
		2685	DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
		2686	}
		2687
3031	serge	2688	gen6_gt_force_wake_put(dev_priv);
		2689	}
		2690
		2691	#if 0
		2692	static void gen6_update_ring_freq(struct drm_device *dev)
		2693	{
		2694	struct drm_i915_private *dev_priv = dev->dev_private;
		2695	int min_freq = 15;
3243	Serge	2696	int gpu_freq;
		2697	unsigned int ia_freq, max_ia_freq;
3031	serge	2698	int scaling_factor = 180;
		2699
3243	Serge	2700	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	2701
		2702	max_ia_freq = cpufreq_quick_get_max(0);
		2703	/*
		2704	* Default to measured freq if none found, PCU will ensure we don't go
		2705	* over
		2706	*/
		2707	if (!max_ia_freq)
		2708	max_ia_freq = tsc_khz;
		2709
		2710	/* Convert from kHz to MHz */
		2711	max_ia_freq /= 1000;
		2712
		2713	/*
		2714	* For each potential GPU frequency, load a ring frequency we'd like
		2715	* to use for memory access. We do this by specifying the IA frequency
		2716	* the PCU should use as a reference to determine the ring frequency.
		2717	*/
		2718	for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
		2719	gpu_freq--) {
		2720	int diff = dev_priv->rps.max_delay - gpu_freq;
		2721
		2722	/*
		2723	* For GPU frequencies less than 750MHz, just use the lowest
		2724	* ring freq.
		2725	*/
		2726	if (gpu_freq < min_freq)
		2727	ia_freq = 800;
		2728	else
		2729	ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
		2730	ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
3243	Serge	2731	ia_freq <<= GEN6_PCODE_FREQ_IA_RATIO_SHIFT;
3031	serge	2732
3243	Serge	2733	sandybridge_pcode_write(dev_priv,
		2734	GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
		2735	ia_freq \| gpu_freq);
3031	serge	2736	}
		2737	}
		2738	#endif
		2739
		2740	void ironlake_teardown_rc6(struct drm_device *dev)
		2741	{
		2742	struct drm_i915_private *dev_priv = dev->dev_private;
		2743
3243	Serge	2744	if (dev_priv->ips.renderctx) {
		2745	i915_gem_object_unpin(dev_priv->ips.renderctx);
		2746	drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
		2747	dev_priv->ips.renderctx = NULL;
3031	serge	2748	}
		2749
3243	Serge	2750	if (dev_priv->ips.pwrctx) {
		2751	i915_gem_object_unpin(dev_priv->ips.pwrctx);
		2752	drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
		2753	dev_priv->ips.pwrctx = NULL;
3031	serge	2754	}
		2755	}
		2756
		2757	static void ironlake_disable_rc6(struct drm_device *dev)
		2758	{
		2759	struct drm_i915_private *dev_priv = dev->dev_private;
		2760
		2761	if (I915_READ(PWRCTXA)) {
		2762	/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
		2763	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) \| RCX_SW_EXIT);
		2764	wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
		2765	50);
		2766
		2767	I915_WRITE(PWRCTXA, 0);
		2768	POSTING_READ(PWRCTXA);
		2769
		2770	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		2771	POSTING_READ(RSTDBYCTL);
		2772	}
		2773	}
		2774
		2775	static int ironlake_setup_rc6(struct drm_device *dev)
		2776	{
		2777	struct drm_i915_private *dev_priv = dev->dev_private;
		2778
3243	Serge	2779	if (dev_priv->ips.renderctx == NULL)
		2780	dev_priv->ips.renderctx = intel_alloc_context_page(dev);
		2781	if (!dev_priv->ips.renderctx)
3031	serge	2782	return -ENOMEM;
		2783
3243	Serge	2784	if (dev_priv->ips.pwrctx == NULL)
		2785	dev_priv->ips.pwrctx = intel_alloc_context_page(dev);
		2786	if (!dev_priv->ips.pwrctx) {
3031	serge	2787	ironlake_teardown_rc6(dev);
		2788	return -ENOMEM;
		2789	}
		2790
		2791	return 0;
		2792	}
		2793
		2794	static void ironlake_enable_rc6(struct drm_device *dev)
		2795	{
		2796	struct drm_i915_private *dev_priv = dev->dev_private;
		2797	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
3243	Serge	2798	bool was_interruptible;
3031	serge	2799	int ret;
		2800
		2801	/* rc6 disabled by default due to repeated reports of hanging during
		2802	* boot and resume.
		2803	*/
		2804	if (!intel_enable_rc6(dev))
		2805	return;
		2806
		2807	WARN_ON(!mutex_is_locked(&dev->struct_mutex));
		2808
		2809	ret = ironlake_setup_rc6(dev);
		2810	if (ret)
		2811	return;
		2812
3243	Serge	2813	was_interruptible = dev_priv->mm.interruptible;
		2814	dev_priv->mm.interruptible = false;
		2815
3031	serge	2816	/*
		2817	* GPU can automatically power down the render unit if given a page
		2818	* to save state.
		2819	*/
		2820	ret = intel_ring_begin(ring, 6);
		2821	if (ret) {
		2822	ironlake_teardown_rc6(dev);
3243	Serge	2823	dev_priv->mm.interruptible = was_interruptible;
3031	serge	2824	return;
		2825	}
		2826
		2827	intel_ring_emit(ring, MI_SUSPEND_FLUSH \| MI_SUSPEND_FLUSH_EN);
		2828	intel_ring_emit(ring, MI_SET_CONTEXT);
3243	Serge	2829	intel_ring_emit(ring, dev_priv->ips.renderctx->gtt_offset \|
3031	serge	2830	MI_MM_SPACE_GTT \|
		2831	MI_SAVE_EXT_STATE_EN \|
		2832	MI_RESTORE_EXT_STATE_EN \|
		2833	MI_RESTORE_INHIBIT);
		2834	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
		2835	intel_ring_emit(ring, MI_NOOP);
		2836	intel_ring_emit(ring, MI_FLUSH);
		2837	intel_ring_advance(ring);
		2838
		2839	/*
		2840	* Wait for the command parser to advance past MI_SET_CONTEXT. The HW
		2841	* does an implicit flush, combined with MI_FLUSH above, it should be
		2842	* safe to assume that renderctx is valid
		2843	*/
3243	Serge	2844	ret = intel_ring_idle(ring);
		2845	dev_priv->mm.interruptible = was_interruptible;
3031	serge	2846	if (ret) {
		2847	DRM_ERROR("failed to enable ironlake power power savings\n");
		2848	ironlake_teardown_rc6(dev);
		2849	return;
		2850	}
		2851
3243	Serge	2852	I915_WRITE(PWRCTXA, dev_priv->ips.pwrctx->gtt_offset \| PWRCTX_EN);
3031	serge	2853	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
		2854	}
		2855
		2856	static unsigned long intel_pxfreq(u32 vidfreq)
		2857	{
		2858	unsigned long freq;
		2859	int div = (vidfreq & 0x3f0000) >> 16;
		2860	int post = (vidfreq & 0x3000) >> 12;
		2861	int pre = (vidfreq & 0x7);
		2862
		2863	if (!pre)
		2864	return 0;
		2865
		2866	freq = ((div * 133333) / ((1<
		2867
		2868	return freq;
		2869	}
		2870
		2871	static const struct cparams {
		2872	u16 i;
		2873	u16 t;
		2874	u16 m;
		2875	u16 c;
		2876	} cparams[] = {
		2877	{ 1, 1333, 301, 28664 },
		2878	{ 1, 1066, 294, 24460 },
		2879	{ 1, 800, 294, 25192 },
		2880	{ 0, 1333, 276, 27605 },
		2881	{ 0, 1066, 276, 27605 },
		2882	{ 0, 800, 231, 23784 },
		2883	};
		2884
		2885	static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
		2886	{
		2887	u64 total_count, diff, ret;
		2888	u32 count1, count2, count3, m = 0, c = 0;
		2889	unsigned long now = jiffies_to_msecs(GetTimerTicks()), diff1;
		2890	int i;
		2891
		2892	assert_spin_locked(&mchdev_lock);
		2893
		2894	diff1 = now - dev_priv->ips.last_time1;
		2895
		2896	/* Prevent division-by-zero if we are asking too fast.
		2897	* Also, we don't get interesting results if we are polling
		2898	* faster than once in 10ms, so just return the saved value
		2899	* in such cases.
		2900	*/
		2901	if (diff1 <= 10)
		2902	return dev_priv->ips.chipset_power;
		2903
		2904	count1 = I915_READ(DMIEC);
		2905	count2 = I915_READ(DDREC);
		2906	count3 = I915_READ(CSIEC);
		2907
		2908	total_count = count1 + count2 + count3;
		2909
		2910	/* FIXME: handle per-counter overflow */
		2911	if (total_count < dev_priv->ips.last_count1) {
		2912	diff = ~0UL - dev_priv->ips.last_count1;
		2913	diff += total_count;
		2914	} else {
		2915	diff = total_count - dev_priv->ips.last_count1;
		2916	}
		2917
		2918	for (i = 0; i < ARRAY_SIZE(cparams); i++) {
		2919	if (cparams[i].i == dev_priv->ips.c_m &&
		2920	cparams[i].t == dev_priv->ips.r_t) {
		2921	m = cparams[i].m;
		2922	c = cparams[i].c;
		2923	break;
		2924	}
		2925	}
		2926
		2927	diff = div_u64(diff, diff1);
		2928	ret = ((m * diff) + c);
		2929	ret = div_u64(ret, 10);
		2930
		2931	dev_priv->ips.last_count1 = total_count;
		2932	dev_priv->ips.last_time1 = now;
		2933
		2934	dev_priv->ips.chipset_power = ret;
		2935
		2936	return ret;
		2937	}
		2938
		2939	unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
		2940	{
		2941	unsigned long val;
		2942
		2943	if (dev_priv->info->gen != 5)
		2944	return 0;
		2945
		2946	spin_lock_irq(&mchdev_lock);
		2947
		2948	val = __i915_chipset_val(dev_priv);
		2949
		2950	spin_unlock_irq(&mchdev_lock);
		2951
		2952	return val;
		2953	}
		2954
		2955	unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
		2956	{
		2957	unsigned long m, x, b;
		2958	u32 tsfs;
		2959
		2960	tsfs = I915_READ(TSFS);
		2961
		2962	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
		2963	x = I915_READ8(TR1);
		2964
		2965	b = tsfs & TSFS_INTR_MASK;
		2966
		2967	return ((m * x) / 127) - b;
		2968	}
		2969
		2970	static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
		2971	{
		2972	static const struct v_table {
		2973	u16 vd; /* in .1 mil */
		2974	u16 vm; /* in .1 mil */
		2975	} v_table[] = {
		2976	{ 0, 0, },
		2977	{ 375, 0, },
		2978	{ 500, 0, },
		2979	{ 625, 0, },
		2980	{ 750, 0, },
		2981	{ 875, 0, },
		2982	{ 1000, 0, },
		2983	{ 1125, 0, },
		2984	{ 4125, 3000, },
		2985	{ 4125, 3000, },
		2986	{ 4125, 3000, },
		2987	{ 4125, 3000, },
		2988	{ 4125, 3000, },
		2989	{ 4125, 3000, },
		2990	{ 4125, 3000, },
		2991	{ 4125, 3000, },
		2992	{ 4125, 3000, },
		2993	{ 4125, 3000, },
		2994	{ 4125, 3000, },
		2995	{ 4125, 3000, },
		2996	{ 4125, 3000, },
		2997	{ 4125, 3000, },
		2998	{ 4125, 3000, },
		2999	{ 4125, 3000, },
		3000	{ 4125, 3000, },
		3001	{ 4125, 3000, },
		3002	{ 4125, 3000, },
		3003	{ 4125, 3000, },
		3004	{ 4125, 3000, },
		3005	{ 4125, 3000, },
		3006	{ 4125, 3000, },
		3007	{ 4125, 3000, },
		3008	{ 4250, 3125, },
		3009	{ 4375, 3250, },
		3010	{ 4500, 3375, },
		3011	{ 4625, 3500, },
		3012	{ 4750, 3625, },
		3013	{ 4875, 3750, },
		3014	{ 5000, 3875, },
		3015	{ 5125, 4000, },
		3016	{ 5250, 4125, },
		3017	{ 5375, 4250, },
		3018	{ 5500, 4375, },
		3019	{ 5625, 4500, },
		3020	{ 5750, 4625, },
		3021	{ 5875, 4750, },
		3022	{ 6000, 4875, },
		3023	{ 6125, 5000, },
		3024	{ 6250, 5125, },
		3025	{ 6375, 5250, },
		3026	{ 6500, 5375, },
		3027	{ 6625, 5500, },
		3028	{ 6750, 5625, },
		3029	{ 6875, 5750, },
		3030	{ 7000, 5875, },
		3031	{ 7125, 6000, },
		3032	{ 7250, 6125, },
		3033	{ 7375, 6250, },
		3034	{ 7500, 6375, },
		3035	{ 7625, 6500, },
		3036	{ 7750, 6625, },
		3037	{ 7875, 6750, },
		3038	{ 8000, 6875, },
		3039	{ 8125, 7000, },
		3040	{ 8250, 7125, },
		3041	{ 8375, 7250, },
		3042	{ 8500, 7375, },
		3043	{ 8625, 7500, },
		3044	{ 8750, 7625, },
		3045	{ 8875, 7750, },
		3046	{ 9000, 7875, },
		3047	{ 9125, 8000, },
		3048	{ 9250, 8125, },
		3049	{ 9375, 8250, },
		3050	{ 9500, 8375, },
		3051	{ 9625, 8500, },
		3052	{ 9750, 8625, },
		3053	{ 9875, 8750, },
		3054	{ 10000, 8875, },
		3055	{ 10125, 9000, },
		3056	{ 10250, 9125, },
		3057	{ 10375, 9250, },
		3058	{ 10500, 9375, },
		3059	{ 10625, 9500, },
		3060	{ 10750, 9625, },
		3061	{ 10875, 9750, },
		3062	{ 11000, 9875, },
		3063	{ 11125, 10000, },
		3064	{ 11250, 10125, },
		3065	{ 11375, 10250, },
		3066	{ 11500, 10375, },
		3067	{ 11625, 10500, },
		3068	{ 11750, 10625, },
		3069	{ 11875, 10750, },
		3070	{ 12000, 10875, },
		3071	{ 12125, 11000, },
		3072	{ 12250, 11125, },
		3073	{ 12375, 11250, },
		3074	{ 12500, 11375, },
		3075	{ 12625, 11500, },
		3076	{ 12750, 11625, },
		3077	{ 12875, 11750, },
		3078	{ 13000, 11875, },
		3079	{ 13125, 12000, },
		3080	{ 13250, 12125, },
		3081	{ 13375, 12250, },
		3082	{ 13500, 12375, },
		3083	{ 13625, 12500, },
		3084	{ 13750, 12625, },
		3085	{ 13875, 12750, },
		3086	{ 14000, 12875, },
		3087	{ 14125, 13000, },
		3088	{ 14250, 13125, },
		3089	{ 14375, 13250, },
		3090	{ 14500, 13375, },
		3091	{ 14625, 13500, },
		3092	{ 14750, 13625, },
		3093	{ 14875, 13750, },
		3094	{ 15000, 13875, },
		3095	{ 15125, 14000, },
		3096	{ 15250, 14125, },
		3097	{ 15375, 14250, },
		3098	{ 15500, 14375, },
		3099	{ 15625, 14500, },
		3100	{ 15750, 14625, },
		3101	{ 15875, 14750, },
		3102	{ 16000, 14875, },
		3103	{ 16125, 15000, },
		3104	};
		3105	if (dev_priv->info->is_mobile)
		3106	return v_table[pxvid].vm;
		3107	else
		3108	return v_table[pxvid].vd;
		3109	}
		3110
		3111	static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
		3112	{
		3113	struct timespec now, diff1;
		3114	u64 diff;
		3115	unsigned long diffms;
		3116	u32 count;
		3117
		3118	assert_spin_locked(&mchdev_lock);
		3119
		3120	getrawmonotonic(&now);
		3121	diff1 = timespec_sub(now, dev_priv->ips.last_time2);
		3122
		3123	/* Don't divide by 0 */
		3124	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
		3125	if (!diffms)
		3126	return;
		3127
		3128	count = I915_READ(GFXEC);
		3129
		3130	if (count < dev_priv->ips.last_count2) {
		3131	diff = ~0UL - dev_priv->ips.last_count2;
		3132	diff += count;
		3133	} else {
		3134	diff = count - dev_priv->ips.last_count2;
		3135	}
		3136
		3137	dev_priv->ips.last_count2 = count;
		3138	dev_priv->ips.last_time2 = now;
		3139
		3140	/* More magic constants... */
		3141	diff = diff * 1181;
		3142	diff = div_u64(diff, diffms * 10);
		3143	dev_priv->ips.gfx_power = diff;
		3144	}
		3145
		3146	void i915_update_gfx_val(struct drm_i915_private *dev_priv)
		3147	{
		3148	if (dev_priv->info->gen != 5)
		3149	return;
		3150
		3151	spin_lock_irq(&mchdev_lock);
		3152
		3153	__i915_update_gfx_val(dev_priv);
		3154
		3155	spin_unlock_irq(&mchdev_lock);
		3156	}
		3157
		3158	static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
		3159	{
		3160	unsigned long t, corr, state1, corr2, state2;
		3161	u32 pxvid, ext_v;
		3162
		3163	assert_spin_locked(&mchdev_lock);
		3164
		3165	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
		3166	pxvid = (pxvid >> 24) & 0x7f;
		3167	ext_v = pvid_to_extvid(dev_priv, pxvid);
		3168
		3169	state1 = ext_v;
		3170
		3171	t = i915_mch_val(dev_priv);
		3172
		3173	/* Revel in the empirically derived constants */
		3174
		3175	/* Correction factor in 1/100000 units */
		3176	if (t > 80)
		3177	corr = ((t * 2349) + 135940);
		3178	else if (t >= 50)
		3179	corr = ((t * 964) + 29317);
		3180	else /* < 50 */
		3181	corr = ((t * 301) + 1004);
		3182
		3183	corr = corr * ((150142 * state1) / 10000 - 78642);
		3184	corr /= 100000;
		3185	corr2 = (corr * dev_priv->ips.corr);
		3186
		3187	state2 = (corr2 * state1) / 10000;
		3188	state2 /= 100; /* convert to mW */
		3189
		3190	__i915_update_gfx_val(dev_priv);
		3191
		3192	return dev_priv->ips.gfx_power + state2;
		3193	}
		3194
		3195	unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
		3196	{
		3197	unsigned long val;
		3198
		3199	if (dev_priv->info->gen != 5)
		3200	return 0;
		3201
		3202	spin_lock_irq(&mchdev_lock);
		3203
		3204	val = __i915_gfx_val(dev_priv);
		3205
		3206	spin_unlock_irq(&mchdev_lock);
		3207
		3208	return val;
		3209	}
		3210
		3211	/**
		3212	* i915_read_mch_val - return value for IPS use
		3213	*
		3214	* Calculate and return a value for the IPS driver to use when deciding whether
		3215	* we have thermal and power headroom to increase CPU or GPU power budget.
		3216	*/
		3217	unsigned long i915_read_mch_val(void)
		3218	{
		3219	struct drm_i915_private *dev_priv;
		3220	unsigned long chipset_val, graphics_val, ret = 0;
		3221
		3222	spin_lock_irq(&mchdev_lock);
		3223	if (!i915_mch_dev)
		3224	goto out_unlock;
		3225	dev_priv = i915_mch_dev;
		3226
		3227	chipset_val = __i915_chipset_val(dev_priv);
		3228	graphics_val = __i915_gfx_val(dev_priv);
		3229
		3230	ret = chipset_val + graphics_val;
		3231
		3232	out_unlock:
		3233	spin_unlock_irq(&mchdev_lock);
		3234
		3235	return ret;
		3236	}
		3237	EXPORT_SYMBOL_GPL(i915_read_mch_val);
		3238
		3239	/**
		3240	* i915_gpu_raise - raise GPU frequency limit
		3241	*
		3242	* Raise the limit; IPS indicates we have thermal headroom.
		3243	*/
		3244	bool i915_gpu_raise(void)
		3245	{
		3246	struct drm_i915_private *dev_priv;
		3247	bool ret = true;
		3248
		3249	spin_lock_irq(&mchdev_lock);
		3250	if (!i915_mch_dev) {
		3251	ret = false;
		3252	goto out_unlock;
		3253	}
		3254	dev_priv = i915_mch_dev;
		3255
		3256	if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
		3257	dev_priv->ips.max_delay--;
		3258
		3259	out_unlock:
		3260	spin_unlock_irq(&mchdev_lock);
		3261
		3262	return ret;
		3263	}
		3264	EXPORT_SYMBOL_GPL(i915_gpu_raise);
		3265
		3266	/**
		3267	* i915_gpu_lower - lower GPU frequency limit
		3268	*
		3269	* IPS indicates we're close to a thermal limit, so throttle back the GPU
		3270	* frequency maximum.
		3271	*/
		3272	bool i915_gpu_lower(void)
		3273	{
		3274	struct drm_i915_private *dev_priv;
		3275	bool ret = true;
		3276
		3277	spin_lock_irq(&mchdev_lock);
		3278	if (!i915_mch_dev) {
		3279	ret = false;
		3280	goto out_unlock;
		3281	}
		3282	dev_priv = i915_mch_dev;
		3283
		3284	if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
		3285	dev_priv->ips.max_delay++;
		3286
		3287	out_unlock:
		3288	spin_unlock_irq(&mchdev_lock);
		3289
		3290	return ret;
		3291	}
		3292	EXPORT_SYMBOL_GPL(i915_gpu_lower);
		3293
		3294	/**
		3295	* i915_gpu_busy - indicate GPU business to IPS
		3296	*
		3297	* Tell the IPS driver whether or not the GPU is busy.
		3298	*/
		3299	bool i915_gpu_busy(void)
		3300	{
		3301	struct drm_i915_private *dev_priv;
		3302	struct intel_ring_buffer *ring;
		3303	bool ret = false;
		3304	int i;
		3305
		3306	spin_lock_irq(&mchdev_lock);
		3307	if (!i915_mch_dev)
		3308	goto out_unlock;
		3309	dev_priv = i915_mch_dev;
		3310
		3311	for_each_ring(ring, dev_priv, i)
		3312	ret \|= !list_empty(&ring->request_list);
		3313
		3314	out_unlock:
		3315	spin_unlock_irq(&mchdev_lock);
		3316
		3317	return ret;
		3318	}
		3319	EXPORT_SYMBOL_GPL(i915_gpu_busy);
		3320
		3321	/**
		3322	* i915_gpu_turbo_disable - disable graphics turbo
		3323	*
		3324	* Disable graphics turbo by resetting the max frequency and setting the
		3325	* current frequency to the default.
		3326	*/
		3327	bool i915_gpu_turbo_disable(void)
		3328	{
		3329	struct drm_i915_private *dev_priv;
		3330	bool ret = true;
		3331
		3332	spin_lock_irq(&mchdev_lock);
		3333	if (!i915_mch_dev) {
		3334	ret = false;
		3335	goto out_unlock;
		3336	}
		3337	dev_priv = i915_mch_dev;
		3338
		3339	dev_priv->ips.max_delay = dev_priv->ips.fstart;
		3340
		3341	if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
		3342	ret = false;
		3343
		3344	out_unlock:
		3345	spin_unlock_irq(&mchdev_lock);
		3346
		3347	return ret;
		3348	}
		3349	EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
		3350
		3351	/**
		3352	* Tells the intel_ips driver that the i915 driver is now loaded, if
		3353	* IPS got loaded first.
		3354	*
		3355	* This awkward dance is so that neither module has to depend on the
		3356	* other in order for IPS to do the appropriate communication of
		3357	* GPU turbo limits to i915.
		3358	*/
		3359	static void
		3360	ips_ping_for_i915_load(void)
		3361	{
		3362	void (*link)(void);
		3363
		3364	// link = symbol_get(ips_link_to_i915_driver);
		3365	// if (link) {
		3366	// link();
		3367	// symbol_put(ips_link_to_i915_driver);
		3368	// }
		3369	}
		3370
		3371	void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
		3372	{
		3373	/* We only register the i915 ips part with intel-ips once everything is
		3374	* set up, to avoid intel-ips sneaking in and reading bogus values. */
		3375	spin_lock_irq(&mchdev_lock);
		3376	i915_mch_dev = dev_priv;
		3377	spin_unlock_irq(&mchdev_lock);
		3378
		3379	ips_ping_for_i915_load();
		3380	}
		3381
		3382	void intel_gpu_ips_teardown(void)
		3383	{
		3384	spin_lock_irq(&mchdev_lock);
		3385	i915_mch_dev = NULL;
		3386	spin_unlock_irq(&mchdev_lock);
		3387	}
		3388	static void intel_init_emon(struct drm_device *dev)
		3389	{
		3390	struct drm_i915_private *dev_priv = dev->dev_private;
		3391	u32 lcfuse;
		3392	u8 pxw[16];
		3393	int i;
		3394
		3395	/* Disable to program */
		3396	I915_WRITE(ECR, 0);
		3397	POSTING_READ(ECR);
		3398
		3399	/* Program energy weights for various events */
		3400	I915_WRITE(SDEW, 0x15040d00);
		3401	I915_WRITE(CSIEW0, 0x007f0000);
		3402	I915_WRITE(CSIEW1, 0x1e220004);
		3403	I915_WRITE(CSIEW2, 0x04000004);
		3404
		3405	for (i = 0; i < 5; i++)
		3406	I915_WRITE(PEW + (i * 4), 0);
		3407	for (i = 0; i < 3; i++)
		3408	I915_WRITE(DEW + (i * 4), 0);
		3409
		3410	/* Program P-state weights to account for frequency power adjustment */
		3411	for (i = 0; i < 16; i++) {
		3412	u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
		3413	unsigned long freq = intel_pxfreq(pxvidfreq);
		3414	unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
		3415	PXVFREQ_PX_SHIFT;
		3416	unsigned long val;
		3417
		3418	val = vid * vid;
		3419	val *= (freq / 1000);
		3420	val *= 255;
		3421	val /= (127127900);
		3422	if (val > 0xff)
		3423	DRM_ERROR("bad pxval: %ld\n", val);
		3424	pxw[i] = val;
		3425	}
		3426	/* Render standby states get 0 weight */
		3427	pxw[14] = 0;
		3428	pxw[15] = 0;
		3429
		3430	for (i = 0; i < 4; i++) {
		3431	u32 val = (pxw[i4] << 24) \| (pxw[(i4)+1] << 16) \|
		3432	(pxw[(i4)+2] << 8) \| (pxw[(i4)+3]);
		3433	I915_WRITE(PXW + (i * 4), val);
		3434	}
		3435
		3436	/* Adjust magic regs to magic values (more experimental results) */
		3437	I915_WRITE(OGW0, 0);
		3438	I915_WRITE(OGW1, 0);
		3439	I915_WRITE(EG0, 0x00007f00);
		3440	I915_WRITE(EG1, 0x0000000e);
		3441	I915_WRITE(EG2, 0x000e0000);
		3442	I915_WRITE(EG3, 0x68000300);
		3443	I915_WRITE(EG4, 0x42000000);
		3444	I915_WRITE(EG5, 0x00140031);
		3445	I915_WRITE(EG6, 0);
		3446	I915_WRITE(EG7, 0);
		3447
		3448	for (i = 0; i < 8; i++)
		3449	I915_WRITE(PXWL + (i * 4), 0);
		3450
		3451	/* Enable PMON + select events */
		3452	I915_WRITE(ECR, 0x80000019);
		3453
		3454	lcfuse = I915_READ(LCFUSE02);
		3455
		3456	dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
		3457	}
		3458
		3459	void intel_disable_gt_powersave(struct drm_device *dev)
		3460	{
3243	Serge	3461	struct drm_i915_private *dev_priv = dev->dev_private;
		3462
3031	serge	3463	if (IS_IRONLAKE_M(dev)) {
		3464	ironlake_disable_drps(dev);
		3465	ironlake_disable_rc6(dev);
		3466	} else if (INTEL_INFO(dev)->gen >= 6 && !IS_VALLEYVIEW(dev)) {
		3467	gen6_disable_rps(dev);
		3468	}
		3469	}
		3470
		3471	void intel_enable_gt_powersave(struct drm_device *dev)
		3472	{
3243	Serge	3473	struct drm_i915_private *dev_priv = dev->dev_private;
		3474
3031	serge	3475	if (IS_IRONLAKE_M(dev)) {
		3476	ironlake_enable_drps(dev);
		3477	ironlake_enable_rc6(dev);
		3478	intel_init_emon(dev);
		3479	} else if ((IS_GEN6(dev) \|\| IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) {
3243	Serge	3480	/*
		3481	* PCU communication is slow and this doesn't need to be
		3482	* done at any specific time, so do this out of our fast path
		3483	* to make resume and init faster.
		3484	*/
		3485	// schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
		3486	// round_jiffies_up_relative(HZ));
3031	serge	3487	}
		3488	}
		3489
3243	Serge	3490	static void ibx_init_clock_gating(struct drm_device *dev)
		3491	{
		3492	struct drm_i915_private *dev_priv = dev->dev_private;
		3493
		3494	/*
		3495	* On Ibex Peak and Cougar Point, we need to disable clock
		3496	* gating for the panel power sequencer or it will fail to
		3497	* start up when no ports are active.
		3498	*/
		3499	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		3500	}
		3501
3031	serge	3502	static void ironlake_init_clock_gating(struct drm_device *dev)
		3503	{
		3504	struct drm_i915_private *dev_priv = dev->dev_private;
3243	Serge	3505	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	3506
		3507	/* Required for FBC */
3243	Serge	3508	dspclk_gate \|= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE \|
		3509	ILK_DPFCUNIT_CLOCK_GATE_DISABLE \|
		3510	ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
3031	serge	3511
		3512	I915_WRITE(PCH_3DCGDIS0,
		3513	MARIUNIT_CLOCK_GATE_DISABLE \|
		3514	SVSMUNIT_CLOCK_GATE_DISABLE);
		3515	I915_WRITE(PCH_3DCGDIS1,
		3516	VFMUNIT_CLOCK_GATE_DISABLE);
		3517
		3518	/*
		3519	* According to the spec the following bits should be set in
		3520	* order to enable memory self-refresh
		3521	* The bit 22/21 of 0x42004
		3522	* The bit 5 of 0x42020
		3523	* The bit 15 of 0x45000
		3524	*/
		3525	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3526	(I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3527	ILK_DPARB_GATE \| ILK_VSDPFD_FULL));
3243	Serge	3528	dspclk_gate \|= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
3031	serge	3529	I915_WRITE(DISP_ARB_CTL,
		3530	(I915_READ(DISP_ARB_CTL) \|
		3531	DISP_FBC_WM_DIS));
		3532	I915_WRITE(WM3_LP_ILK, 0);
		3533	I915_WRITE(WM2_LP_ILK, 0);
		3534	I915_WRITE(WM1_LP_ILK, 0);
		3535
		3536	/*
		3537	* Based on the document from hardware guys the following bits
		3538	* should be set unconditionally in order to enable FBC.
		3539	* The bit 22 of 0x42000
		3540	* The bit 22 of 0x42004
		3541	* The bit 7,8,9 of 0x42020.
		3542	*/
		3543	if (IS_IRONLAKE_M(dev)) {
		3544	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		3545	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		3546	ILK_FBCQ_DIS);
		3547	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3548	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3549	ILK_DPARB_GATE);
		3550	}
		3551
3243	Serge	3552	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
		3553
3031	serge	3554	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3555	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3556	ILK_ELPIN_409_SELECT);
		3557	I915_WRITE(_3D_CHICKEN2,
		3558	_3D_CHICKEN2_WM_READ_PIPELINED << 16 \|
		3559	_3D_CHICKEN2_WM_READ_PIPELINED);
3243	Serge	3560
		3561	/* WaDisableRenderCachePipelinedFlush */
		3562	I915_WRITE(CACHE_MODE_0,
		3563	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
		3564
		3565	ibx_init_clock_gating(dev);
3031	serge	3566	}
		3567
3243	Serge	3568	static void cpt_init_clock_gating(struct drm_device *dev)
		3569	{
		3570	struct drm_i915_private *dev_priv = dev->dev_private;
		3571	int pipe;
		3572
		3573	/*
		3574	* On Ibex Peak and Cougar Point, we need to disable clock
		3575	* gating for the panel power sequencer or it will fail to
		3576	* start up when no ports are active.
		3577	*/
		3578	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
		3579	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) \|
		3580	DPLS_EDP_PPS_FIX_DIS);
		3581	/* The below fixes the weird display corruption, a few pixels shifted
		3582	* downward, on (only) LVDS of some HP laptops with IVY.
		3583	*/
		3584	for_each_pipe(pipe)
		3585	I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_CHICKEN2_TIMING_OVERRIDE);
		3586	/* WADP0ClockGatingDisable */
		3587	for_each_pipe(pipe) {
		3588	I915_WRITE(TRANS_CHICKEN1(pipe),
		3589	TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
		3590	}
		3591	}
		3592
3031	serge	3593	static void gen6_init_clock_gating(struct drm_device *dev)
		3594	{
		3595	struct drm_i915_private *dev_priv = dev->dev_private;
		3596	int pipe;
3243	Serge	3597	uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
3031	serge	3598
3243	Serge	3599	I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
3031	serge	3600
		3601	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3602	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3603	ILK_ELPIN_409_SELECT);
		3604
3243	Serge	3605	/* WaDisableHiZPlanesWhenMSAAEnabled */
		3606	I915_WRITE(_3D_CHICKEN,
		3607	_MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
		3608
		3609	/* WaSetupGtModeTdRowDispatch */
		3610	if (IS_SNB_GT1(dev))
		3611	I915_WRITE(GEN6_GT_MODE,
		3612	_MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
		3613
3031	serge	3614	I915_WRITE(WM3_LP_ILK, 0);
		3615	I915_WRITE(WM2_LP_ILK, 0);
		3616	I915_WRITE(WM1_LP_ILK, 0);
		3617
		3618	I915_WRITE(CACHE_MODE_0,
		3619	_MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
		3620
		3621	I915_WRITE(GEN6_UCGCTL1,
		3622	I915_READ(GEN6_UCGCTL1) \|
		3623	GEN6_BLBUNIT_CLOCK_GATE_DISABLE \|
		3624	GEN6_CSUNIT_CLOCK_GATE_DISABLE);
		3625
		3626	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3627	* gating disable must be set. Failure to set it results in
		3628	* flickering pixels due to Z write ordering failures after
		3629	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3630	* Sanctuary and Tropics, and apparently anything else with
		3631	* alpha test or pixel discard.
		3632	*
		3633	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3634	* but we didn't debug actual testcases to find it out.
		3635	*
		3636	* Also apply WaDisableVDSUnitClockGating and
		3637	* WaDisableRCPBUnitClockGating.
		3638	*/
		3639	I915_WRITE(GEN6_UCGCTL2,
		3640	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		3641	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		3642	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3643
		3644	/* Bspec says we need to always set all mask bits. */
		3645	I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) \|
		3646	_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
		3647
		3648	/*
		3649	* According to the spec the following bits should be
		3650	* set in order to enable memory self-refresh and fbc:
		3651	* The bit21 and bit22 of 0x42000
		3652	* The bit21 and bit22 of 0x42004
		3653	* The bit5 and bit7 of 0x42020
		3654	* The bit14 of 0x70180
		3655	* The bit14 of 0x71180
		3656	*/
		3657	I915_WRITE(ILK_DISPLAY_CHICKEN1,
		3658	I915_READ(ILK_DISPLAY_CHICKEN1) \|
		3659	ILK_FBCQ_DIS \| ILK_PABSTRETCH_DIS);
		3660	I915_WRITE(ILK_DISPLAY_CHICKEN2,
		3661	I915_READ(ILK_DISPLAY_CHICKEN2) \|
		3662	ILK_DPARB_GATE \| ILK_VSDPFD_FULL);
3243	Serge	3663	I915_WRITE(ILK_DSPCLK_GATE_D,
		3664	I915_READ(ILK_DSPCLK_GATE_D) \|
		3665	ILK_DPARBUNIT_CLOCK_GATE_ENABLE \|
		3666	ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
3031	serge	3667
3243	Serge	3668	/* WaMbcDriverBootEnable */
3031	serge	3669	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3670	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3671
		3672	for_each_pipe(pipe) {
		3673	I915_WRITE(DSPCNTR(pipe),
		3674	I915_READ(DSPCNTR(pipe)) \|
		3675	DISPPLANE_TRICKLE_FEED_DISABLE);
		3676	intel_flush_display_plane(dev_priv, pipe);
		3677	}
		3678
		3679	/* The default value should be 0x200 according to docs, but the two
		3680	* platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
		3681	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
		3682	I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
3243	Serge	3683
		3684	cpt_init_clock_gating(dev);
3031	serge	3685	}
		3686
		3687	static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
		3688	{
		3689	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
		3690
		3691	reg &= ~GEN7_FF_SCHED_MASK;
		3692	reg \|= GEN7_FF_TS_SCHED_HW;
		3693	reg \|= GEN7_FF_VS_SCHED_HW;
		3694	reg \|= GEN7_FF_DS_SCHED_HW;
		3695
		3696	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
		3697	}
		3698
3243	Serge	3699	static void lpt_init_clock_gating(struct drm_device *dev)
		3700	{
		3701	struct drm_i915_private *dev_priv = dev->dev_private;
		3702
		3703	/*
		3704	* TODO: this bit should only be enabled when really needed, then
		3705	* disabled when not needed anymore in order to save power.
		3706	*/
		3707	if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
		3708	I915_WRITE(SOUTH_DSPCLK_GATE_D,
		3709	I915_READ(SOUTH_DSPCLK_GATE_D) \|
		3710	PCH_LP_PARTITION_LEVEL_DISABLE);
		3711	}
		3712
3031	serge	3713	static void haswell_init_clock_gating(struct drm_device *dev)
		3714	{
		3715	struct drm_i915_private *dev_priv = dev->dev_private;
		3716	int pipe;
		3717
		3718	I915_WRITE(WM3_LP_ILK, 0);
		3719	I915_WRITE(WM2_LP_ILK, 0);
		3720	I915_WRITE(WM1_LP_ILK, 0);
		3721
		3722	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3723	* This implements the WaDisableRCZUnitClockGating workaround.
		3724	*/
		3725	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
		3726
		3727	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3728	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3729	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3730
		3731	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3732	I915_WRITE(GEN7_L3CNTLREG1,
		3733	GEN7_WA_FOR_GEN7_L3_CONTROL);
		3734	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		3735	GEN7_WA_L3_CHICKEN_MODE);
		3736
		3737	/* This is required by WaCatErrorRejectionIssue */
		3738	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3739	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3740	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3741
		3742	for_each_pipe(pipe) {
		3743	I915_WRITE(DSPCNTR(pipe),
		3744	I915_READ(DSPCNTR(pipe)) \|
		3745	DISPPLANE_TRICKLE_FEED_DISABLE);
		3746	intel_flush_display_plane(dev_priv, pipe);
		3747	}
		3748
		3749	gen7_setup_fixed_func_scheduler(dev_priv);
		3750
		3751	/* WaDisable4x2SubspanOptimization */
		3752	I915_WRITE(CACHE_MODE_1,
		3753	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3754
3243	Serge	3755	/* WaMbcDriverBootEnable */
		3756	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3757	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3758
3031	serge	3759	/* XXX: This is a workaround for early silicon revisions and should be
		3760	* removed later.
		3761	*/
		3762	I915_WRITE(WM_DBG,
		3763	I915_READ(WM_DBG) \|
		3764	WM_DBG_DISALLOW_MULTIPLE_LP \|
		3765	WM_DBG_DISALLOW_SPRITE \|
		3766	WM_DBG_DISALLOW_MAXFIFO);
		3767
3243	Serge	3768	lpt_init_clock_gating(dev);
3031	serge	3769	}
		3770
		3771	static void ivybridge_init_clock_gating(struct drm_device *dev)
		3772	{
		3773	struct drm_i915_private *dev_priv = dev->dev_private;
		3774	int pipe;
		3775	uint32_t snpcr;
		3776
		3777	I915_WRITE(WM3_LP_ILK, 0);
		3778	I915_WRITE(WM2_LP_ILK, 0);
		3779	I915_WRITE(WM1_LP_ILK, 0);
		3780
3243	Serge	3781	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	3782
3243	Serge	3783	/* WaDisableEarlyCull */
		3784	I915_WRITE(_3D_CHICKEN3,
		3785	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		3786
		3787	/* WaDisableBackToBackFlipFix */
3031	serge	3788	I915_WRITE(IVB_CHICKEN3,
		3789	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3790	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3791
3243	Serge	3792	/* WaDisablePSDDualDispatchEnable */
		3793	if (IS_IVB_GT1(dev))
		3794	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		3795	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3796	else
		3797	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
		3798	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3799
3031	serge	3800	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3801	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3802	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3803
		3804	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
		3805	I915_WRITE(GEN7_L3CNTLREG1,
		3806	GEN7_WA_FOR_GEN7_L3_CONTROL);
		3807	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
		3808	GEN7_WA_L3_CHICKEN_MODE);
3243	Serge	3809	if (IS_IVB_GT1(dev))
		3810	I915_WRITE(GEN7_ROW_CHICKEN2,
		3811	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		3812	else
		3813	I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
		3814	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
3031	serge	3815
3243	Serge	3816
		3817	/* WaForceL3Serialization */
		3818	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3819	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3820
3031	serge	3821	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3822	* gating disable must be set. Failure to set it results in
		3823	* flickering pixels due to Z write ordering failures after
		3824	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3825	* Sanctuary and Tropics, and apparently anything else with
		3826	* alpha test or pixel discard.
		3827	*
		3828	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3829	* but we didn't debug actual testcases to find it out.
		3830	*
		3831	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3832	* This implements the WaDisableRCZUnitClockGating workaround.
		3833	*/
		3834	I915_WRITE(GEN6_UCGCTL2,
		3835	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		3836	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3837
		3838	/* This is required by WaCatErrorRejectionIssue */
		3839	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3840	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3841	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3842
		3843	for_each_pipe(pipe) {
		3844	I915_WRITE(DSPCNTR(pipe),
		3845	I915_READ(DSPCNTR(pipe)) \|
		3846	DISPPLANE_TRICKLE_FEED_DISABLE);
		3847	intel_flush_display_plane(dev_priv, pipe);
		3848	}
		3849
3243	Serge	3850	/* WaMbcDriverBootEnable */
3031	serge	3851	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3852	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3853
		3854	gen7_setup_fixed_func_scheduler(dev_priv);
		3855
		3856	/* WaDisable4x2SubspanOptimization */
		3857	I915_WRITE(CACHE_MODE_1,
		3858	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3859
		3860	snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
		3861	snpcr &= ~GEN6_MBC_SNPCR_MASK;
		3862	snpcr \|= GEN6_MBC_SNPCR_MED;
		3863	I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
3243	Serge	3864
		3865	cpt_init_clock_gating(dev);
3031	serge	3866	}
		3867
		3868	static void valleyview_init_clock_gating(struct drm_device *dev)
		3869	{
		3870	struct drm_i915_private *dev_priv = dev->dev_private;
		3871	int pipe;
		3872
		3873	I915_WRITE(WM3_LP_ILK, 0);
		3874	I915_WRITE(WM2_LP_ILK, 0);
		3875	I915_WRITE(WM1_LP_ILK, 0);
		3876
3243	Serge	3877	I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
3031	serge	3878
3243	Serge	3879	/* WaDisableEarlyCull */
		3880	I915_WRITE(_3D_CHICKEN3,
		3881	_MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
		3882
		3883	/* WaDisableBackToBackFlipFix */
3031	serge	3884	I915_WRITE(IVB_CHICKEN3,
		3885	CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE \|
		3886	CHICKEN3_DGMG_DONE_FIX_DISABLE);
		3887
3243	Serge	3888	I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
		3889	_MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
		3890
3031	serge	3891	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
		3892	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
		3893	GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
		3894
		3895	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
3243	Serge	3896	I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) \| GEN7_L3AGDIS);
3031	serge	3897	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
		3898
3243	Serge	3899	/* WaForceL3Serialization */
		3900	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3901	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3902
		3903	/* WaDisableDopClockGating */
		3904	I915_WRITE(GEN7_ROW_CHICKEN2,
		3905	_MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
		3906
		3907	/* WaForceL3Serialization */
		3908	I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
		3909	~L3SQ_URB_READ_CAM_MATCH_DISABLE);
		3910
3031	serge	3911	/* This is required by WaCatErrorRejectionIssue */
		3912	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
		3913	I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) \|
		3914	GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
		3915
3243	Serge	3916	/* WaMbcDriverBootEnable */
3031	serge	3917	I915_WRITE(GEN6_MBCTL, I915_READ(GEN6_MBCTL) \|
		3918	GEN6_MBCTL_ENABLE_BOOT_FETCH);
		3919
		3920
		3921	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
		3922	* gating disable must be set. Failure to set it results in
		3923	* flickering pixels due to Z write ordering failures after
		3924	* some amount of runtime in the Mesa "fire" demo, and Unigine
		3925	* Sanctuary and Tropics, and apparently anything else with
		3926	* alpha test or pixel discard.
		3927	*
		3928	* According to the spec, bit 11 (RCCUNIT) must also be set,
		3929	* but we didn't debug actual testcases to find it out.
		3930	*
		3931	* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
		3932	* This implements the WaDisableRCZUnitClockGating workaround.
		3933	*
		3934	* Also apply WaDisableVDSUnitClockGating and
		3935	* WaDisableRCPBUnitClockGating.
		3936	*/
		3937	I915_WRITE(GEN6_UCGCTL2,
		3938	GEN7_VDSUNIT_CLOCK_GATE_DISABLE \|
		3939	GEN7_TDLUNIT_CLOCK_GATE_DISABLE \|
		3940	GEN6_RCZUNIT_CLOCK_GATE_DISABLE \|
		3941	GEN6_RCPBUNIT_CLOCK_GATE_DISABLE \|
		3942	GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
		3943
		3944	I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
		3945
		3946	for_each_pipe(pipe) {
		3947	I915_WRITE(DSPCNTR(pipe),
		3948	I915_READ(DSPCNTR(pipe)) \|
		3949	DISPPLANE_TRICKLE_FEED_DISABLE);
		3950	intel_flush_display_plane(dev_priv, pipe);
		3951	}
		3952
		3953	I915_WRITE(CACHE_MODE_1,
		3954	_MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
		3955
		3956	/*
		3957	* On ValleyView, the GUnit needs to signal the GT
		3958	* when flip and other events complete. So enable
		3959	* all the GUnit->GT interrupts here
		3960	*/
		3961	I915_WRITE(VLV_DPFLIPSTAT, PIPEB_LINE_COMPARE_INT_EN \|
		3962	PIPEB_HLINE_INT_EN \| PIPEB_VBLANK_INT_EN \|
		3963	SPRITED_FLIPDONE_INT_EN \| SPRITEC_FLIPDONE_INT_EN \|
		3964	PLANEB_FLIPDONE_INT_EN \| PIPEA_LINE_COMPARE_INT_EN \|
		3965	PIPEA_HLINE_INT_EN \| PIPEA_VBLANK_INT_EN \|
		3966	SPRITEB_FLIPDONE_INT_EN \| SPRITEA_FLIPDONE_INT_EN \|
		3967	PLANEA_FLIPDONE_INT_EN);
3243	Serge	3968
		3969	/*
		3970	* WaDisableVLVClockGating_VBIIssue
		3971	* Disable clock gating on th GCFG unit to prevent a delay
		3972	* in the reporting of vblank events.
		3973	*/
		3974	I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
3031	serge	3975	}
		3976
		3977	static void g4x_init_clock_gating(struct drm_device *dev)
		3978	{
		3979	struct drm_i915_private *dev_priv = dev->dev_private;
		3980	uint32_t dspclk_gate;
		3981
		3982	I915_WRITE(RENCLK_GATE_D1, 0);
		3983	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE \|
		3984	GS_UNIT_CLOCK_GATE_DISABLE \|
		3985	CL_UNIT_CLOCK_GATE_DISABLE);
		3986	I915_WRITE(RAMCLK_GATE_D, 0);
		3987	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE \|
		3988	OVRUNIT_CLOCK_GATE_DISABLE \|
		3989	OVCUNIT_CLOCK_GATE_DISABLE;
		3990	if (IS_GM45(dev))
		3991	dspclk_gate \|= DSSUNIT_CLOCK_GATE_DISABLE;
		3992	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
3243	Serge	3993
		3994	/* WaDisableRenderCachePipelinedFlush */
		3995	I915_WRITE(CACHE_MODE_0,
		3996	_MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
3031	serge	3997	}
		3998
		3999	static void crestline_init_clock_gating(struct drm_device *dev)
		4000	{
		4001	struct drm_i915_private *dev_priv = dev->dev_private;
		4002
		4003	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
		4004	I915_WRITE(RENCLK_GATE_D2, 0);
		4005	I915_WRITE(DSPCLK_GATE_D, 0);
		4006	I915_WRITE(RAMCLK_GATE_D, 0);
		4007	I915_WRITE16(DEUC, 0);
		4008	}
		4009
		4010	static void broadwater_init_clock_gating(struct drm_device *dev)
		4011	{
		4012	struct drm_i915_private *dev_priv = dev->dev_private;
		4013
		4014	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE \|
		4015	I965_RCC_CLOCK_GATE_DISABLE \|
		4016	I965_RCPB_CLOCK_GATE_DISABLE \|
		4017	I965_ISC_CLOCK_GATE_DISABLE \|
		4018	I965_FBC_CLOCK_GATE_DISABLE);
		4019	I915_WRITE(RENCLK_GATE_D2, 0);
		4020	}
		4021
		4022	static void gen3_init_clock_gating(struct drm_device *dev)
		4023	{
		4024	struct drm_i915_private *dev_priv = dev->dev_private;
		4025	u32 dstate = I915_READ(D_STATE);
		4026
		4027	dstate \|= DSTATE_PLL_D3_OFF \| DSTATE_GFX_CLOCK_GATING \|
		4028	DSTATE_DOT_CLOCK_GATING;
		4029	I915_WRITE(D_STATE, dstate);
		4030
		4031	if (IS_PINEVIEW(dev))
		4032	I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
		4033
		4034	/* IIR "flip pending" means done if this bit is set */
		4035	I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
		4036	}
		4037
		4038	static void i85x_init_clock_gating(struct drm_device *dev)
		4039	{
		4040	struct drm_i915_private *dev_priv = dev->dev_private;
		4041
		4042	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
		4043	}
		4044
		4045	static void i830_init_clock_gating(struct drm_device *dev)
		4046	{
		4047	struct drm_i915_private *dev_priv = dev->dev_private;
		4048
		4049	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
		4050	}
		4051
		4052	void intel_init_clock_gating(struct drm_device *dev)
		4053	{
		4054	struct drm_i915_private *dev_priv = dev->dev_private;
		4055
		4056	dev_priv->display.init_clock_gating(dev);
		4057	}
		4058
		4059	/* Starting with Haswell, we have different power wells for
		4060	* different parts of the GPU. This attempts to enable them all.
		4061	*/
		4062	void intel_init_power_wells(struct drm_device *dev)
		4063	{
		4064	struct drm_i915_private *dev_priv = dev->dev_private;
		4065	unsigned long power_wells[] = {
		4066	HSW_PWR_WELL_CTL1,
		4067	HSW_PWR_WELL_CTL2,
		4068	HSW_PWR_WELL_CTL4
		4069	};
		4070	int i;
		4071
		4072	if (!IS_HASWELL(dev))
		4073	return;
		4074
		4075	mutex_lock(&dev->struct_mutex);
		4076
		4077	for (i = 0; i < ARRAY_SIZE(power_wells); i++) {
		4078	int well = I915_READ(power_wells[i]);
		4079
		4080	if ((well & HSW_PWR_WELL_STATE) == 0) {
		4081	I915_WRITE(power_wells[i], well & HSW_PWR_WELL_ENABLE);
3243	Serge	4082	if (wait_for((I915_READ(power_wells[i]) & HSW_PWR_WELL_STATE), 20))
3031	serge	4083	DRM_ERROR("Error enabling power well %lx\n", power_wells[i]);
		4084	}
		4085	}
		4086
		4087	mutex_unlock(&dev->struct_mutex);
		4088	}
		4089
		4090	/* Set up chip specific power management-related functions */
		4091	void intel_init_pm(struct drm_device *dev)
		4092	{
		4093	struct drm_i915_private *dev_priv = dev->dev_private;
		4094
		4095	if (I915_HAS_FBC(dev)) {
		4096	if (HAS_PCH_SPLIT(dev)) {
		4097	dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
		4098	dev_priv->display.enable_fbc = ironlake_enable_fbc;
		4099	dev_priv->display.disable_fbc = ironlake_disable_fbc;
		4100	} else if (IS_GM45(dev)) {
		4101	dev_priv->display.fbc_enabled = g4x_fbc_enabled;
		4102	dev_priv->display.enable_fbc = g4x_enable_fbc;
		4103	dev_priv->display.disable_fbc = g4x_disable_fbc;
		4104	} else if (IS_CRESTLINE(dev)) {
		4105	dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
		4106	dev_priv->display.enable_fbc = i8xx_enable_fbc;
		4107	dev_priv->display.disable_fbc = i8xx_disable_fbc;
		4108	}
		4109	/* 855GM needs testing */
		4110	}
		4111
		4112	/* For cxsr */
		4113	if (IS_PINEVIEW(dev))
		4114	i915_pineview_get_mem_freq(dev);
		4115	else if (IS_GEN5(dev))
		4116	i915_ironlake_get_mem_freq(dev);
		4117
		4118	/* For FIFO watermark updates */
		4119	if (HAS_PCH_SPLIT(dev)) {
		4120	if (IS_GEN5(dev)) {
		4121	if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
		4122	dev_priv->display.update_wm = ironlake_update_wm;
		4123	else {
		4124	DRM_DEBUG_KMS("Failed to get proper latency. "
		4125	"Disable CxSR\n");
		4126	dev_priv->display.update_wm = NULL;
		4127	}
		4128	dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
		4129	} else if (IS_GEN6(dev)) {
		4130	if (SNB_READ_WM0_LATENCY()) {
		4131	dev_priv->display.update_wm = sandybridge_update_wm;
		4132	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4133	} else {
		4134	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4135	"Disable CxSR\n");
		4136	dev_priv->display.update_wm = NULL;
		4137	}
		4138	dev_priv->display.init_clock_gating = gen6_init_clock_gating;
		4139	} else if (IS_IVYBRIDGE(dev)) {
		4140	/* FIXME: detect B0+ stepping and use auto training */
		4141	if (SNB_READ_WM0_LATENCY()) {
3243	Serge	4142	dev_priv->display.update_wm = ivybridge_update_wm;
3031	serge	4143	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4144	} else {
		4145	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4146	"Disable CxSR\n");
		4147	dev_priv->display.update_wm = NULL;
		4148	}
		4149	dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
		4150	} else if (IS_HASWELL(dev)) {
		4151	if (SNB_READ_WM0_LATENCY()) {
		4152	dev_priv->display.update_wm = sandybridge_update_wm;
		4153	dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
		4154	dev_priv->display.update_linetime_wm = haswell_update_linetime_wm;
		4155	} else {
		4156	DRM_DEBUG_KMS("Failed to read display plane latency. "
		4157	"Disable CxSR\n");
		4158	dev_priv->display.update_wm = NULL;
		4159	}
		4160	dev_priv->display.init_clock_gating = haswell_init_clock_gating;
		4161	} else
		4162	dev_priv->display.update_wm = NULL;
		4163	} else if (IS_VALLEYVIEW(dev)) {
		4164	dev_priv->display.update_wm = valleyview_update_wm;
		4165	dev_priv->display.init_clock_gating =
		4166	valleyview_init_clock_gating;
		4167	} else if (IS_PINEVIEW(dev)) {
		4168	if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
		4169	dev_priv->is_ddr3,
		4170	dev_priv->fsb_freq,
		4171	dev_priv->mem_freq)) {
		4172	DRM_INFO("failed to find known CxSR latency "
		4173	"(found ddr%s fsb freq %d, mem freq %d), "
		4174	"disabling CxSR\n",
		4175	(dev_priv->is_ddr3 == 1) ? "3" : "2",
		4176	dev_priv->fsb_freq, dev_priv->mem_freq);
		4177	/* Disable CxSR and never update its watermark again */
		4178	pineview_disable_cxsr(dev);
		4179	dev_priv->display.update_wm = NULL;
		4180	} else
		4181	dev_priv->display.update_wm = pineview_update_wm;
		4182	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		4183	} else if (IS_G4X(dev)) {
		4184	dev_priv->display.update_wm = g4x_update_wm;
		4185	dev_priv->display.init_clock_gating = g4x_init_clock_gating;
		4186	} else if (IS_GEN4(dev)) {
		4187	dev_priv->display.update_wm = i965_update_wm;
		4188	if (IS_CRESTLINE(dev))
		4189	dev_priv->display.init_clock_gating = crestline_init_clock_gating;
		4190	else if (IS_BROADWATER(dev))
		4191	dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
		4192	} else if (IS_GEN3(dev)) {
		4193	dev_priv->display.update_wm = i9xx_update_wm;
		4194	dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
		4195	dev_priv->display.init_clock_gating = gen3_init_clock_gating;
		4196	} else if (IS_I865G(dev)) {
		4197	dev_priv->display.update_wm = i830_update_wm;
		4198	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		4199	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		4200	} else if (IS_I85X(dev)) {
		4201	dev_priv->display.update_wm = i9xx_update_wm;
		4202	dev_priv->display.get_fifo_size = i85x_get_fifo_size;
		4203	dev_priv->display.init_clock_gating = i85x_init_clock_gating;
		4204	} else {
		4205	dev_priv->display.update_wm = i830_update_wm;
		4206	dev_priv->display.init_clock_gating = i830_init_clock_gating;
		4207	if (IS_845G(dev))
		4208	dev_priv->display.get_fifo_size = i845_get_fifo_size;
		4209	else
		4210	dev_priv->display.get_fifo_size = i830_get_fifo_size;
		4211	}
		4212	}
		4213
		4214	static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
		4215	{
		4216	u32 gt_thread_status_mask;
		4217
		4218	if (IS_HASWELL(dev_priv->dev))
		4219	gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW;
		4220	else
		4221	gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK;
		4222
		4223	/* w/a for a sporadic read returning 0 by waiting for the GT
		4224	* thread to wake up.
		4225	*/
		4226	if (wait_for_atomic_us((I915_READ_NOTRACE(GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500))
		4227	DRM_ERROR("GT thread status wait timed out\n");
		4228	}
		4229
3243	Serge	4230	static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv)
		4231	{
		4232	I915_WRITE_NOTRACE(FORCEWAKE, 0);
		4233	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
		4234	}
		4235
3031	serge	4236	static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
		4237	{
		4238	u32 forcewake_ack;
		4239
		4240	if (IS_HASWELL(dev_priv->dev))
		4241	forcewake_ack = FORCEWAKE_ACK_HSW;
		4242	else
		4243	forcewake_ack = FORCEWAKE_ACK;
		4244
		4245	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1) == 0,
		4246	FORCEWAKE_ACK_TIMEOUT_MS))
		4247	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4248
3243	Serge	4249	I915_WRITE_NOTRACE(FORCEWAKE, FORCEWAKE_KERNEL);
3031	serge	4250	POSTING_READ(ECOBUS); /* something from same cacheline, but !FORCEWAKE */
		4251
		4252	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
		4253	FORCEWAKE_ACK_TIMEOUT_MS))
		4254	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4255
		4256	__gen6_gt_wait_for_thread_c0(dev_priv);
		4257	}
		4258
3243	Serge	4259	static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
		4260	{
		4261	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
		4262	/* something from same cacheline, but !FORCEWAKE_MT */
		4263	POSTING_READ(ECOBUS);
		4264	}
		4265
3031	serge	4266	static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
		4267	{
		4268	u32 forcewake_ack;
		4269
		4270	if (IS_HASWELL(dev_priv->dev))
		4271	forcewake_ack = FORCEWAKE_ACK_HSW;
		4272	else
		4273	forcewake_ack = FORCEWAKE_MT_ACK;
		4274
		4275	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1) == 0,
		4276	FORCEWAKE_ACK_TIMEOUT_MS))
		4277	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4278
3243	Serge	4279	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
		4280	/* something from same cacheline, but !FORCEWAKE_MT */
		4281	POSTING_READ(ECOBUS);
3031	serge	4282
		4283	if (wait_for_atomic((I915_READ_NOTRACE(forcewake_ack) & 1),
		4284	FORCEWAKE_ACK_TIMEOUT_MS))
		4285	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4286
		4287	__gen6_gt_wait_for_thread_c0(dev_priv);
		4288	}
		4289
		4290	/*
		4291	* Generally this is called implicitly by the register read function. However,
		4292	* if some sequence requires the GT to not power down then this function should
		4293	* be called at the beginning of the sequence followed by a call to
		4294	* gen6_gt_force_wake_put() at the end of the sequence.
		4295	*/
		4296	void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
		4297	{
		4298	unsigned long irqflags;
		4299
		4300	spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
		4301	if (dev_priv->forcewake_count++ == 0)
		4302	dev_priv->gt.force_wake_get(dev_priv);
		4303	spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
		4304	}
		4305
		4306	void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
		4307	{
		4308	u32 gtfifodbg;
		4309	gtfifodbg = I915_READ_NOTRACE(GTFIFODBG);
		4310	if (WARN(gtfifodbg & GT_FIFO_CPU_ERROR_MASK,
		4311	"MMIO read or write has been dropped %x\n", gtfifodbg))
		4312	I915_WRITE_NOTRACE(GTFIFODBG, GT_FIFO_CPU_ERROR_MASK);
		4313	}
		4314
		4315	static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
		4316	{
		4317	I915_WRITE_NOTRACE(FORCEWAKE, 0);
3243	Serge	4318	/* something from same cacheline, but !FORCEWAKE */
		4319	POSTING_READ(ECOBUS);
3031	serge	4320	gen6_gt_check_fifodbg(dev_priv);
		4321	}
		4322
		4323	static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
		4324	{
3243	Serge	4325	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
		4326	/* something from same cacheline, but !FORCEWAKE_MT */
		4327	POSTING_READ(ECOBUS);
3031	serge	4328	gen6_gt_check_fifodbg(dev_priv);
		4329	}
		4330
		4331	/*
		4332	* see gen6_gt_force_wake_get()
		4333	*/
		4334	void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv)
		4335	{
		4336	unsigned long irqflags;
		4337
		4338	spin_lock_irqsave(&dev_priv->gt_lock, irqflags);
		4339	if (--dev_priv->forcewake_count == 0)
		4340	dev_priv->gt.force_wake_put(dev_priv);
		4341	spin_unlock_irqrestore(&dev_priv->gt_lock, irqflags);
		4342	}
		4343
		4344	int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
		4345	{
		4346	int ret = 0;
		4347
		4348	if (dev_priv->gt_fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
		4349	int loop = 500;
		4350	u32 fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
		4351	while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
		4352	udelay(10);
		4353	fifo = I915_READ_NOTRACE(GT_FIFO_FREE_ENTRIES);
		4354	}
		4355	if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
		4356	++ret;
		4357	dev_priv->gt_fifo_count = fifo;
		4358	}
		4359	dev_priv->gt_fifo_count--;
		4360
		4361	return ret;
		4362	}
		4363
3243	Serge	4364	static void vlv_force_wake_reset(struct drm_i915_private *dev_priv)
		4365	{
		4366	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(0xffff));
		4367	/* something from same cacheline, but !FORCEWAKE_VLV */
		4368	POSTING_READ(FORCEWAKE_ACK_VLV);
		4369	}
		4370
3031	serge	4371	static void vlv_force_wake_get(struct drm_i915_private *dev_priv)
		4372	{
		4373	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1) == 0,
		4374	FORCEWAKE_ACK_TIMEOUT_MS))
		4375	DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n");
		4376
3243	Serge	4377	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL));
3031	serge	4378
		4379	if (wait_for_atomic((I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1),
		4380	FORCEWAKE_ACK_TIMEOUT_MS))
		4381	DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
		4382
		4383	__gen6_gt_wait_for_thread_c0(dev_priv);
		4384	}
		4385
		4386	static void vlv_force_wake_put(struct drm_i915_private *dev_priv)
		4387	{
3243	Serge	4388	I915_WRITE_NOTRACE(FORCEWAKE_VLV, _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
		4389	/* something from same cacheline, but !FORCEWAKE_VLV */
		4390	POSTING_READ(FORCEWAKE_ACK_VLV);
3031	serge	4391	gen6_gt_check_fifodbg(dev_priv);
		4392	}
		4393
3243	Serge	4394	void intel_gt_reset(struct drm_device *dev)
		4395	{
		4396	struct drm_i915_private *dev_priv = dev->dev_private;
		4397
		4398	if (IS_VALLEYVIEW(dev)) {
		4399	vlv_force_wake_reset(dev_priv);
		4400	} else if (INTEL_INFO(dev)->gen >= 6) {
		4401	__gen6_gt_force_wake_reset(dev_priv);
		4402	if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev))
		4403	__gen6_gt_force_wake_mt_reset(dev_priv);
		4404	}
		4405	}
		4406
3031	serge	4407	void intel_gt_init(struct drm_device *dev)
		4408	{
		4409	struct drm_i915_private *dev_priv = dev->dev_private;
		4410
		4411	spin_lock_init(&dev_priv->gt_lock);
		4412
3243	Serge	4413	intel_gt_reset(dev);
		4414
3031	serge	4415	if (IS_VALLEYVIEW(dev)) {
		4416	dev_priv->gt.force_wake_get = vlv_force_wake_get;
		4417	dev_priv->gt.force_wake_put = vlv_force_wake_put;
3243	Serge	4418	} else if (IS_IVYBRIDGE(dev) \|\| IS_HASWELL(dev)) {
		4419	dev_priv->gt.force_wake_get = __gen6_gt_force_wake_mt_get;
		4420	dev_priv->gt.force_wake_put = __gen6_gt_force_wake_mt_put;
		4421	} else if (IS_GEN6(dev)) {
3031	serge	4422	dev_priv->gt.force_wake_get = __gen6_gt_force_wake_get;
		4423	dev_priv->gt.force_wake_put = __gen6_gt_force_wake_put;
3243	Serge	4424	}
		4425	}
3031	serge	4426
3243	Serge	4427	int sandybridge_pcode_read(struct drm_i915_private dev_priv, u8 mbox, u32 val)
		4428	{
		4429	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3031	serge	4430
3243	Serge	4431	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		4432	DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
		4433	return -EAGAIN;
		4434	}
3031	serge	4435
3243	Serge	4436	I915_WRITE(GEN6_PCODE_DATA, *val);
		4437	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		4438
		4439	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		4440	500)) {
		4441	DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
		4442	return -ETIMEDOUT;
3031	serge	4443	}
3243	Serge	4444
		4445	*val = I915_READ(GEN6_PCODE_DATA);
		4446	I915_WRITE(GEN6_PCODE_DATA, 0);
		4447
		4448	return 0;
		4449	}
		4450
		4451	int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val)
		4452	{
		4453	WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
		4454
		4455	if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
		4456	DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
		4457	return -EAGAIN;
3031	serge	4458	}
3243	Serge	4459
		4460	I915_WRITE(GEN6_PCODE_DATA, val);
		4461	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY \| mbox);
		4462
		4463	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
		4464	500)) {
		4465	DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
		4466	return -ETIMEDOUT;
3031	serge	4467	}
3243	Serge	4468
		4469	I915_WRITE(GEN6_PCODE_DATA, 0);
		4470
		4471	return 0;
3031	serge	4472	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/i915/intel_pm.c – Rev 3243