WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/r600.c

Rev	Author	Line No.	Line
1128	serge	1	/*
		2	* Copyright 2008 Advanced Micro Devices, Inc.
		3	* Copyright 2008 Red Hat Inc.
		4	* Copyright 2009 Jerome Glisse.
		5	*
		6	* Permission is hereby granted, free of charge, to any person obtaining a
		7	* copy of this software and associated documentation files (the "Software"),
		8	* to deal in the Software without restriction, including without limitation
		9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		10	* and/or sell copies of the Software, and to permit persons to whom the
		11	* Software is furnished to do so, subject to the following conditions:
		12	*
		13	* The above copyright notice and this permission notice shall be included in
		14	* all copies or substantial portions of the Software.
		15	*
		16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		22	* OTHER DEALINGS IN THE SOFTWARE.
		23	*
		24	* Authors: Dave Airlie
		25	* Alex Deucher
		26	* Jerome Glisse
		27	*/
1963	serge	28	#include
1233	serge	29	#include
1221	serge	30	#include
2997	Serge	31	#include
		32	#include
		33	#include
1128	serge	34	#include "radeon.h"
1963	serge	35	#include "radeon_asic.h"
1221	serge	36	#include "radeon_mode.h"
		37	#include "r600d.h"
		38	#include "atom.h"
		39	#include "avivod.h"
5078	serge	40	#include "radeon_ucode.h"
1128	serge	41
1221	serge	42	/* Firmware Names */
		43	MODULE_FIRMWARE("radeon/R600_pfp.bin");
		44	MODULE_FIRMWARE("radeon/R600_me.bin");
		45	MODULE_FIRMWARE("radeon/RV610_pfp.bin");
		46	MODULE_FIRMWARE("radeon/RV610_me.bin");
		47	MODULE_FIRMWARE("radeon/RV630_pfp.bin");
		48	MODULE_FIRMWARE("radeon/RV630_me.bin");
		49	MODULE_FIRMWARE("radeon/RV620_pfp.bin");
		50	MODULE_FIRMWARE("radeon/RV620_me.bin");
		51	MODULE_FIRMWARE("radeon/RV635_pfp.bin");
		52	MODULE_FIRMWARE("radeon/RV635_me.bin");
		53	MODULE_FIRMWARE("radeon/RV670_pfp.bin");
		54	MODULE_FIRMWARE("radeon/RV670_me.bin");
		55	MODULE_FIRMWARE("radeon/RS780_pfp.bin");
		56	MODULE_FIRMWARE("radeon/RS780_me.bin");
		57	MODULE_FIRMWARE("radeon/RV770_pfp.bin");
		58	MODULE_FIRMWARE("radeon/RV770_me.bin");
5078	serge	59	MODULE_FIRMWARE("radeon/RV770_smc.bin");
1221	serge	60	MODULE_FIRMWARE("radeon/RV730_pfp.bin");
		61	MODULE_FIRMWARE("radeon/RV730_me.bin");
5078	serge	62	MODULE_FIRMWARE("radeon/RV730_smc.bin");
		63	MODULE_FIRMWARE("radeon/RV740_smc.bin");
1221	serge	64	MODULE_FIRMWARE("radeon/RV710_pfp.bin");
		65	MODULE_FIRMWARE("radeon/RV710_me.bin");
5078	serge	66	MODULE_FIRMWARE("radeon/RV710_smc.bin");
1321	serge	67	MODULE_FIRMWARE("radeon/R600_rlc.bin");
		68	MODULE_FIRMWARE("radeon/R700_rlc.bin");
1963	serge	69	MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
		70	MODULE_FIRMWARE("radeon/CEDAR_me.bin");
		71	MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
5078	serge	72	MODULE_FIRMWARE("radeon/CEDAR_smc.bin");
1963	serge	73	MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
		74	MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
		75	MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
5078	serge	76	MODULE_FIRMWARE("radeon/REDWOOD_smc.bin");
1963	serge	77	MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
		78	MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
		79	MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
5078	serge	80	MODULE_FIRMWARE("radeon/JUNIPER_smc.bin");
1963	serge	81	MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
		82	MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
		83	MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
5078	serge	84	MODULE_FIRMWARE("radeon/CYPRESS_smc.bin");
1963	serge	85	MODULE_FIRMWARE("radeon/PALM_pfp.bin");
		86	MODULE_FIRMWARE("radeon/PALM_me.bin");
		87	MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
		88	MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
		89	MODULE_FIRMWARE("radeon/SUMO_me.bin");
		90	MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
		91	MODULE_FIRMWARE("radeon/SUMO2_me.bin");
1221	serge	92
3764	Serge	93	static const u32 crtc_offsets[2] =
		94	{
		95	0,
		96	AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
		97	};
		98
1221	serge	99	int r600_debugfs_mc_info_init(struct radeon_device *rdev);
		100
		101	/* r600,rv610,rv630,rv620,rv635,rv670 */
1128	serge	102	int r600_mc_wait_for_idle(struct radeon_device *rdev);
2997	Serge	103	static void r600_gpu_init(struct radeon_device *rdev);
1221	serge	104	void r600_fini(struct radeon_device *rdev);
1963	serge	105	void r600_irq_disable(struct radeon_device *rdev);
		106	static void r600_pcie_gen2_enable(struct radeon_device *rdev);
5078	serge	107	extern int evergreen_rlc_resume(struct radeon_device *rdev);
		108	extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
1128	serge	109
3764	Serge	110	/**
		111	* r600_get_xclk - get the xclk
		112	*
		113	* @rdev: radeon_device pointer
		114	*
		115	* Returns the reference clock used by the gfx engine
		116	* (r6xx, IGPs, APUs).
		117	*/
		118	u32 r600_get_xclk(struct radeon_device *rdev)
		119	{
		120	return rdev->clock.spll.reference_freq;
		121	}
		122
5078	serge	123	int r600_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
		124	{
5271	serge	125	unsigned fb_div = 0, ref_div, vclk_div = 0, dclk_div = 0;
		126	int r;
		127
		128	/* bypass vclk and dclk with bclk */
		129	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		130	VCLK_SRC_SEL(1) \| DCLK_SRC_SEL(1),
		131	~(VCLK_SRC_SEL_MASK \| DCLK_SRC_SEL_MASK));
		132
		133	/* assert BYPASS_EN, deassert UPLL_RESET, UPLL_SLEEP and UPLL_CTLREQ */
		134	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~(
		135	UPLL_RESET_MASK \| UPLL_SLEEP_MASK \| UPLL_CTLREQ_MASK));
		136
		137	if (rdev->family >= CHIP_RS780)
		138	WREG32_P(GFX_MACRO_BYPASS_CNTL, UPLL_BYPASS_CNTL,
		139	~UPLL_BYPASS_CNTL);
		140
		141	if (!vclk \|\| !dclk) {
		142	/* keep the Bypass mode, put PLL to sleep */
		143	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
		144	return 0;
		145	}
		146
		147	if (rdev->clock.spll.reference_freq == 10000)
		148	ref_div = 34;
		149	else
		150	ref_div = 4;
		151
		152	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 50000, 160000,
		153	ref_div + 1, 0xFFF, 2, 30, ~0,
		154	&fb_div, &vclk_div, &dclk_div);
		155	if (r)
		156	return r;
		157
		158	if (rdev->family >= CHIP_RV670 && rdev->family < CHIP_RS780)
		159	fb_div >>= 1;
		160	else
		161	fb_div \|= 1;
		162
		163	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
		164	if (r)
		165	return r;
		166
		167	/* assert PLL_RESET */
		168	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
		169
		170	/* For RS780 we have to choose ref clk */
		171	if (rdev->family >= CHIP_RS780)
		172	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_REFCLK_SRC_SEL_MASK,
		173	~UPLL_REFCLK_SRC_SEL_MASK);
		174
		175	/* set the required fb, ref and post divder values */
		176	WREG32_P(CG_UPLL_FUNC_CNTL,
		177	UPLL_FB_DIV(fb_div) \|
		178	UPLL_REF_DIV(ref_div),
		179	~(UPLL_FB_DIV_MASK \| UPLL_REF_DIV_MASK));
		180	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		181	UPLL_SW_HILEN(vclk_div >> 1) \|
		182	UPLL_SW_LOLEN((vclk_div >> 1) + (vclk_div & 1)) \|
		183	UPLL_SW_HILEN2(dclk_div >> 1) \|
		184	UPLL_SW_LOLEN2((dclk_div >> 1) + (dclk_div & 1)) \|
		185	UPLL_DIVEN_MASK \| UPLL_DIVEN2_MASK,
		186	~UPLL_SW_MASK);
		187
		188	/* give the PLL some time to settle */
		189	mdelay(15);
		190
		191	/* deassert PLL_RESET */
		192	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
		193
		194	mdelay(15);
		195
		196	/* deassert BYPASS EN */
		197	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
		198
		199	if (rdev->family >= CHIP_RS780)
		200	WREG32_P(GFX_MACRO_BYPASS_CNTL, 0, ~UPLL_BYPASS_CNTL);
		201
		202	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
		203	if (r)
		204	return r;
		205
		206	/* switch VCLK and DCLK selection */
		207	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		208	VCLK_SRC_SEL(2) \| DCLK_SRC_SEL(2),
		209	~(VCLK_SRC_SEL_MASK \| DCLK_SRC_SEL_MASK));
		210
		211	mdelay(100);
		212
5078	serge	213	return 0;
		214	}
		215
		216	void dce3_program_fmt(struct drm_encoder *encoder)
		217	{
		218	struct drm_device *dev = encoder->dev;
		219	struct radeon_device *rdev = dev->dev_private;
		220	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
		221	struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
		222	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
		223	int bpc = 0;
		224	u32 tmp = 0;
		225	enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
		226
		227	if (connector) {
		228	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		229	bpc = radeon_get_monitor_bpc(connector);
		230	dither = radeon_connector->dither;
		231	}
		232
		233	/* LVDS FMT is set up by atom */
		234	if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
		235	return;
		236
		237	/* not needed for analog */
		238	if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) \|\|
		239	(radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
		240	return;
		241
		242	if (bpc == 0)
		243	return;
		244
		245	switch (bpc) {
		246	case 6:
		247	if (dither == RADEON_FMT_DITHER_ENABLE)
		248	/* XXX sort out optimal dither settings */
		249	tmp \|= FMT_SPATIAL_DITHER_EN;
		250	else
		251	tmp \|= FMT_TRUNCATE_EN;
		252	break;
		253	case 8:
		254	if (dither == RADEON_FMT_DITHER_ENABLE)
		255	/* XXX sort out optimal dither settings */
		256	tmp \|= (FMT_SPATIAL_DITHER_EN \| FMT_SPATIAL_DITHER_DEPTH);
		257	else
		258	tmp \|= (FMT_TRUNCATE_EN \| FMT_TRUNCATE_DEPTH);
		259	break;
		260	case 10:
		261	default:
		262	/* not needed */
		263	break;
		264	}
		265
		266	WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
		267	}
		268
1963	serge	269	/* get temperature in millidegrees */
		270	int rv6xx_get_temp(struct radeon_device *rdev)
		271	{
		272	u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
		273	ASIC_T_SHIFT;
		274	int actual_temp = temp & 0xff;
		275
		276	if (temp & 0x100)
		277	actual_temp -= 256;
		278
		279	return actual_temp * 1000;
		280	}
		281
5078	serge	282	void r600_pm_get_dynpm_state(struct radeon_device *rdev)
		283	{
		284	int i;
1963	serge	285
5078	serge	286	rdev->pm.dynpm_can_upclock = true;
		287	rdev->pm.dynpm_can_downclock = true;
1963	serge	288
5078	serge	289	/* power state array is low to high, default is first */
		290	if ((rdev->flags & RADEON_IS_IGP) \|\| (rdev->family == CHIP_R600)) {
		291	int min_power_state_index = 0;
1963	serge	292
5078	serge	293	if (rdev->pm.num_power_states > 2)
		294	min_power_state_index = 1;
1963	serge	295
5078	serge	296	switch (rdev->pm.dynpm_planned_action) {
		297	case DYNPM_ACTION_MINIMUM:
		298	rdev->pm.requested_power_state_index = min_power_state_index;
		299	rdev->pm.requested_clock_mode_index = 0;
		300	rdev->pm.dynpm_can_downclock = false;
		301	break;
		302	case DYNPM_ACTION_DOWNCLOCK:
		303	if (rdev->pm.current_power_state_index == min_power_state_index) {
		304	rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
		305	rdev->pm.dynpm_can_downclock = false;
		306	} else {
		307	if (rdev->pm.active_crtc_count > 1) {
		308	for (i = 0; i < rdev->pm.num_power_states; i++) {
		309	if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
		310	continue;
		311	else if (i >= rdev->pm.current_power_state_index) {
		312	rdev->pm.requested_power_state_index =
		313	rdev->pm.current_power_state_index;
		314	break;
		315	} else {
		316	rdev->pm.requested_power_state_index = i;
		317	break;
		318	}
		319	}
		320	} else {
		321	if (rdev->pm.current_power_state_index == 0)
		322	rdev->pm.requested_power_state_index =
		323	rdev->pm.num_power_states - 1;
		324	else
		325	rdev->pm.requested_power_state_index =
		326	rdev->pm.current_power_state_index - 1;
		327	}
		328	}
		329	rdev->pm.requested_clock_mode_index = 0;
		330	/* don't use the power state if crtcs are active and no display flag is set */
		331	if ((rdev->pm.active_crtc_count > 0) &&
		332	(rdev->pm.power_state[rdev->pm.requested_power_state_index].
		333	clock_info[rdev->pm.requested_clock_mode_index].flags &
		334	RADEON_PM_MODE_NO_DISPLAY)) {
		335	rdev->pm.requested_power_state_index++;
		336	}
		337	break;
		338	case DYNPM_ACTION_UPCLOCK:
		339	if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
		340	rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
		341	rdev->pm.dynpm_can_upclock = false;
		342	} else {
		343	if (rdev->pm.active_crtc_count > 1) {
		344	for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
		345	if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
		346	continue;
		347	else if (i <= rdev->pm.current_power_state_index) {
		348	rdev->pm.requested_power_state_index =
		349	rdev->pm.current_power_state_index;
		350	break;
		351	} else {
		352	rdev->pm.requested_power_state_index = i;
		353	break;
		354	}
		355	}
		356	} else
		357	rdev->pm.requested_power_state_index =
		358	rdev->pm.current_power_state_index + 1;
		359	}
		360	rdev->pm.requested_clock_mode_index = 0;
		361	break;
		362	case DYNPM_ACTION_DEFAULT:
		363	rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
		364	rdev->pm.requested_clock_mode_index = 0;
		365	rdev->pm.dynpm_can_upclock = false;
		366	break;
		367	case DYNPM_ACTION_NONE:
		368	default:
		369	DRM_ERROR("Requested mode for not defined action\n");
		370	return;
		371	}
		372	} else {
		373	/* XXX select a power state based on AC/DC, single/dualhead, etc. */
		374	/* for now just select the first power state and switch between clock modes */
		375	/* power state array is low to high, default is first (0) */
		376	if (rdev->pm.active_crtc_count > 1) {
		377	rdev->pm.requested_power_state_index = -1;
		378	/* start at 1 as we don't want the default mode */
		379	for (i = 1; i < rdev->pm.num_power_states; i++) {
		380	if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
		381	continue;
		382	else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) \|\|
		383	(rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
		384	rdev->pm.requested_power_state_index = i;
		385	break;
		386	}
		387	}
		388	/* if nothing selected, grab the default state. */
		389	if (rdev->pm.requested_power_state_index == -1)
		390	rdev->pm.requested_power_state_index = 0;
		391	} else
		392	rdev->pm.requested_power_state_index = 1;
1963	serge	393
5078	serge	394	switch (rdev->pm.dynpm_planned_action) {
		395	case DYNPM_ACTION_MINIMUM:
		396	rdev->pm.requested_clock_mode_index = 0;
		397	rdev->pm.dynpm_can_downclock = false;
		398	break;
		399	case DYNPM_ACTION_DOWNCLOCK:
		400	if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
		401	if (rdev->pm.current_clock_mode_index == 0) {
		402	rdev->pm.requested_clock_mode_index = 0;
		403	rdev->pm.dynpm_can_downclock = false;
		404	} else
		405	rdev->pm.requested_clock_mode_index =
		406	rdev->pm.current_clock_mode_index - 1;
		407	} else {
		408	rdev->pm.requested_clock_mode_index = 0;
		409	rdev->pm.dynpm_can_downclock = false;
		410	}
		411	/* don't use the power state if crtcs are active and no display flag is set */
		412	if ((rdev->pm.active_crtc_count > 0) &&
		413	(rdev->pm.power_state[rdev->pm.requested_power_state_index].
		414	clock_info[rdev->pm.requested_clock_mode_index].flags &
		415	RADEON_PM_MODE_NO_DISPLAY)) {
		416	rdev->pm.requested_clock_mode_index++;
		417	}
		418	break;
		419	case DYNPM_ACTION_UPCLOCK:
		420	if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
		421	if (rdev->pm.current_clock_mode_index ==
		422	(rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
		423	rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
		424	rdev->pm.dynpm_can_upclock = false;
		425	} else
		426	rdev->pm.requested_clock_mode_index =
		427	rdev->pm.current_clock_mode_index + 1;
		428	} else {
		429	rdev->pm.requested_clock_mode_index =
		430	rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
		431	rdev->pm.dynpm_can_upclock = false;
		432	}
		433	break;
		434	case DYNPM_ACTION_DEFAULT:
		435	rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
		436	rdev->pm.requested_clock_mode_index = 0;
		437	rdev->pm.dynpm_can_upclock = false;
		438	break;
		439	case DYNPM_ACTION_NONE:
		440	default:
		441	DRM_ERROR("Requested mode for not defined action\n");
		442	return;
		443	}
		444	}
		445
		446	DRM_DEBUG_DRIVER("Requested: e: %d m: %d p: %d\n",
		447	rdev->pm.power_state[rdev->pm.requested_power_state_index].
		448	clock_info[rdev->pm.requested_clock_mode_index].sclk,
		449	rdev->pm.power_state[rdev->pm.requested_power_state_index].
		450	clock_info[rdev->pm.requested_clock_mode_index].mclk,
		451	rdev->pm.power_state[rdev->pm.requested_power_state_index].
		452	pcie_lanes);
		453	}
		454
		455	void rs780_pm_init_profile(struct radeon_device *rdev)
		456	{
		457	if (rdev->pm.num_power_states == 2) {
		458	/* default */
		459	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		460	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		461	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		462	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
		463	/* low sh */
		464	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;
		465	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;
		466	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		467	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		468	/* mid sh */
		469	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;
		470	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;
		471	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		472	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
		473	/* high sh */
		474	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;
		475	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
		476	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		477	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
		478	/* low mh */
		479	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;
		480	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
		481	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		482	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		483	/* mid mh */
		484	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;
		485	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
		486	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		487	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
		488	/* high mh */
		489	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;
		490	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;
		491	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		492	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
		493	} else if (rdev->pm.num_power_states == 3) {
		494	/* default */
		495	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		496	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		497	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		498	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
		499	/* low sh */
		500	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
		501	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
		502	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		503	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		504	/* mid sh */
		505	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
		506	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
		507	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		508	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
		509	/* high sh */
		510	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
		511	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;
		512	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		513	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
		514	/* low mh */
		515	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;
		516	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;
		517	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		518	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		519	/* mid mh */
		520	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;
		521	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;
		522	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		523	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
		524	/* high mh */
		525	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;
		526	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
		527	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		528	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
		529	} else {
		530	/* default */
		531	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		532	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		533	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		534	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
		535	/* low sh */
		536	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;
		537	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;
		538	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		539	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		540	/* mid sh */
		541	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;
		542	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;
		543	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		544	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
		545	/* high sh */
		546	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;
		547	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;
		548	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		549	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
		550	/* low mh */
		551	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
		552	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
		553	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		554	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		555	/* mid mh */
		556	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
		557	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
		558	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		559	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
		560	/* high mh */
		561	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
		562	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;
		563	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		564	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
		565	}
		566	}
		567
		568	void r600_pm_init_profile(struct radeon_device *rdev)
		569	{
		570	int idx;
		571
		572	if (rdev->family == CHIP_R600) {
		573	/* XXX */
		574	/* default */
		575	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		576	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		577	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		578	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
		579	/* low sh */
		580	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		581	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		582	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		583	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		584	/* mid sh */
		585	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		586	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		587	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		588	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
		589	/* high sh */
		590	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		591	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		592	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		593	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
		594	/* low mh */
		595	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		596	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		597	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		598	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		599	/* mid mh */
		600	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		601	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		602	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		603	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
		604	/* high mh */
		605	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		606	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		607	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		608	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
		609	} else {
		610	if (rdev->pm.num_power_states < 4) {
		611	/* default */
		612	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		613	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		614	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		615	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
		616	/* low sh */
		617	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
		618	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
		619	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		620	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		621	/* mid sh */
		622	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
		623	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
		624	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		625	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
		626	/* high sh */
		627	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
		628	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
		629	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		630	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
		631	/* low mh */
		632	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
		633	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;
		634	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		635	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		636	/* low mh */
		637	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
		638	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;
		639	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		640	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
		641	/* high mh */
		642	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
		643	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
		644	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		645	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
		646	} else {
		647	/* default */
		648	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		649	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		650	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		651	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
		652	/* low sh */
		653	if (rdev->flags & RADEON_IS_MOBILITY)
		654	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
		655	else
		656	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		657	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
		658	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
		659	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		660	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		661	/* mid sh */
		662	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
		663	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
		664	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		665	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
		666	/* high sh */
		667	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		668	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
		669	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
		670	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		671	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
		672	/* low mh */
		673	if (rdev->flags & RADEON_IS_MOBILITY)
		674	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);
		675	else
		676	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
		677	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
		678	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
		679	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		680	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		681	/* mid mh */
		682	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
		683	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
		684	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		685	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
		686	/* high mh */
		687	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
		688	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
		689	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
		690	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		691	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
		692	}
		693	}
		694	}
		695
		696	void r600_pm_misc(struct radeon_device *rdev)
		697	{
		698	int req_ps_idx = rdev->pm.requested_power_state_index;
		699	int req_cm_idx = rdev->pm.requested_clock_mode_index;
		700	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
		701	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
		702
		703	if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
		704	/* 0xff01 is a flag rather then an actual voltage */
		705	if (voltage->voltage == 0xff01)
		706	return;
		707	if (voltage->voltage != rdev->pm.current_vddc) {
		708	radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
		709	rdev->pm.current_vddc = voltage->voltage;
		710	DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
		711	}
		712	}
		713	}
		714
1963	serge	715	bool r600_gui_idle(struct radeon_device *rdev)
		716	{
		717	if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
		718	return false;
		719	else
		720	return true;
		721	}
		722
1321	serge	723	/* hpd for digital panel detect/disconnect */
		724	bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		725	{
		726	bool connected = false;
		727
		728	if (ASIC_IS_DCE3(rdev)) {
		729	switch (hpd) {
		730	case RADEON_HPD_1:
		731	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		732	connected = true;
		733	break;
		734	case RADEON_HPD_2:
		735	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		736	connected = true;
		737	break;
		738	case RADEON_HPD_3:
		739	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		740	connected = true;
		741	break;
		742	case RADEON_HPD_4:
		743	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		744	connected = true;
		745	break;
		746	/* DCE 3.2 */
		747	case RADEON_HPD_5:
		748	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		749	connected = true;
		750	break;
		751	case RADEON_HPD_6:
		752	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		753	connected = true;
		754	break;
		755	default:
		756	break;
		757	}
		758	} else {
		759	switch (hpd) {
		760	case RADEON_HPD_1:
		761	if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		762	connected = true;
		763	break;
		764	case RADEON_HPD_2:
		765	if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		766	connected = true;
		767	break;
		768	case RADEON_HPD_3:
		769	if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		770	connected = true;
		771	break;
		772	default:
		773	break;
		774	}
		775	}
		776	return connected;
		777	}
		778
		779	void r600_hpd_set_polarity(struct radeon_device *rdev,
		780	enum radeon_hpd_id hpd)
		781	{
		782	u32 tmp;
		783	bool connected = r600_hpd_sense(rdev, hpd);
		784
		785	if (ASIC_IS_DCE3(rdev)) {
		786	switch (hpd) {
		787	case RADEON_HPD_1:
		788	tmp = RREG32(DC_HPD1_INT_CONTROL);
		789	if (connected)
		790	tmp &= ~DC_HPDx_INT_POLARITY;
		791	else
		792	tmp \|= DC_HPDx_INT_POLARITY;
		793	WREG32(DC_HPD1_INT_CONTROL, tmp);
		794	break;
		795	case RADEON_HPD_2:
		796	tmp = RREG32(DC_HPD2_INT_CONTROL);
		797	if (connected)
		798	tmp &= ~DC_HPDx_INT_POLARITY;
		799	else
		800	tmp \|= DC_HPDx_INT_POLARITY;
		801	WREG32(DC_HPD2_INT_CONTROL, tmp);
		802	break;
		803	case RADEON_HPD_3:
		804	tmp = RREG32(DC_HPD3_INT_CONTROL);
		805	if (connected)
		806	tmp &= ~DC_HPDx_INT_POLARITY;
		807	else
		808	tmp \|= DC_HPDx_INT_POLARITY;
		809	WREG32(DC_HPD3_INT_CONTROL, tmp);
		810	break;
		811	case RADEON_HPD_4:
		812	tmp = RREG32(DC_HPD4_INT_CONTROL);
		813	if (connected)
		814	tmp &= ~DC_HPDx_INT_POLARITY;
		815	else
		816	tmp \|= DC_HPDx_INT_POLARITY;
		817	WREG32(DC_HPD4_INT_CONTROL, tmp);
		818	break;
		819	case RADEON_HPD_5:
		820	tmp = RREG32(DC_HPD5_INT_CONTROL);
		821	if (connected)
		822	tmp &= ~DC_HPDx_INT_POLARITY;
		823	else
		824	tmp \|= DC_HPDx_INT_POLARITY;
		825	WREG32(DC_HPD5_INT_CONTROL, tmp);
		826	break;
		827	/* DCE 3.2 */
		828	case RADEON_HPD_6:
		829	tmp = RREG32(DC_HPD6_INT_CONTROL);
		830	if (connected)
		831	tmp &= ~DC_HPDx_INT_POLARITY;
		832	else
		833	tmp \|= DC_HPDx_INT_POLARITY;
		834	WREG32(DC_HPD6_INT_CONTROL, tmp);
		835	break;
		836	default:
		837	break;
		838	}
		839	} else {
		840	switch (hpd) {
		841	case RADEON_HPD_1:
		842	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		843	if (connected)
		844	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		845	else
		846	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		847	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		848	break;
		849	case RADEON_HPD_2:
		850	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		851	if (connected)
		852	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		853	else
		854	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		855	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		856	break;
		857	case RADEON_HPD_3:
		858	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		859	if (connected)
		860	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		861	else
		862	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		863	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		864	break;
		865	default:
		866	break;
		867	}
		868	}
		869	}
		870
		871	void r600_hpd_init(struct radeon_device *rdev)
		872	{
		873	struct drm_device *dev = rdev->ddev;
		874	struct drm_connector *connector;
2997	Serge	875	unsigned enable = 0;
1321	serge	876
2997	Serge	877	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		878	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		879
		880	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP \|\|
		881	connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
		882	/* don't try to enable hpd on eDP or LVDS avoid breaking the
		883	* aux dp channel on imac and help (but not completely fix)
		884	* https://bugzilla.redhat.com/show_bug.cgi?id=726143
		885	*/
		886	continue;
		887	}
1321	serge	888	if (ASIC_IS_DCE3(rdev)) {
		889	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \| DC_HPDx_RX_INT_TIMER(0xfa);
		890	if (ASIC_IS_DCE32(rdev))
		891	tmp \|= DC_HPDx_EN;
		892
		893	switch (radeon_connector->hpd.hpd) {
		894	case RADEON_HPD_1:
		895	WREG32(DC_HPD1_CONTROL, tmp);
		896	break;
		897	case RADEON_HPD_2:
		898	WREG32(DC_HPD2_CONTROL, tmp);
		899	break;
		900	case RADEON_HPD_3:
		901	WREG32(DC_HPD3_CONTROL, tmp);
		902	break;
		903	case RADEON_HPD_4:
		904	WREG32(DC_HPD4_CONTROL, tmp);
		905	break;
		906	/* DCE 3.2 */
		907	case RADEON_HPD_5:
		908	WREG32(DC_HPD5_CONTROL, tmp);
		909	break;
		910	case RADEON_HPD_6:
		911	WREG32(DC_HPD6_CONTROL, tmp);
		912	break;
		913	default:
		914	break;
		915	}
		916	} else {
		917	switch (radeon_connector->hpd.hpd) {
		918	case RADEON_HPD_1:
		919	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		920	break;
		921	case RADEON_HPD_2:
		922	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		923	break;
		924	case RADEON_HPD_3:
		925	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		926	break;
		927	default:
		928	break;
		929	}
		930	}
2997	Serge	931	enable \|= 1 << radeon_connector->hpd.hpd;
		932	radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1321	serge	933	}
2997	Serge	934	// radeon_irq_kms_enable_hpd(rdev, enable);
1321	serge	935	}
		936
		937	void r600_hpd_fini(struct radeon_device *rdev)
		938	{
		939	struct drm_device *dev = rdev->ddev;
		940	struct drm_connector *connector;
2997	Serge	941	unsigned disable = 0;
1321	serge	942
		943	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		944	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2997	Serge	945	if (ASIC_IS_DCE3(rdev)) {
1321	serge	946	switch (radeon_connector->hpd.hpd) {
		947	case RADEON_HPD_1:
		948	WREG32(DC_HPD1_CONTROL, 0);
		949	break;
		950	case RADEON_HPD_2:
		951	WREG32(DC_HPD2_CONTROL, 0);
		952	break;
		953	case RADEON_HPD_3:
		954	WREG32(DC_HPD3_CONTROL, 0);
		955	break;
		956	case RADEON_HPD_4:
		957	WREG32(DC_HPD4_CONTROL, 0);
		958	break;
		959	/* DCE 3.2 */
		960	case RADEON_HPD_5:
		961	WREG32(DC_HPD5_CONTROL, 0);
		962	break;
		963	case RADEON_HPD_6:
		964	WREG32(DC_HPD6_CONTROL, 0);
		965	break;
		966	default:
		967	break;
		968	}
		969	} else {
		970	switch (radeon_connector->hpd.hpd) {
		971	case RADEON_HPD_1:
		972	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
		973	break;
		974	case RADEON_HPD_2:
		975	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
		976	break;
		977	case RADEON_HPD_3:
		978	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
		979	break;
		980	default:
		981	break;
		982	}
		983	}
2997	Serge	984	disable \|= 1 << radeon_connector->hpd.hpd;
1321	serge	985	}
2997	Serge	986	// radeon_irq_kms_disable_hpd(rdev, disable);
1321	serge	987	}
		988
1128	serge	989	/*
1221	serge	990	* R600 PCIE GART
1128	serge	991	*/
1221	serge	992	void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
		993	{
		994	unsigned i;
		995	u32 tmp;
1128	serge	996
1430	serge	997	/* flush hdp cache so updates hit vram */
1963	serge	998	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		999	!(rdev->flags & RADEON_IS_AGP)) {
2997	Serge	1000	void __iomem ptr = (void )rdev->gart.ptr;
1963	serge	1001	u32 tmp;
		1002
		1003	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		1004	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
		1005	* This seems to cause problems on some AGP cards. Just use the old
		1006	* method for them.
		1007	*/
		1008	WREG32(HDP_DEBUG1, 0);
		1009	tmp = readl((void __iomem *)ptr);
		1010	} else
1430	serge	1011	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		1012
1221	serge	1013	WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
		1014	WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
		1015	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		1016	for (i = 0; i < rdev->usec_timeout; i++) {
		1017	/* read MC_STATUS */
		1018	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		1019	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		1020	if (tmp == 2) {
		1021	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		1022	return;
		1023	}
		1024	if (tmp) {
		1025	return;
		1026	}
		1027	udelay(1);
1128	serge	1028	}
1221	serge	1029	}
1128	serge	1030
1221	serge	1031	int r600_pcie_gart_init(struct radeon_device *rdev)
		1032	{
		1033	int r;
		1034
2997	Serge	1035	if (rdev->gart.robj) {
1963	serge	1036	WARN(1, "R600 PCIE GART already initialized\n");
1221	serge	1037	return 0;
		1038	}
		1039	/* Initialize common gart structure */
		1040	r = radeon_gart_init(rdev);
		1041	if (r)
		1042	return r;
		1043	rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
		1044	return radeon_gart_table_vram_alloc(rdev);
		1045	}
		1046
2997	Serge	1047	static int r600_pcie_gart_enable(struct radeon_device *rdev)
1221	serge	1048	{
		1049	u32 tmp;
		1050	int r, i;
		1051
2997	Serge	1052	if (rdev->gart.robj == NULL) {
1221	serge	1053	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		1054	return -EINVAL;
		1055	}
		1056	r = radeon_gart_table_vram_pin(rdev);
		1057	if (r)
		1058	return r;
		1059
		1060	/* Setup L2 cache */
		1061	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		1062	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1063	EFFECTIVE_L2_QUEUE_SIZE(7));
		1064	WREG32(VM_L2_CNTL2, 0);
		1065	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		1066	/* Setup TLB control */
		1067	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		1068	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		1069	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		1070	ENABLE_WAIT_L2_QUERY;
		1071	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		1072	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		1073	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		1074	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		1075	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		1076	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		1077	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		1078	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		1079	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		1080	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		1081	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		1082	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
5271	serge	1083	WREG32(MC_VM_L1_TLB_MCB_RD_UVD_CNTL, tmp);
		1084	WREG32(MC_VM_L1_TLB_MCB_WR_UVD_CNTL, tmp);
1221	serge	1085	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		1086	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		1087	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		1088	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		1089	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		1090	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		1091	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		1092	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		1093	(u32)(rdev->dummy_page.addr >> 12));
		1094	for (i = 1; i < 7; i++)
		1095	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		1096
		1097	r600_pcie_gart_tlb_flush(rdev);
2997	Serge	1098	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		1099	(unsigned)(rdev->mc.gtt_size >> 20),
		1100	(unsigned long long)rdev->gart.table_addr);
1221	serge	1101	rdev->gart.ready = true;
1128	serge	1102	return 0;
		1103	}
		1104
2997	Serge	1105	static void r600_pcie_gart_disable(struct radeon_device *rdev)
1128	serge	1106	{
1221	serge	1107	u32 tmp;
2997	Serge	1108	int i;
1221	serge	1109
		1110	/* Disable all tables */
		1111	for (i = 0; i < 7; i++)
		1112	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		1113
		1114	/* Disable L2 cache */
		1115	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		1116	EFFECTIVE_L2_QUEUE_SIZE(7));
		1117	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		1118	/* Setup L1 TLB control */
		1119	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		1120	ENABLE_WAIT_L2_QUERY;
		1121	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		1122	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		1123	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		1124	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		1125	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		1126	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		1127	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		1128	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		1129	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
		1130	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
		1131	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		1132	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		1133	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
		1134	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
5271	serge	1135	WREG32(MC_VM_L1_TLB_MCB_RD_UVD_CNTL, tmp);
		1136	WREG32(MC_VM_L1_TLB_MCB_WR_UVD_CNTL, tmp);
2997	Serge	1137	radeon_gart_table_vram_unpin(rdev);
1128	serge	1138	}
		1139
2997	Serge	1140	static void r600_pcie_gart_fini(struct radeon_device *rdev)
1221	serge	1141	{
1963	serge	1142	radeon_gart_fini(rdev);
1221	serge	1143	r600_pcie_gart_disable(rdev);
		1144	radeon_gart_table_vram_free(rdev);
		1145	}
1128	serge	1146
2997	Serge	1147	static void r600_agp_enable(struct radeon_device *rdev)
1128	serge	1148	{
1221	serge	1149	u32 tmp;
		1150	int i;
		1151
		1152	/* Setup L2 cache */
		1153	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		1154	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1155	EFFECTIVE_L2_QUEUE_SIZE(7));
		1156	WREG32(VM_L2_CNTL2, 0);
		1157	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		1158	/* Setup TLB control */
		1159	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		1160	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		1161	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		1162	ENABLE_WAIT_L2_QUERY;
		1163	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		1164	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		1165	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		1166	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		1167	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		1168	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		1169	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		1170	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		1171	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		1172	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		1173	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		1174	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		1175	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		1176	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		1177	for (i = 0; i < 7; i++)
		1178	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1128	serge	1179	}
		1180
		1181	int r600_mc_wait_for_idle(struct radeon_device *rdev)
		1182	{
1221	serge	1183	unsigned i;
		1184	u32 tmp;
		1185
		1186	for (i = 0; i < rdev->usec_timeout; i++) {
		1187	/* read MC_STATUS */
		1188	tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
		1189	if (!tmp)
1128	serge	1190	return 0;
1221	serge	1191	udelay(1);
		1192	}
		1193	return -1;
1128	serge	1194	}
		1195
3764	Serge	1196	uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg)
		1197	{
5078	serge	1198	unsigned long flags;
3764	Serge	1199	uint32_t r;
		1200
5078	serge	1201	spin_lock_irqsave(&rdev->mc_idx_lock, flags);
3764	Serge	1202	WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg));
		1203	r = RREG32(R_0028FC_MC_DATA);
		1204	WREG32(R_0028F8_MC_INDEX, ~C_0028F8_MC_IND_ADDR);
5078	serge	1205	spin_unlock_irqrestore(&rdev->mc_idx_lock, flags);
3764	Serge	1206	return r;
		1207	}
		1208
		1209	void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
		1210	{
5078	serge	1211	unsigned long flags;
		1212
		1213	spin_lock_irqsave(&rdev->mc_idx_lock, flags);
3764	Serge	1214	WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg) \|
		1215	S_0028F8_MC_IND_WR_EN(1));
		1216	WREG32(R_0028FC_MC_DATA, v);
		1217	WREG32(R_0028F8_MC_INDEX, 0x7F);
5078	serge	1218	spin_unlock_irqrestore(&rdev->mc_idx_lock, flags);
3764	Serge	1219	}
		1220
1221	serge	1221	static void r600_mc_program(struct radeon_device *rdev)
1128	serge	1222	{
1221	serge	1223	struct rv515_mc_save save;
		1224	u32 tmp;
		1225	int i, j;
		1226
		1227	/* Initialize HDP */
		1228	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1229	WREG32((0x2c14 + j), 0x00000000);
		1230	WREG32((0x2c18 + j), 0x00000000);
		1231	WREG32((0x2c1c + j), 0x00000000);
		1232	WREG32((0x2c20 + j), 0x00000000);
		1233	WREG32((0x2c24 + j), 0x00000000);
		1234	}
		1235	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		1236
		1237	rv515_mc_stop(rdev, &save);
		1238	if (r600_mc_wait_for_idle(rdev)) {
		1239	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1240	}
		1241	/* Lockout access through VGA aperture (doesn't exist before R600) */
		1242	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		1243	/* Update configuration */
		1244	if (rdev->flags & RADEON_IS_AGP) {
		1245	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		1246	/* VRAM before AGP */
		1247	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1248	rdev->mc.vram_start >> 12);
		1249	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1250	rdev->mc.gtt_end >> 12);
		1251	} else {
		1252	/* VRAM after AGP */
		1253	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1254	rdev->mc.gtt_start >> 12);
		1255	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1256	rdev->mc.vram_end >> 12);
		1257	}
		1258	} else {
		1259	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
		1260	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
		1261	}
2997	Serge	1262	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1221	serge	1263	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		1264	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		1265	WREG32(MC_VM_FB_LOCATION, tmp);
		1266	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
		1267	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1963	serge	1268	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1221	serge	1269	if (rdev->flags & RADEON_IS_AGP) {
		1270	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
		1271	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
		1272	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		1273	} else {
		1274	WREG32(MC_VM_AGP_BASE, 0);
		1275	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		1276	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		1277	}
		1278	if (r600_mc_wait_for_idle(rdev)) {
		1279	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1280	}
		1281	rv515_mc_resume(rdev, &save);
		1282	/* we need to own VRAM, so turn off the VGA renderer here
		1283	* to stop it overwriting our objects */
		1284	rv515_vga_render_disable(rdev);
1128	serge	1285	}
		1286
1430	serge	1287	/**
		1288	* r600_vram_gtt_location - try to find VRAM & GTT location
		1289	* @rdev: radeon device structure holding all necessary informations
		1290	* @mc: memory controller structure holding memory informations
		1291	*
		1292	* Function will place try to place VRAM at same place as in CPU (PCI)
		1293	* address space as some GPU seems to have issue when we reprogram at
		1294	* different address space.
		1295	*
		1296	* If there is not enough space to fit the unvisible VRAM after the
		1297	* aperture then we limit the VRAM size to the aperture.
		1298	*
		1299	* If we are using AGP then place VRAM adjacent to AGP aperture are we need
		1300	* them to be in one from GPU point of view so that we can program GPU to
		1301	* catch access outside them (weird GPU policy see ??).
		1302	*
		1303	* This function will never fails, worst case are limiting VRAM or GTT.
		1304	*
		1305	* Note: GTT start, end, size should be initialized before calling this
		1306	* function on AGP platform.
		1307	*/
1963	serge	1308	static void r600_vram_gtt_location(struct radeon_device rdev, struct radeon_mc mc)
1430	serge	1309	{
		1310	u64 size_bf, size_af;
		1311
		1312	if (mc->mc_vram_size > 0xE0000000) {
		1313	/* leave room for at least 512M GTT */
		1314	dev_warn(rdev->dev, "limiting VRAM\n");
		1315	mc->real_vram_size = 0xE0000000;
		1316	mc->mc_vram_size = 0xE0000000;
		1317	}
		1318	if (rdev->flags & RADEON_IS_AGP) {
		1319	size_bf = mc->gtt_start;
3764	Serge	1320	size_af = mc->mc_mask - mc->gtt_end;
1430	serge	1321	if (size_bf > size_af) {
		1322	if (mc->mc_vram_size > size_bf) {
		1323	dev_warn(rdev->dev, "limiting VRAM\n");
		1324	mc->real_vram_size = size_bf;
		1325	mc->mc_vram_size = size_bf;
		1326	}
		1327	mc->vram_start = mc->gtt_start - mc->mc_vram_size;
		1328	} else {
		1329	if (mc->mc_vram_size > size_af) {
		1330	dev_warn(rdev->dev, "limiting VRAM\n");
		1331	mc->real_vram_size = size_af;
		1332	mc->mc_vram_size = size_af;
		1333	}
2997	Serge	1334	mc->vram_start = mc->gtt_end + 1;
1430	serge	1335	}
		1336	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
		1337	dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
		1338	mc->mc_vram_size >> 20, mc->vram_start,
		1339	mc->vram_end, mc->real_vram_size >> 20);
		1340	} else {
		1341	u64 base = 0;
1963	serge	1342	if (rdev->flags & RADEON_IS_IGP) {
		1343	base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
		1344	base <<= 24;
		1345	}
1430	serge	1346	radeon_vram_location(rdev, &rdev->mc, base);
1963	serge	1347	rdev->mc.gtt_base_align = 0;
1430	serge	1348	radeon_gtt_location(rdev, mc);
		1349	}
		1350	}
		1351
2997	Serge	1352	static int r600_mc_init(struct radeon_device *rdev)
1128	serge	1353	{
1221	serge	1354	u32 tmp;
1268	serge	1355	int chansize, numchan;
3764	Serge	1356	uint32_t h_addr, l_addr;
		1357	unsigned long long k8_addr;
1128	serge	1358
1221	serge	1359	/* Get VRAM informations */
1128	serge	1360	rdev->mc.vram_is_ddr = true;
1221	serge	1361	tmp = RREG32(RAMCFG);
		1362	if (tmp & CHANSIZE_OVERRIDE) {
1128	serge	1363	chansize = 16;
1221	serge	1364	} else if (tmp & CHANSIZE_MASK) {
1128	serge	1365	chansize = 64;
		1366	} else {
		1367	chansize = 32;
		1368	}
1268	serge	1369	tmp = RREG32(CHMAP);
		1370	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		1371	case 0:
		1372	default:
		1373	numchan = 1;
		1374	break;
		1375	case 1:
		1376	numchan = 2;
		1377	break;
		1378	case 2:
		1379	numchan = 4;
		1380	break;
		1381	case 3:
		1382	numchan = 8;
		1383	break;
1128	serge	1384	}
1268	serge	1385	rdev->mc.vram_width = numchan * chansize;
1221	serge	1386	/* Could aper size report 0 ? */
1963	serge	1387	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		1388	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1221	serge	1389	/* Setup GPU memory space */
		1390	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		1391	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1430	serge	1392	rdev->mc.visible_vram_size = rdev->mc.aper_size;
		1393	r600_vram_gtt_location(rdev, &rdev->mc);
1963	serge	1394
		1395	if (rdev->flags & RADEON_IS_IGP) {
		1396	rs690_pm_info(rdev);
1403	serge	1397	rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
3764	Serge	1398
		1399	if (rdev->family == CHIP_RS780 \|\| rdev->family == CHIP_RS880) {
		1400	/* Use K8 direct mapping for fast fb access. */
		1401	rdev->fastfb_working = false;
		1402	h_addr = G_000012_K8_ADDR_EXT(RREG32_MC(R_000012_MC_MISC_UMA_CNTL));
		1403	l_addr = RREG32_MC(R_000011_K8_FB_LOCATION);
		1404	k8_addr = ((unsigned long long)h_addr) << 32 \| l_addr;
		1405	#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
		1406	if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL)
		1407	#endif
		1408	{
		1409	/* FastFB shall be used with UMA memory. Here it is simply disabled when sideport
		1410	* memory is present.
		1411	*/
		1412	if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) {
		1413	DRM_INFO("Direct mapping: aper base at 0x%llx, replaced by direct mapping base 0x%llx.\n",
		1414	(unsigned long long)rdev->mc.aper_base, k8_addr);
		1415	rdev->mc.aper_base = (resource_size_t)k8_addr;
		1416	rdev->fastfb_working = true;
1963	serge	1417	}
3764	Serge	1418	}
		1419	}
		1420	}
		1421
1963	serge	1422	radeon_update_bandwidth_info(rdev);
1221	serge	1423	return 0;
1128	serge	1424	}
		1425
2997	Serge	1426	int r600_vram_scratch_init(struct radeon_device *rdev)
		1427	{
		1428	int r;
		1429
		1430	if (rdev->vram_scratch.robj == NULL) {
		1431	r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
		1432	PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
5271	serge	1433	0, NULL, NULL, &rdev->vram_scratch.robj);
2997	Serge	1434	if (r) {
		1435	return r;
		1436	}
		1437	}
		1438
		1439	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
		1440	if (unlikely(r != 0))
		1441	return r;
		1442	r = radeon_bo_pin(rdev->vram_scratch.robj,
		1443	RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
		1444	if (r) {
		1445	radeon_bo_unreserve(rdev->vram_scratch.robj);
		1446	return r;
		1447	}
		1448	r = radeon_bo_kmap(rdev->vram_scratch.robj,
		1449	(void **)&rdev->vram_scratch.ptr);
		1450	if (r)
		1451	radeon_bo_unpin(rdev->vram_scratch.robj);
		1452	radeon_bo_unreserve(rdev->vram_scratch.robj);
		1453
		1454	return r;
		1455	}
3764	Serge	1456
		1457	void r600_vram_scratch_fini(struct radeon_device *rdev)
1128	serge	1458	{
3764	Serge	1459	int r;
1128	serge	1460
3764	Serge	1461	if (rdev->vram_scratch.robj == NULL) {
3192	Serge	1462	return;
3764	Serge	1463	}
		1464	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
		1465	if (likely(r == 0)) {
		1466	radeon_bo_kunmap(rdev->vram_scratch.robj);
		1467	radeon_bo_unpin(rdev->vram_scratch.robj);
		1468	radeon_bo_unreserve(rdev->vram_scratch.robj);
		1469	}
		1470	radeon_bo_unref(&rdev->vram_scratch.robj);
		1471	}
1963	serge	1472
3764	Serge	1473	void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
		1474	{
		1475	u32 tmp = RREG32(R600_BIOS_3_SCRATCH);
		1476
		1477	if (hung)
		1478	tmp \|= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
		1479	else
		1480	tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
		1481
		1482	WREG32(R600_BIOS_3_SCRATCH, tmp);
		1483	}
		1484
		1485	static void r600_print_gpu_status_regs(struct radeon_device *rdev)
		1486	{
3192	Serge	1487	dev_info(rdev->dev, " R_008010_GRBM_STATUS = 0x%08X\n",
1221	serge	1488	RREG32(R_008010_GRBM_STATUS));
3192	Serge	1489	dev_info(rdev->dev, " R_008014_GRBM_STATUS2 = 0x%08X\n",
1221	serge	1490	RREG32(R_008014_GRBM_STATUS2));
3192	Serge	1491	dev_info(rdev->dev, " R_000E50_SRBM_STATUS = 0x%08X\n",
1221	serge	1492	RREG32(R_000E50_SRBM_STATUS));
2997	Serge	1493	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		1494	RREG32(CP_STALLED_STAT1));
		1495	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		1496	RREG32(CP_STALLED_STAT2));
		1497	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		1498	RREG32(CP_BUSY_STAT));
		1499	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		1500	RREG32(CP_STAT));
3764	Serge	1501	dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
		1502	RREG32(DMA_STATUS_REG));
		1503	}
3192	Serge	1504
3764	Serge	1505	static bool r600_is_display_hung(struct radeon_device *rdev)
		1506	{
		1507	u32 crtc_hung = 0;
		1508	u32 crtc_status[2];
		1509	u32 i, j, tmp;
		1510
		1511	for (i = 0; i < rdev->num_crtc; i++) {
		1512	if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) {
		1513	crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
		1514	crtc_hung \|= (1 << i);
		1515	}
		1516	}
		1517
		1518	for (j = 0; j < 10; j++) {
		1519	for (i = 0; i < rdev->num_crtc; i++) {
		1520	if (crtc_hung & (1 << i)) {
		1521	tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
		1522	if (tmp != crtc_status[i])
		1523	crtc_hung &= ~(1 << i);
		1524	}
		1525	}
		1526	if (crtc_hung == 0)
		1527	return false;
		1528	udelay(100);
		1529	}
		1530
		1531	return true;
		1532	}
		1533
5078	serge	1534	u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
3764	Serge	1535	{
		1536	u32 reset_mask = 0;
		1537	u32 tmp;
		1538
		1539	/* GRBM_STATUS */
		1540	tmp = RREG32(R_008010_GRBM_STATUS);
		1541	if (rdev->family >= CHIP_RV770) {
		1542	if (G_008010_PA_BUSY(tmp) \| G_008010_SC_BUSY(tmp) \|
		1543	G_008010_SH_BUSY(tmp) \| G_008010_SX_BUSY(tmp) \|
		1544	G_008010_TA_BUSY(tmp) \| G_008010_VGT_BUSY(tmp) \|
		1545	G_008010_DB03_BUSY(tmp) \| G_008010_CB03_BUSY(tmp) \|
		1546	G_008010_SPI03_BUSY(tmp) \| G_008010_VGT_BUSY_NO_DMA(tmp))
		1547	reset_mask \|= RADEON_RESET_GFX;
		1548	} else {
		1549	if (G_008010_PA_BUSY(tmp) \| G_008010_SC_BUSY(tmp) \|
		1550	G_008010_SH_BUSY(tmp) \| G_008010_SX_BUSY(tmp) \|
		1551	G_008010_TA03_BUSY(tmp) \| G_008010_VGT_BUSY(tmp) \|
		1552	G_008010_DB03_BUSY(tmp) \| G_008010_CB03_BUSY(tmp) \|
		1553	G_008010_SPI03_BUSY(tmp) \| G_008010_VGT_BUSY_NO_DMA(tmp))
		1554	reset_mask \|= RADEON_RESET_GFX;
		1555	}
		1556
		1557	if (G_008010_CF_RQ_PENDING(tmp) \| G_008010_PF_RQ_PENDING(tmp) \|
		1558	G_008010_CP_BUSY(tmp) \| G_008010_CP_COHERENCY_BUSY(tmp))
		1559	reset_mask \|= RADEON_RESET_CP;
		1560
		1561	if (G_008010_GRBM_EE_BUSY(tmp))
		1562	reset_mask \|= RADEON_RESET_GRBM \| RADEON_RESET_GFX \| RADEON_RESET_CP;
		1563
		1564	/* DMA_STATUS_REG */
		1565	tmp = RREG32(DMA_STATUS_REG);
		1566	if (!(tmp & DMA_IDLE))
		1567	reset_mask \|= RADEON_RESET_DMA;
		1568
		1569	/* SRBM_STATUS */
		1570	tmp = RREG32(R_000E50_SRBM_STATUS);
		1571	if (G_000E50_RLC_RQ_PENDING(tmp) \| G_000E50_RLC_BUSY(tmp))
		1572	reset_mask \|= RADEON_RESET_RLC;
		1573
		1574	if (G_000E50_IH_BUSY(tmp))
		1575	reset_mask \|= RADEON_RESET_IH;
		1576
		1577	if (G_000E50_SEM_BUSY(tmp))
		1578	reset_mask \|= RADEON_RESET_SEM;
		1579
		1580	if (G_000E50_GRBM_RQ_PENDING(tmp))
		1581	reset_mask \|= RADEON_RESET_GRBM;
		1582
		1583	if (G_000E50_VMC_BUSY(tmp))
		1584	reset_mask \|= RADEON_RESET_VMC;
		1585
		1586	if (G_000E50_MCB_BUSY(tmp) \| G_000E50_MCDZ_BUSY(tmp) \|
		1587	G_000E50_MCDY_BUSY(tmp) \| G_000E50_MCDX_BUSY(tmp) \|
		1588	G_000E50_MCDW_BUSY(tmp))
		1589	reset_mask \|= RADEON_RESET_MC;
		1590
		1591	if (r600_is_display_hung(rdev))
		1592	reset_mask \|= RADEON_RESET_DISPLAY;
		1593
		1594	/* Skip MC reset as it's mostly likely not hung, just busy */
		1595	if (reset_mask & RADEON_RESET_MC) {
		1596	DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
		1597	reset_mask &= ~RADEON_RESET_MC;
		1598	}
		1599
		1600	return reset_mask;
		1601	}
		1602
		1603	static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
		1604	{
		1605	struct rv515_mc_save save;
		1606	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
		1607	u32 tmp;
		1608
		1609	if (reset_mask == 0)
		1610	return;
		1611
		1612	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
		1613
		1614	r600_print_gpu_status_regs(rdev);
		1615
1221	serge	1616	/* Disable CP parsing/prefetching */
3764	Serge	1617	if (rdev->family >= CHIP_RV770)
		1618	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) \| S_0086D8_CP_PFP_HALT(1));
		1619	else
1963	serge	1620	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
3192	Serge	1621
3764	Serge	1622	/* disable the RLC */
		1623	WREG32(RLC_CNTL, 0);
		1624
		1625	if (reset_mask & RADEON_RESET_DMA) {
		1626	/* Disable DMA */
		1627	tmp = RREG32(DMA_RB_CNTL);
		1628	tmp &= ~DMA_RB_ENABLE;
		1629	WREG32(DMA_RB_CNTL, tmp);
		1630	}
		1631
		1632	mdelay(50);
		1633
		1634	rv515_mc_stop(rdev, &save);
		1635	if (r600_mc_wait_for_idle(rdev)) {
		1636	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1637	}
		1638
		1639	if (reset_mask & (RADEON_RESET_GFX \| RADEON_RESET_COMPUTE)) {
		1640	if (rdev->family >= CHIP_RV770)
		1641	grbm_soft_reset \|= S_008020_SOFT_RESET_DB(1) \|
		1642	S_008020_SOFT_RESET_CB(1) \|
		1643	S_008020_SOFT_RESET_PA(1) \|
		1644	S_008020_SOFT_RESET_SC(1) \|
		1645	S_008020_SOFT_RESET_SPI(1) \|
		1646	S_008020_SOFT_RESET_SX(1) \|
		1647	S_008020_SOFT_RESET_SH(1) \|
		1648	S_008020_SOFT_RESET_TC(1) \|
		1649	S_008020_SOFT_RESET_TA(1) \|
		1650	S_008020_SOFT_RESET_VC(1) \|
		1651	S_008020_SOFT_RESET_VGT(1);
		1652	else
		1653	grbm_soft_reset \|= S_008020_SOFT_RESET_CR(1) \|
1221	serge	1654	S_008020_SOFT_RESET_DB(1) \|
		1655	S_008020_SOFT_RESET_CB(1) \|
		1656	S_008020_SOFT_RESET_PA(1) \|
		1657	S_008020_SOFT_RESET_SC(1) \|
		1658	S_008020_SOFT_RESET_SMX(1) \|
		1659	S_008020_SOFT_RESET_SPI(1) \|
		1660	S_008020_SOFT_RESET_SX(1) \|
		1661	S_008020_SOFT_RESET_SH(1) \|
		1662	S_008020_SOFT_RESET_TC(1) \|
		1663	S_008020_SOFT_RESET_TA(1) \|
		1664	S_008020_SOFT_RESET_VC(1) \|
		1665	S_008020_SOFT_RESET_VGT(1);
		1666	}
3764	Serge	1667
		1668	if (reset_mask & RADEON_RESET_CP) {
		1669	grbm_soft_reset \|= S_008020_SOFT_RESET_CP(1) \|
		1670	S_008020_SOFT_RESET_VGT(1);
		1671
		1672	srbm_soft_reset \|= S_000E60_SOFT_RESET_GRBM(1);
		1673	}
		1674
		1675	if (reset_mask & RADEON_RESET_DMA) {
		1676	if (rdev->family >= CHIP_RV770)
		1677	srbm_soft_reset \|= RV770_SOFT_RESET_DMA;
		1678	else
		1679	srbm_soft_reset \|= SOFT_RESET_DMA;
		1680	}
		1681
		1682	if (reset_mask & RADEON_RESET_RLC)
		1683	srbm_soft_reset \|= S_000E60_SOFT_RESET_RLC(1);
		1684
		1685	if (reset_mask & RADEON_RESET_SEM)
		1686	srbm_soft_reset \|= S_000E60_SOFT_RESET_SEM(1);
		1687
		1688	if (reset_mask & RADEON_RESET_IH)
		1689	srbm_soft_reset \|= S_000E60_SOFT_RESET_IH(1);
		1690
		1691	if (reset_mask & RADEON_RESET_GRBM)
		1692	srbm_soft_reset \|= S_000E60_SOFT_RESET_GRBM(1);
		1693
		1694	if (!(rdev->flags & RADEON_IS_IGP)) {
		1695	if (reset_mask & RADEON_RESET_MC)
		1696	srbm_soft_reset \|= S_000E60_SOFT_RESET_MC(1);
		1697	}
		1698
		1699	if (reset_mask & RADEON_RESET_VMC)
		1700	srbm_soft_reset \|= S_000E60_SOFT_RESET_VMC(1);
		1701
		1702	if (grbm_soft_reset) {
		1703	tmp = RREG32(R_008020_GRBM_SOFT_RESET);
		1704	tmp \|= grbm_soft_reset;
1221	serge	1705	dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		1706	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
3764	Serge	1707	tmp = RREG32(R_008020_GRBM_SOFT_RESET);
3192	Serge	1708
3764	Serge	1709	udelay(50);
3192	Serge	1710
3764	Serge	1711	tmp &= ~grbm_soft_reset;
		1712	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
		1713	tmp = RREG32(R_008020_GRBM_SOFT_RESET);
		1714	}
3192	Serge	1715
3764	Serge	1716	if (srbm_soft_reset) {
		1717	tmp = RREG32(SRBM_SOFT_RESET);
		1718	tmp \|= srbm_soft_reset;
		1719	dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
		1720	WREG32(SRBM_SOFT_RESET, tmp);
		1721	tmp = RREG32(SRBM_SOFT_RESET);
3192	Serge	1722
3764	Serge	1723	udelay(50);
3192	Serge	1724
3764	Serge	1725	tmp &= ~srbm_soft_reset;
		1726	WREG32(SRBM_SOFT_RESET, tmp);
		1727	tmp = RREG32(SRBM_SOFT_RESET);
		1728	}
3192	Serge	1729
3764	Serge	1730	/* Wait a little for things to settle down */
		1731	mdelay(1);
3192	Serge	1732
3764	Serge	1733	rv515_mc_resume(rdev, &save);
3192	Serge	1734	udelay(50);
		1735
3764	Serge	1736	r600_print_gpu_status_regs(rdev);
3192	Serge	1737	}
		1738
5078	serge	1739	static void r600_gpu_pci_config_reset(struct radeon_device *rdev)
		1740	{
		1741	struct rv515_mc_save save;
		1742	u32 tmp, i;
		1743
		1744	dev_info(rdev->dev, "GPU pci config reset\n");
		1745
		1746	/* disable dpm? */
		1747
		1748	/* Disable CP parsing/prefetching */
		1749	if (rdev->family >= CHIP_RV770)
		1750	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) \| S_0086D8_CP_PFP_HALT(1));
		1751	else
		1752	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
		1753
		1754	/* disable the RLC */
		1755	WREG32(RLC_CNTL, 0);
		1756
		1757	/* Disable DMA */
		1758	tmp = RREG32(DMA_RB_CNTL);
		1759	tmp &= ~DMA_RB_ENABLE;
		1760	WREG32(DMA_RB_CNTL, tmp);
		1761
		1762	mdelay(50);
		1763
		1764	/* set mclk/sclk to bypass */
		1765	if (rdev->family >= CHIP_RV770)
		1766	rv770_set_clk_bypass_mode(rdev);
		1767	/* disable BM */
		1768	pci_clear_master(rdev->pdev);
		1769	/* disable mem access */
		1770	rv515_mc_stop(rdev, &save);
		1771	if (r600_mc_wait_for_idle(rdev)) {
		1772	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1773	}
		1774
		1775	/* BIF reset workaround. Not sure if this is needed on 6xx */
		1776	tmp = RREG32(BUS_CNTL);
		1777	tmp \|= VGA_COHE_SPEC_TIMER_DIS;
		1778	WREG32(BUS_CNTL, tmp);
		1779
		1780	tmp = RREG32(BIF_SCRATCH0);
		1781
		1782	/* reset */
		1783	radeon_pci_config_reset(rdev);
		1784	mdelay(1);
		1785
		1786	/* BIF reset workaround. Not sure if this is needed on 6xx */
		1787	tmp = SOFT_RESET_BIF;
		1788	WREG32(SRBM_SOFT_RESET, tmp);
		1789	mdelay(1);
		1790	WREG32(SRBM_SOFT_RESET, 0);
		1791
		1792	/* wait for asic to come out of reset */
		1793	for (i = 0; i < rdev->usec_timeout; i++) {
		1794	if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
		1795	break;
		1796	udelay(1);
		1797	}
		1798	}
		1799
3764	Serge	1800	int r600_asic_reset(struct radeon_device *rdev)
3192	Serge	1801	{
3764	Serge	1802	u32 reset_mask;
3192	Serge	1803
3764	Serge	1804	reset_mask = r600_gpu_check_soft_reset(rdev);
3192	Serge	1805
3764	Serge	1806	if (reset_mask)
		1807	r600_set_bios_scratch_engine_hung(rdev, true);
3192	Serge	1808
5078	serge	1809	/* try soft reset */
3764	Serge	1810	r600_gpu_soft_reset(rdev, reset_mask);
3192	Serge	1811
3764	Serge	1812	reset_mask = r600_gpu_check_soft_reset(rdev);
3192	Serge	1813
5078	serge	1814	/* try pci config reset */
		1815	if (reset_mask && radeon_hard_reset)
		1816	r600_gpu_pci_config_reset(rdev);
		1817
		1818	reset_mask = r600_gpu_check_soft_reset(rdev);
		1819
3764	Serge	1820	if (!reset_mask)
		1821	r600_set_bios_scratch_engine_hung(rdev, false);
3192	Serge	1822
1221	serge	1823	return 0;
1128	serge	1824	}
		1825
3764	Serge	1826	/**
		1827	* r600_gfx_is_lockup - Check if the GFX engine is locked up
		1828	*
		1829	* @rdev: radeon_device pointer
		1830	* @ring: radeon_ring structure holding ring information
		1831	*
		1832	* Check if the GFX engine is locked up.
		1833	* Returns true if the engine appears to be locked up, false if not.
		1834	*/
		1835	bool r600_gfx_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
1221	serge	1836	{
3764	Serge	1837	u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1963	serge	1838
3764	Serge	1839	if (!(reset_mask & (RADEON_RESET_GFX \|
		1840	RADEON_RESET_COMPUTE \|
		1841	RADEON_RESET_CP))) {
5078	serge	1842	radeon_ring_lockup_update(rdev, ring);
1963	serge	1843	return false;
		1844	}
2997	Serge	1845	return radeon_ring_test_lockup(rdev, ring);
1963	serge	1846	}
		1847
2997	Serge	1848	u32 r6xx_remap_render_backend(struct radeon_device *rdev,
		1849	u32 tiling_pipe_num,
		1850	u32 max_rb_num,
		1851	u32 total_max_rb_num,
		1852	u32 disabled_rb_mask)
1221	serge	1853	{
2997	Serge	1854	u32 rendering_pipe_num, rb_num_width, req_rb_num;
3764	Serge	1855	u32 pipe_rb_ratio, pipe_rb_remain, tmp;
2997	Serge	1856	u32 data = 0, mask = 1 << (max_rb_num - 1);
		1857	unsigned i, j;
1221	serge	1858
2997	Serge	1859	/* mask out the RBs that don't exist on that asic */
3764	Serge	1860	tmp = disabled_rb_mask \| ((0xff << max_rb_num) & 0xff);
		1861	/* make sure at least one RB is available */
		1862	if ((tmp & 0xff) != 0xff)
		1863	disabled_rb_mask = tmp;
1221	serge	1864
2997	Serge	1865	rendering_pipe_num = 1 << tiling_pipe_num;
		1866	req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
		1867	BUG_ON(rendering_pipe_num < req_rb_num);
1221	serge	1868
2997	Serge	1869	pipe_rb_ratio = rendering_pipe_num / req_rb_num;
		1870	pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1221	serge	1871
2997	Serge	1872	if (rdev->family <= CHIP_RV740) {
		1873	/* r6xx/r7xx */
		1874	rb_num_width = 2;
		1875	} else {
		1876	/* eg+ */
		1877	rb_num_width = 4;
		1878	}
1221	serge	1879
2997	Serge	1880	for (i = 0; i < max_rb_num; i++) {
		1881	if (!(mask & disabled_rb_mask)) {
		1882	for (j = 0; j < pipe_rb_ratio; j++) {
		1883	data <<= rb_num_width;
		1884	data \|= max_rb_num - i - 1;
1221	serge	1885	}
2997	Serge	1886	if (pipe_rb_remain) {
		1887	data <<= rb_num_width;
		1888	data \|= max_rb_num - i - 1;
		1889	pipe_rb_remain--;
1221	serge	1890	}
2997	Serge	1891	}
		1892	mask >>= 1;
		1893	}
1221	serge	1894
2997	Serge	1895	return data;
1221	serge	1896	}
		1897
		1898	int r600_count_pipe_bits(uint32_t val)
		1899	{
3192	Serge	1900	return hweight32(val);
1221	serge	1901	}
		1902
2997	Serge	1903	static void r600_gpu_init(struct radeon_device *rdev)
1221	serge	1904	{
		1905	u32 tiling_config;
		1906	u32 ramcfg;
1430	serge	1907	u32 cc_gc_shader_pipe_config;
1221	serge	1908	u32 tmp;
		1909	int i, j;
		1910	u32 sq_config;
		1911	u32 sq_gpr_resource_mgmt_1 = 0;
		1912	u32 sq_gpr_resource_mgmt_2 = 0;
		1913	u32 sq_thread_resource_mgmt = 0;
		1914	u32 sq_stack_resource_mgmt_1 = 0;
		1915	u32 sq_stack_resource_mgmt_2 = 0;
2997	Serge	1916	u32 disabled_rb_mask;
1221	serge	1917
2997	Serge	1918	rdev->config.r600.tiling_group_size = 256;
1221	serge	1919	switch (rdev->family) {
		1920	case CHIP_R600:
		1921	rdev->config.r600.max_pipes = 4;
		1922	rdev->config.r600.max_tile_pipes = 8;
		1923	rdev->config.r600.max_simds = 4;
		1924	rdev->config.r600.max_backends = 4;
		1925	rdev->config.r600.max_gprs = 256;
		1926	rdev->config.r600.max_threads = 192;
		1927	rdev->config.r600.max_stack_entries = 256;
		1928	rdev->config.r600.max_hw_contexts = 8;
		1929	rdev->config.r600.max_gs_threads = 16;
		1930	rdev->config.r600.sx_max_export_size = 128;
		1931	rdev->config.r600.sx_max_export_pos_size = 16;
		1932	rdev->config.r600.sx_max_export_smx_size = 128;
		1933	rdev->config.r600.sq_num_cf_insts = 2;
		1934	break;
		1935	case CHIP_RV630:
		1936	case CHIP_RV635:
		1937	rdev->config.r600.max_pipes = 2;
		1938	rdev->config.r600.max_tile_pipes = 2;
		1939	rdev->config.r600.max_simds = 3;
		1940	rdev->config.r600.max_backends = 1;
		1941	rdev->config.r600.max_gprs = 128;
		1942	rdev->config.r600.max_threads = 192;
		1943	rdev->config.r600.max_stack_entries = 128;
		1944	rdev->config.r600.max_hw_contexts = 8;
		1945	rdev->config.r600.max_gs_threads = 4;
		1946	rdev->config.r600.sx_max_export_size = 128;
		1947	rdev->config.r600.sx_max_export_pos_size = 16;
		1948	rdev->config.r600.sx_max_export_smx_size = 128;
		1949	rdev->config.r600.sq_num_cf_insts = 2;
		1950	break;
		1951	case CHIP_RV610:
		1952	case CHIP_RV620:
		1953	case CHIP_RS780:
		1954	case CHIP_RS880:
		1955	rdev->config.r600.max_pipes = 1;
		1956	rdev->config.r600.max_tile_pipes = 1;
		1957	rdev->config.r600.max_simds = 2;
		1958	rdev->config.r600.max_backends = 1;
		1959	rdev->config.r600.max_gprs = 128;
		1960	rdev->config.r600.max_threads = 192;
		1961	rdev->config.r600.max_stack_entries = 128;
		1962	rdev->config.r600.max_hw_contexts = 4;
		1963	rdev->config.r600.max_gs_threads = 4;
		1964	rdev->config.r600.sx_max_export_size = 128;
		1965	rdev->config.r600.sx_max_export_pos_size = 16;
		1966	rdev->config.r600.sx_max_export_smx_size = 128;
		1967	rdev->config.r600.sq_num_cf_insts = 1;
		1968	break;
		1969	case CHIP_RV670:
		1970	rdev->config.r600.max_pipes = 4;
		1971	rdev->config.r600.max_tile_pipes = 4;
		1972	rdev->config.r600.max_simds = 4;
		1973	rdev->config.r600.max_backends = 4;
		1974	rdev->config.r600.max_gprs = 192;
		1975	rdev->config.r600.max_threads = 192;
		1976	rdev->config.r600.max_stack_entries = 256;
		1977	rdev->config.r600.max_hw_contexts = 8;
		1978	rdev->config.r600.max_gs_threads = 16;
		1979	rdev->config.r600.sx_max_export_size = 128;
		1980	rdev->config.r600.sx_max_export_pos_size = 16;
		1981	rdev->config.r600.sx_max_export_smx_size = 128;
		1982	rdev->config.r600.sq_num_cf_insts = 2;
		1983	break;
		1984	default:
		1985	break;
		1986	}
		1987
		1988	/* Initialize HDP */
		1989	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1990	WREG32((0x2c14 + j), 0x00000000);
		1991	WREG32((0x2c18 + j), 0x00000000);
		1992	WREG32((0x2c1c + j), 0x00000000);
		1993	WREG32((0x2c20 + j), 0x00000000);
		1994	WREG32((0x2c24 + j), 0x00000000);
		1995	}
		1996
		1997	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1998
		1999	/* Setup tiling */
		2000	tiling_config = 0;
		2001	ramcfg = RREG32(RAMCFG);
		2002	switch (rdev->config.r600.max_tile_pipes) {
		2003	case 1:
		2004	tiling_config \|= PIPE_TILING(0);
		2005	break;
		2006	case 2:
		2007	tiling_config \|= PIPE_TILING(1);
		2008	break;
		2009	case 4:
		2010	tiling_config \|= PIPE_TILING(2);
		2011	break;
		2012	case 8:
		2013	tiling_config \|= PIPE_TILING(3);
		2014	break;
		2015	default:
		2016	break;
		2017	}
1430	serge	2018	rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
		2019	rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1221	serge	2020	tiling_config \|= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1963	serge	2021	tiling_config \|= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
2997	Serge	2022
1221	serge	2023	tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
		2024	if (tmp > 3) {
		2025	tiling_config \|= ROW_TILING(3);
		2026	tiling_config \|= SAMPLE_SPLIT(3);
		2027	} else {
		2028	tiling_config \|= ROW_TILING(tmp);
		2029	tiling_config \|= SAMPLE_SPLIT(tmp);
		2030	}
		2031	tiling_config \|= BANK_SWAPS(1);
1430	serge	2032
2997	Serge	2033	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
5078	serge	2034	tmp = rdev->config.r600.max_simds -
2997	Serge	2035	r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
5078	serge	2036	rdev->config.r600.active_simds = tmp;
1430	serge	2037
2997	Serge	2038	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
5078	serge	2039	tmp = 0;
		2040	for (i = 0; i < rdev->config.r600.max_backends; i++)
		2041	tmp \|= (1 << i);
		2042	/* if all the backends are disabled, fix it up here */
		2043	if ((disabled_rb_mask & tmp) == tmp) {
		2044	for (i = 0; i < rdev->config.r600.max_backends; i++)
		2045	disabled_rb_mask &= ~(1 << i);
		2046	}
2997	Serge	2047	tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
		2048	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
		2049	R6XX_MAX_BACKENDS, disabled_rb_mask);
		2050	tiling_config \|= tmp << 16;
		2051	rdev->config.r600.backend_map = tmp;
		2052
1963	serge	2053	rdev->config.r600.tile_config = tiling_config;
1221	serge	2054	WREG32(GB_TILING_CONFIG, tiling_config);
		2055	WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
		2056	WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
3192	Serge	2057	WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
1221	serge	2058
1430	serge	2059	tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1221	serge	2060	WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
		2061	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
		2062
		2063	/* Setup some CP states */
		2064	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \| ROQ_IB2_START(0x2b)));
		2065	WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) \| ROQ_END(0x40)));
		2066
		2067	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \| SYNC_GRADIENT \|
		2068	SYNC_WALKER \| SYNC_ALIGNER));
		2069	/* Setup various GPU states */
		2070	if (rdev->family == CHIP_RV670)
		2071	WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
		2072
		2073	tmp = RREG32(SX_DEBUG_1);
		2074	tmp \|= SMX_EVENT_RELEASE;
		2075	if ((rdev->family > CHIP_R600))
		2076	tmp \|= ENABLE_NEW_SMX_ADDRESS;
		2077	WREG32(SX_DEBUG_1, tmp);
		2078
		2079	if (((rdev->family) == CHIP_R600) \|\|
		2080	((rdev->family) == CHIP_RV630) \|\|
		2081	((rdev->family) == CHIP_RV610) \|\|
		2082	((rdev->family) == CHIP_RV620) \|\|
1268	serge	2083	((rdev->family) == CHIP_RS780) \|\|
		2084	((rdev->family) == CHIP_RS880)) {
1221	serge	2085	WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
		2086	} else {
		2087	WREG32(DB_DEBUG, 0);
		2088	}
		2089	WREG32(DB_WATERMARKS, (DEPTH_FREE(4) \| DEPTH_CACHELINE_FREE(16) \|
		2090	DEPTH_FLUSH(16) \| DEPTH_PENDING_FREE(4)));
		2091
		2092	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		2093	WREG32(VGT_NUM_INSTANCES, 0);
		2094
		2095	WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
		2096	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
		2097
		2098	tmp = RREG32(SQ_MS_FIFO_SIZES);
		2099	if (((rdev->family) == CHIP_RV610) \|\|
		2100	((rdev->family) == CHIP_RV620) \|\|
1268	serge	2101	((rdev->family) == CHIP_RS780) \|\|
		2102	((rdev->family) == CHIP_RS880)) {
1221	serge	2103	tmp = (CACHE_FIFO_SIZE(0xa) \|
		2104	FETCH_FIFO_HIWATER(0xa) \|
		2105	DONE_FIFO_HIWATER(0xe0) \|
		2106	ALU_UPDATE_FIFO_HIWATER(0x8));
		2107	} else if (((rdev->family) == CHIP_R600) \|\|
		2108	((rdev->family) == CHIP_RV630)) {
		2109	tmp &= ~DONE_FIFO_HIWATER(0xff);
		2110	tmp \|= DONE_FIFO_HIWATER(0x4);
		2111	}
		2112	WREG32(SQ_MS_FIFO_SIZES, tmp);
		2113
		2114	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
		2115	* should be adjusted as needed by the 2D/3D drivers. This just sets default values
		2116	*/
		2117	sq_config = RREG32(SQ_CONFIG);
		2118	sq_config &= ~(PS_PRIO(3) \|
		2119	VS_PRIO(3) \|
		2120	GS_PRIO(3) \|
		2121	ES_PRIO(3));
		2122	sq_config \|= (DX9_CONSTS \|
		2123	VC_ENABLE \|
		2124	PS_PRIO(0) \|
		2125	VS_PRIO(1) \|
		2126	GS_PRIO(2) \|
		2127	ES_PRIO(3));
		2128
		2129	if ((rdev->family) == CHIP_R600) {
		2130	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) \|
		2131	NUM_VS_GPRS(124) \|
		2132	NUM_CLAUSE_TEMP_GPRS(4));
		2133	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) \|
		2134	NUM_ES_GPRS(0));
		2135	sq_thread_resource_mgmt = (NUM_PS_THREADS(136) \|
		2136	NUM_VS_THREADS(48) \|
		2137	NUM_GS_THREADS(4) \|
		2138	NUM_ES_THREADS(4));
		2139	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) \|
		2140	NUM_VS_STACK_ENTRIES(128));
		2141	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) \|
		2142	NUM_ES_STACK_ENTRIES(0));
		2143	} else if (((rdev->family) == CHIP_RV610) \|\|
		2144	((rdev->family) == CHIP_RV620) \|\|
1268	serge	2145	((rdev->family) == CHIP_RS780) \|\|
		2146	((rdev->family) == CHIP_RS880)) {
1221	serge	2147	/* no vertex cache */
		2148	sq_config &= ~VC_ENABLE;
		2149
		2150	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		2151	NUM_VS_GPRS(44) \|
		2152	NUM_CLAUSE_TEMP_GPRS(2));
		2153	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		2154	NUM_ES_GPRS(17));
		2155	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		2156	NUM_VS_THREADS(78) \|
		2157	NUM_GS_THREADS(4) \|
		2158	NUM_ES_THREADS(31));
		2159	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		2160	NUM_VS_STACK_ENTRIES(40));
		2161	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		2162	NUM_ES_STACK_ENTRIES(16));
		2163	} else if (((rdev->family) == CHIP_RV630) \|\|
		2164	((rdev->family) == CHIP_RV635)) {
		2165	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		2166	NUM_VS_GPRS(44) \|
		2167	NUM_CLAUSE_TEMP_GPRS(2));
		2168	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) \|
		2169	NUM_ES_GPRS(18));
		2170	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		2171	NUM_VS_THREADS(78) \|
		2172	NUM_GS_THREADS(4) \|
		2173	NUM_ES_THREADS(31));
		2174	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		2175	NUM_VS_STACK_ENTRIES(40));
		2176	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		2177	NUM_ES_STACK_ENTRIES(16));
		2178	} else if ((rdev->family) == CHIP_RV670) {
		2179	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		2180	NUM_VS_GPRS(44) \|
		2181	NUM_CLAUSE_TEMP_GPRS(2));
		2182	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		2183	NUM_ES_GPRS(17));
		2184	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		2185	NUM_VS_THREADS(78) \|
		2186	NUM_GS_THREADS(4) \|
		2187	NUM_ES_THREADS(31));
		2188	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) \|
		2189	NUM_VS_STACK_ENTRIES(64));
		2190	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) \|
		2191	NUM_ES_STACK_ENTRIES(64));
		2192	}
		2193
		2194	WREG32(SQ_CONFIG, sq_config);
		2195	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		2196	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		2197	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		2198	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		2199	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		2200
		2201	if (((rdev->family) == CHIP_RV610) \|\|
		2202	((rdev->family) == CHIP_RV620) \|\|
1268	serge	2203	((rdev->family) == CHIP_RS780) \|\|
		2204	((rdev->family) == CHIP_RS880)) {
1221	serge	2205	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
		2206	} else {
		2207	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
		2208	}
		2209
		2210	/* More default values. 2D/3D driver should adjust as needed */
		2211	WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) \| S0_Y(0x4) \|
		2212	S1_X(0x4) \| S1_Y(0xc)));
		2213	WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) \| S0_Y(0xe) \|
		2214	S1_X(0x2) \| S1_Y(0x2) \|
		2215	S2_X(0xa) \| S2_Y(0x6) \|
		2216	S3_X(0x6) \| S3_Y(0xa)));
		2217	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) \| S0_Y(0xb) \|
		2218	S1_X(0x4) \| S1_Y(0xc) \|
		2219	S2_X(0x1) \| S2_Y(0x6) \|
		2220	S3_X(0xa) \| S3_Y(0xe)));
		2221	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) \| S4_Y(0x1) \|
		2222	S5_X(0x0) \| S5_Y(0x0) \|
		2223	S6_X(0xb) \| S6_Y(0x4) \|
		2224	S7_X(0x7) \| S7_Y(0x8)));
		2225
		2226	WREG32(VGT_STRMOUT_EN, 0);
		2227	tmp = rdev->config.r600.max_pipes * 16;
		2228	switch (rdev->family) {
		2229	case CHIP_RV610:
1268	serge	2230	case CHIP_RV620:
1221	serge	2231	case CHIP_RS780:
1268	serge	2232	case CHIP_RS880:
1221	serge	2233	tmp += 32;
		2234	break;
		2235	case CHIP_RV670:
		2236	tmp += 128;
		2237	break;
		2238	default:
		2239	break;
		2240	}
		2241	if (tmp > 256) {
		2242	tmp = 256;
		2243	}
		2244	WREG32(VGT_ES_PER_GS, 128);
		2245	WREG32(VGT_GS_PER_ES, tmp);
		2246	WREG32(VGT_GS_PER_VS, 2);
		2247	WREG32(VGT_GS_VERTEX_REUSE, 16);
		2248
		2249	/* more default values. 2D/3D driver should adjust as needed */
		2250	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		2251	WREG32(VGT_STRMOUT_EN, 0);
		2252	WREG32(SX_MISC, 0);
		2253	WREG32(PA_SC_MODE_CNTL, 0);
		2254	WREG32(PA_SC_AA_CONFIG, 0);
		2255	WREG32(PA_SC_LINE_STIPPLE, 0);
		2256	WREG32(SPI_INPUT_Z, 0);
		2257	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
		2258	WREG32(CB_COLOR7_FRAG, 0);
		2259
		2260	/* Clear render buffer base addresses */
		2261	WREG32(CB_COLOR0_BASE, 0);
		2262	WREG32(CB_COLOR1_BASE, 0);
		2263	WREG32(CB_COLOR2_BASE, 0);
		2264	WREG32(CB_COLOR3_BASE, 0);
		2265	WREG32(CB_COLOR4_BASE, 0);
		2266	WREG32(CB_COLOR5_BASE, 0);
		2267	WREG32(CB_COLOR6_BASE, 0);
		2268	WREG32(CB_COLOR7_BASE, 0);
		2269	WREG32(CB_COLOR7_FRAG, 0);
		2270
		2271	switch (rdev->family) {
		2272	case CHIP_RV610:
1268	serge	2273	case CHIP_RV620:
1221	serge	2274	case CHIP_RS780:
1268	serge	2275	case CHIP_RS880:
1221	serge	2276	tmp = TC_L2_SIZE(8);
		2277	break;
		2278	case CHIP_RV630:
		2279	case CHIP_RV635:
		2280	tmp = TC_L2_SIZE(4);
		2281	break;
		2282	case CHIP_R600:
		2283	tmp = TC_L2_SIZE(0) \| L2_DISABLE_LATE_HIT;
		2284	break;
		2285	default:
		2286	tmp = TC_L2_SIZE(0);
		2287	break;
		2288	}
		2289	WREG32(TC_CNTL, tmp);
		2290
		2291	tmp = RREG32(HDP_HOST_PATH_CNTL);
		2292	WREG32(HDP_HOST_PATH_CNTL, tmp);
		2293
		2294	tmp = RREG32(ARB_POP);
		2295	tmp \|= ENABLE_TC128;
		2296	WREG32(ARB_POP, tmp);
		2297
		2298	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		2299	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA \|
		2300	NUM_CLIP_SEQ(3)));
		2301	WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
2997	Serge	2302	WREG32(VC_ENHANCE, 0);
1221	serge	2303	}
		2304
		2305
1128	serge	2306	/*
		2307	* Indirect registers accessor
		2308	*/
1221	serge	2309	u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1128	serge	2310	{
5078	serge	2311	unsigned long flags;
1221	serge	2312	u32 r;
1128	serge	2313
5078	serge	2314	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
1221	serge	2315	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		2316	(void)RREG32(PCIE_PORT_INDEX);
		2317	r = RREG32(PCIE_PORT_DATA);
5078	serge	2318	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
1128	serge	2319	return r;
		2320	}
		2321
1221	serge	2322	void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1128	serge	2323	{
5078	serge	2324	unsigned long flags;
		2325
		2326	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
1221	serge	2327	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		2328	(void)RREG32(PCIE_PORT_INDEX);
		2329	WREG32(PCIE_PORT_DATA, (v));
		2330	(void)RREG32(PCIE_PORT_DATA);
5078	serge	2331	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
1128	serge	2332	}
1221	serge	2333
		2334	/*
		2335	* CP & Ring
		2336	*/
		2337	void r600_cp_stop(struct radeon_device *rdev)
		2338	{
5078	serge	2339	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3192	Serge	2340	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1221	serge	2341	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1963	serge	2342	WREG32(SCRATCH_UMSK, 0);
3192	Serge	2343	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1221	serge	2344	}
1413	serge	2345
		2346	int r600_init_microcode(struct radeon_device *rdev)
		2347	{
		2348	const char *chip_name;
		2349	const char *rlc_chip_name;
5078	serge	2350	const char *smc_chip_name = "RV770";
		2351	size_t pfp_req_size, me_req_size, rlc_req_size, smc_req_size = 0;
1413	serge	2352	char fw_name[30];
		2353	int err;
		2354
		2355	DRM_DEBUG("\n");
		2356
		2357	switch (rdev->family) {
		2358	case CHIP_R600:
		2359	chip_name = "R600";
		2360	rlc_chip_name = "R600";
		2361	break;
		2362	case CHIP_RV610:
		2363	chip_name = "RV610";
		2364	rlc_chip_name = "R600";
		2365	break;
		2366	case CHIP_RV630:
		2367	chip_name = "RV630";
		2368	rlc_chip_name = "R600";
		2369	break;
		2370	case CHIP_RV620:
		2371	chip_name = "RV620";
		2372	rlc_chip_name = "R600";
		2373	break;
		2374	case CHIP_RV635:
		2375	chip_name = "RV635";
		2376	rlc_chip_name = "R600";
		2377	break;
		2378	case CHIP_RV670:
		2379	chip_name = "RV670";
		2380	rlc_chip_name = "R600";
		2381	break;
		2382	case CHIP_RS780:
		2383	case CHIP_RS880:
		2384	chip_name = "RS780";
		2385	rlc_chip_name = "R600";
		2386	break;
		2387	case CHIP_RV770:
		2388	chip_name = "RV770";
		2389	rlc_chip_name = "R700";
5078	serge	2390	smc_chip_name = "RV770";
		2391	smc_req_size = ALIGN(RV770_SMC_UCODE_SIZE, 4);
1413	serge	2392	break;
		2393	case CHIP_RV730:
		2394	chip_name = "RV730";
		2395	rlc_chip_name = "R700";
5078	serge	2396	smc_chip_name = "RV730";
		2397	smc_req_size = ALIGN(RV730_SMC_UCODE_SIZE, 4);
1413	serge	2398	break;
		2399	case CHIP_RV710:
		2400	chip_name = "RV710";
		2401	rlc_chip_name = "R700";
5078	serge	2402	smc_chip_name = "RV710";
		2403	smc_req_size = ALIGN(RV710_SMC_UCODE_SIZE, 4);
1413	serge	2404	break;
5078	serge	2405	case CHIP_RV740:
		2406	chip_name = "RV730";
		2407	rlc_chip_name = "R700";
		2408	smc_chip_name = "RV740";
		2409	smc_req_size = ALIGN(RV740_SMC_UCODE_SIZE, 4);
		2410	break;
1963	serge	2411	case CHIP_CEDAR:
		2412	chip_name = "CEDAR";
		2413	rlc_chip_name = "CEDAR";
5078	serge	2414	smc_chip_name = "CEDAR";
		2415	smc_req_size = ALIGN(CEDAR_SMC_UCODE_SIZE, 4);
1963	serge	2416	break;
		2417	case CHIP_REDWOOD:
		2418	chip_name = "REDWOOD";
		2419	rlc_chip_name = "REDWOOD";
5078	serge	2420	smc_chip_name = "REDWOOD";
		2421	smc_req_size = ALIGN(REDWOOD_SMC_UCODE_SIZE, 4);
1963	serge	2422	break;
		2423	case CHIP_JUNIPER:
		2424	chip_name = "JUNIPER";
		2425	rlc_chip_name = "JUNIPER";
5078	serge	2426	smc_chip_name = "JUNIPER";
		2427	smc_req_size = ALIGN(JUNIPER_SMC_UCODE_SIZE, 4);
1963	serge	2428	break;
		2429	case CHIP_CYPRESS:
		2430	case CHIP_HEMLOCK:
		2431	chip_name = "CYPRESS";
		2432	rlc_chip_name = "CYPRESS";
5078	serge	2433	smc_chip_name = "CYPRESS";
		2434	smc_req_size = ALIGN(CYPRESS_SMC_UCODE_SIZE, 4);
1963	serge	2435	break;
		2436	case CHIP_PALM:
		2437	chip_name = "PALM";
		2438	rlc_chip_name = "SUMO";
		2439	break;
1986	serge	2440	case CHIP_SUMO:
		2441	chip_name = "SUMO";
		2442	rlc_chip_name = "SUMO";
		2443	break;
		2444	case CHIP_SUMO2:
		2445	chip_name = "SUMO2";
		2446	rlc_chip_name = "SUMO";
		2447	break;
1413	serge	2448	default: BUG();
		2449	}
		2450
1963	serge	2451	if (rdev->family >= CHIP_CEDAR) {
		2452	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		2453	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		2454	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		2455	} else if (rdev->family >= CHIP_RV770) {
1413	serge	2456	pfp_req_size = R700_PFP_UCODE_SIZE * 4;
		2457	me_req_size = R700_PM4_UCODE_SIZE * 4;
		2458	rlc_req_size = R700_RLC_UCODE_SIZE * 4;
		2459	} else {
5078	serge	2460	pfp_req_size = R600_PFP_UCODE_SIZE * 4;
		2461	me_req_size = R600_PM4_UCODE_SIZE * 12;
		2462	rlc_req_size = R600_RLC_UCODE_SIZE * 4;
1413	serge	2463	}
		2464
		2465	DRM_INFO("Loading %s Microcode\n", chip_name);
		2466
		2467	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
5078	serge	2468	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1413	serge	2469	if (err)
		2470	goto out;
		2471	if (rdev->pfp_fw->size != pfp_req_size) {
		2472	printk(KERN_ERR
		2473	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		2474	rdev->pfp_fw->size, fw_name);
		2475	err = -EINVAL;
		2476	goto out;
		2477	}
		2478
		2479	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
5078	serge	2480	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1413	serge	2481	if (err)
		2482	goto out;
		2483	if (rdev->me_fw->size != me_req_size) {
		2484	printk(KERN_ERR
		2485	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		2486	rdev->me_fw->size, fw_name);
		2487	err = -EINVAL;
		2488	}
		2489
		2490	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
5078	serge	2491	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1413	serge	2492	if (err)
		2493	goto out;
		2494	if (rdev->rlc_fw->size != rlc_req_size) {
		2495	printk(KERN_ERR
		2496	"r600_rlc: Bogus length %zu in firmware \"%s\"\n",
		2497	rdev->rlc_fw->size, fw_name);
		2498	err = -EINVAL;
		2499	}
		2500
5078	serge	2501	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_HEMLOCK)) {
		2502	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", smc_chip_name);
		2503	err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
		2504	if (err) {
		2505	printk(KERN_ERR
		2506	"smc: error loading firmware \"%s\"\n",
		2507	fw_name);
		2508	release_firmware(rdev->smc_fw);
		2509	rdev->smc_fw = NULL;
		2510	err = 0;
		2511	} else if (rdev->smc_fw->size != smc_req_size) {
		2512	printk(KERN_ERR
		2513	"smc: Bogus length %zu in firmware \"%s\"\n",
		2514	rdev->smc_fw->size, fw_name);
		2515	err = -EINVAL;
		2516	}
		2517	}
		2518
1413	serge	2519	out:
		2520	if (err) {
		2521	if (err != -EINVAL)
		2522	printk(KERN_ERR
		2523	"r600_cp: Failed to load firmware \"%s\"\n",
		2524	fw_name);
		2525	release_firmware(rdev->pfp_fw);
		2526	rdev->pfp_fw = NULL;
		2527	release_firmware(rdev->me_fw);
		2528	rdev->me_fw = NULL;
		2529	release_firmware(rdev->rlc_fw);
		2530	rdev->rlc_fw = NULL;
5078	serge	2531	release_firmware(rdev->smc_fw);
		2532	rdev->smc_fw = NULL;
1413	serge	2533	}
		2534	return err;
		2535	}
		2536
5078	serge	2537	u32 r600_gfx_get_rptr(struct radeon_device *rdev,
		2538	struct radeon_ring *ring)
		2539	{
		2540	u32 rptr;
		2541
		2542	if (rdev->wb.enabled)
		2543	rptr = rdev->wb.wb[ring->rptr_offs/4];
		2544	else
		2545	rptr = RREG32(R600_CP_RB_RPTR);
		2546
		2547	return rptr;
		2548	}
		2549
		2550	u32 r600_gfx_get_wptr(struct radeon_device *rdev,
		2551	struct radeon_ring *ring)
		2552	{
		2553	u32 wptr;
		2554
		2555	wptr = RREG32(R600_CP_RB_WPTR);
		2556
		2557	return wptr;
		2558	}
		2559
		2560	void r600_gfx_set_wptr(struct radeon_device *rdev,
		2561	struct radeon_ring *ring)
		2562	{
		2563	WREG32(R600_CP_RB_WPTR, ring->wptr);
		2564	(void)RREG32(R600_CP_RB_WPTR);
		2565	}
		2566
1413	serge	2567	static int r600_cp_load_microcode(struct radeon_device *rdev)
		2568	{
		2569	const __be32 *fw_data;
		2570	int i;
		2571
		2572	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		2573	return -EINVAL;
		2574
		2575	r600_cp_stop(rdev);
		2576
1963	serge	2577	WREG32(CP_RB_CNTL,
		2578	#ifdef __BIG_ENDIAN
		2579	BUF_SWAP_32BIT \|
		2580	#endif
		2581	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
1413	serge	2582
		2583	/* Reset cp */
		2584	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		2585	RREG32(GRBM_SOFT_RESET);
		2586	mdelay(15);
		2587	WREG32(GRBM_SOFT_RESET, 0);
		2588
		2589	WREG32(CP_ME_RAM_WADDR, 0);
		2590
		2591	fw_data = (const __be32 *)rdev->me_fw->data;
		2592	WREG32(CP_ME_RAM_WADDR, 0);
5078	serge	2593	for (i = 0; i < R600_PM4_UCODE_SIZE * 3; i++)
1413	serge	2594	WREG32(CP_ME_RAM_DATA,
		2595	be32_to_cpup(fw_data++));
		2596
		2597	fw_data = (const __be32 *)rdev->pfp_fw->data;
		2598	WREG32(CP_PFP_UCODE_ADDR, 0);
5078	serge	2599	for (i = 0; i < R600_PFP_UCODE_SIZE; i++)
1413	serge	2600	WREG32(CP_PFP_UCODE_DATA,
		2601	be32_to_cpup(fw_data++));
		2602
		2603	WREG32(CP_PFP_UCODE_ADDR, 0);
		2604	WREG32(CP_ME_RAM_WADDR, 0);
		2605	WREG32(CP_ME_RAM_RADDR, 0);
		2606	return 0;
		2607	}
		2608
1221	serge	2609	int r600_cp_start(struct radeon_device *rdev)
		2610	{
2997	Serge	2611	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1221	serge	2612	int r;
		2613	uint32_t cp_me;
		2614
2997	Serge	2615	r = radeon_ring_lock(rdev, ring, 7);
1221	serge	2616	if (r) {
		2617	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		2618	return r;
		2619	}
2997	Serge	2620	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		2621	radeon_ring_write(ring, 0x1);
1963	serge	2622	if (rdev->family >= CHIP_RV770) {
2997	Serge	2623	radeon_ring_write(ring, 0x0);
		2624	radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
1963	serge	2625	} else {
2997	Serge	2626	radeon_ring_write(ring, 0x3);
		2627	radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
1221	serge	2628	}
2997	Serge	2629	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		2630	radeon_ring_write(ring, 0);
		2631	radeon_ring_write(ring, 0);
5078	serge	2632	radeon_ring_unlock_commit(rdev, ring, false);
1221	serge	2633
		2634	cp_me = 0xff;
		2635	WREG32(R_0086D8_CP_ME_CNTL, cp_me);
		2636	return 0;
		2637	}
1413	serge	2638
		2639	int r600_cp_resume(struct radeon_device *rdev)
		2640	{
2997	Serge	2641	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1413	serge	2642	u32 tmp;
		2643	u32 rb_bufsz;
		2644	int r;
		2645
		2646	/* Reset cp */
		2647	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		2648	RREG32(GRBM_SOFT_RESET);
		2649	mdelay(15);
		2650	WREG32(GRBM_SOFT_RESET, 0);
		2651
		2652	/* Set ring buffer size */
5078	serge	2653	rb_bufsz = order_base_2(ring->ring_size / 8);
		2654	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
1413	serge	2655	#ifdef __BIG_ENDIAN
		2656	tmp \|= BUF_SWAP_32BIT;
		2657	#endif
		2658	WREG32(CP_RB_CNTL, tmp);
2997	Serge	2659	WREG32(CP_SEM_WAIT_TIMER, 0x0);
1413	serge	2660
		2661	/* Set the write pointer delay */
		2662	WREG32(CP_RB_WPTR_DELAY, 0);
		2663
		2664	/* Initialize the ring buffer's read and write pointers */
		2665	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		2666	WREG32(CP_RB_RPTR_WR, 0);
2997	Serge	2667	ring->wptr = 0;
		2668	WREG32(CP_RB_WPTR, ring->wptr);
1963	serge	2669
		2670	/* set the wb address whether it's enabled or not */
		2671	WREG32(CP_RB_RPTR_ADDR,
		2672	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		2673	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		2674	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		2675
		2676	if (rdev->wb.enabled)
		2677	WREG32(SCRATCH_UMSK, 0xff);
		2678	else {
		2679	tmp \|= RB_NO_UPDATE;
		2680	WREG32(SCRATCH_UMSK, 0);
		2681	}
		2682
1413	serge	2683	mdelay(1);
		2684	WREG32(CP_RB_CNTL, tmp);
		2685
2997	Serge	2686	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1413	serge	2687	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		2688
		2689	r600_cp_start(rdev);
2997	Serge	2690	ring->ready = true;
		2691	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
1413	serge	2692	if (r) {
2997	Serge	2693	ring->ready = false;
1413	serge	2694	return r;
		2695	}
5078	serge	2696
		2697	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
		2698	radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
		2699
1413	serge	2700	return 0;
		2701	}
		2702
2997	Serge	2703	void r600_ring_init(struct radeon_device rdev, struct radeon_ring ring, unsigned ring_size)
1221	serge	2704	{
1233	serge	2705	u32 rb_bufsz;
2997	Serge	2706	int r;
1221	serge	2707
1233	serge	2708	/* Align ring size */
5078	serge	2709	rb_bufsz = order_base_2(ring_size / 8);
1233	serge	2710	ring_size = (1 << (rb_bufsz + 1)) * 4;
2997	Serge	2711	ring->ring_size = ring_size;
		2712	ring->align_mask = 16 - 1;
		2713
		2714	if (radeon_ring_supports_scratch_reg(rdev, ring)) {
		2715	r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
		2716	if (r) {
		2717	DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
		2718	ring->rptr_save_reg = 0;
		2719	}
		2720	}
1233	serge	2721	}
		2722
1963	serge	2723	void r600_cp_fini(struct radeon_device *rdev)
		2724	{
2997	Serge	2725	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	2726	r600_cp_stop(rdev);
2997	Serge	2727	radeon_ring_fini(rdev, ring);
		2728	radeon_scratch_free(rdev, ring->rptr_save_reg);
1963	serge	2729	}
1233	serge	2730
3192	Serge	2731	/*
1233	serge	2732	* GPU scratch registers helpers function.
		2733	*/
		2734	void r600_scratch_init(struct radeon_device *rdev)
		2735	{
		2736	int i;
		2737
		2738	rdev->scratch.num_reg = 7;
1963	serge	2739	rdev->scratch.reg_base = SCRATCH_REG0;
1233	serge	2740	for (i = 0; i < rdev->scratch.num_reg; i++) {
		2741	rdev->scratch.free[i] = true;
1963	serge	2742	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
1233	serge	2743	}
		2744	}
1413	serge	2745
2997	Serge	2746	int r600_ring_test(struct radeon_device rdev, struct radeon_ring ring)
1413	serge	2747	{
		2748	uint32_t scratch;
		2749	uint32_t tmp = 0;
		2750	unsigned i;
		2751	int r;
		2752
		2753	r = radeon_scratch_get(rdev, &scratch);
		2754	if (r) {
		2755	DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
		2756	return r;
		2757	}
		2758	WREG32(scratch, 0xCAFEDEAD);
2997	Serge	2759	r = radeon_ring_lock(rdev, ring, 3);
1413	serge	2760	if (r) {
2997	Serge	2761	DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
1413	serge	2762	radeon_scratch_free(rdev, scratch);
		2763	return r;
		2764	}
2997	Serge	2765	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2766	radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		2767	radeon_ring_write(ring, 0xDEADBEEF);
5078	serge	2768	radeon_ring_unlock_commit(rdev, ring, false);
1413	serge	2769	for (i = 0; i < rdev->usec_timeout; i++) {
		2770	tmp = RREG32(scratch);
		2771	if (tmp == 0xDEADBEEF)
		2772	break;
		2773	DRM_UDELAY(1);
		2774	}
		2775	if (i < rdev->usec_timeout) {
2997	Serge	2776	DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
1413	serge	2777	} else {
2997	Serge	2778	DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
		2779	ring->idx, scratch, tmp);
1413	serge	2780	r = -EINVAL;
		2781	}
		2782	radeon_scratch_free(rdev, scratch);
		2783	return r;
		2784	}
1963	serge	2785
3192	Serge	2786	/*
		2787	* CP fences/semaphores
		2788	*/
		2789
1413	serge	2790	void r600_fence_ring_emit(struct radeon_device *rdev,
		2791	struct radeon_fence *fence)
		2792	{
2997	Serge	2793	struct radeon_ring *ring = &rdev->ring[fence->ring];
5078	serge	2794	u32 cp_coher_cntl = PACKET3_TC_ACTION_ENA \| PACKET3_VC_ACTION_ENA \|
		2795	PACKET3_SH_ACTION_ENA;
2997	Serge	2796
5078	serge	2797	if (rdev->family >= CHIP_RV770)
		2798	cp_coher_cntl \|= PACKET3_FULL_CACHE_ENA;
		2799
1963	serge	2800	if (rdev->wb.use_event) {
2997	Serge	2801	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		2802	/* flush read cache over gart */
		2803	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
5078	serge	2804	radeon_ring_write(ring, cp_coher_cntl);
2997	Serge	2805	radeon_ring_write(ring, 0xFFFFFFFF);
		2806	radeon_ring_write(ring, 0);
		2807	radeon_ring_write(ring, 10); /* poll interval */
1963	serge	2808	/* EVENT_WRITE_EOP - flush caches, send int */
2997	Serge	2809	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		2810	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) \| EVENT_INDEX(5));
5078	serge	2811	radeon_ring_write(ring, lower_32_bits(addr));
2997	Serge	2812	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		2813	radeon_ring_write(ring, fence->seq);
		2814	radeon_ring_write(ring, 0);
1963	serge	2815	} else {
2997	Serge	2816	/* flush read cache over gart */
		2817	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
5078	serge	2818	radeon_ring_write(ring, cp_coher_cntl);
2997	Serge	2819	radeon_ring_write(ring, 0xFFFFFFFF);
		2820	radeon_ring_write(ring, 0);
		2821	radeon_ring_write(ring, 10); /* poll interval */
		2822	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
		2823	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) \| EVENT_INDEX(0));
1430	serge	2824	/* wait for 3D idle clean */
2997	Serge	2825	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2826	radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		2827	radeon_ring_write(ring, WAIT_3D_IDLE_bit \| WAIT_3D_IDLECLEAN_bit);
1413	serge	2828	/* Emit fence sequence & fire IRQ */
2997	Serge	2829	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2830	radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		2831	radeon_ring_write(ring, fence->seq);
1413	serge	2832	/* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2997	Serge	2833	radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
		2834	radeon_ring_write(ring, RB_INT_STAT);
1963	serge	2835	}
1413	serge	2836	}
1963	serge	2837
5078	serge	2838	/**
		2839	* r600_semaphore_ring_emit - emit a semaphore on the CP ring
		2840	*
		2841	* @rdev: radeon_device pointer
		2842	* @ring: radeon ring buffer object
		2843	* @semaphore: radeon semaphore object
		2844	* @emit_wait: Is this a sempahore wait?
		2845	*
		2846	* Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
		2847	* from running ahead of semaphore waits.
		2848	*/
		2849	bool r600_semaphore_ring_emit(struct radeon_device *rdev,
2997	Serge	2850	struct radeon_ring *ring,
		2851	struct radeon_semaphore *semaphore,
		2852	bool emit_wait)
		2853	{
		2854	uint64_t addr = semaphore->gpu_addr;
		2855	unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
		2856
		2857	if (rdev->family < CHIP_CAYMAN)
		2858	sel \|= PACKET3_SEM_WAIT_ON_SIGNAL;
		2859
		2860	radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5078	serge	2861	radeon_ring_write(ring, lower_32_bits(addr));
2997	Serge	2862	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| sel);
		2863
5128	serge	2864	/* PFP_SYNC_ME packet only exists on 7xx+, only enable it on eg+ */
		2865	if (emit_wait && (rdev->family >= CHIP_CEDAR)) {
5078	serge	2866	/* Prevent the PFP from running ahead of the semaphore wait */
		2867	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
		2868	radeon_ring_write(ring, 0x0);
		2869	}
3192	Serge	2870
5078	serge	2871	return true;
3192	Serge	2872	}
		2873
		2874	/**
5078	serge	2875	* r600_copy_cpdma - copy pages using the CP DMA engine
3192	Serge	2876	*
		2877	* @rdev: radeon_device pointer
		2878	* @src_offset: src GPU address
		2879	* @dst_offset: dst GPU address
		2880	* @num_gpu_pages: number of GPU pages to xfer
		2881	* @fence: radeon fence object
		2882	*
5078	serge	2883	* Copy GPU paging using the CP DMA engine (r6xx+).
3192	Serge	2884	* Used by the radeon ttm implementation to move pages if
		2885	* registered as the asic copy callback.
		2886	*/
5271	serge	2887	struct radeon_fence r600_copy_cpdma(struct radeon_device rdev,
3192	Serge	2888	uint64_t src_offset, uint64_t dst_offset,
		2889	unsigned num_gpu_pages,
5271	serge	2890	struct reservation_object *resv)
3192	Serge	2891	{
5271	serge	2892	struct radeon_fence *fence;
		2893	struct radeon_sync sync;
5078	serge	2894	int ring_index = rdev->asic->copy.blit_ring_index;
3192	Serge	2895	struct radeon_ring *ring = &rdev->ring[ring_index];
5078	serge	2896	u32 size_in_bytes, cur_size_in_bytes, tmp;
3192	Serge	2897	int i, num_loops;
		2898	int r = 0;
		2899
5271	serge	2900	radeon_sync_create(&sync);
3192	Serge	2901
5078	serge	2902	size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
		2903	num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
		2904	r = radeon_ring_lock(rdev, ring, num_loops * 6 + 24);
3192	Serge	2905	if (r) {
		2906	DRM_ERROR("radeon: moving bo (%d).\n", r);
5271	serge	2907	radeon_sync_free(rdev, &sync, NULL);
		2908	return ERR_PTR(r);
3192	Serge	2909	}
		2910
5271	serge	2911	radeon_sync_resv(rdev, &sync, resv, false);
		2912	radeon_sync_rings(rdev, &sync, ring->idx);
3192	Serge	2913
5078	serge	2914	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2915	radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		2916	radeon_ring_write(ring, WAIT_3D_IDLE_bit);
3192	Serge	2917	for (i = 0; i < num_loops; i++) {
5078	serge	2918	cur_size_in_bytes = size_in_bytes;
		2919	if (cur_size_in_bytes > 0x1fffff)
		2920	cur_size_in_bytes = 0x1fffff;
		2921	size_in_bytes -= cur_size_in_bytes;
		2922	tmp = upper_32_bits(src_offset) & 0xff;
		2923	if (size_in_bytes == 0)
		2924	tmp \|= PACKET3_CP_DMA_CP_SYNC;
		2925	radeon_ring_write(ring, PACKET3(PACKET3_CP_DMA, 4));
		2926	radeon_ring_write(ring, lower_32_bits(src_offset));
		2927	radeon_ring_write(ring, tmp);
		2928	radeon_ring_write(ring, lower_32_bits(dst_offset));
		2929	radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
		2930	radeon_ring_write(ring, cur_size_in_bytes);
		2931	src_offset += cur_size_in_bytes;
		2932	dst_offset += cur_size_in_bytes;
3192	Serge	2933	}
5078	serge	2934	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2935	radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		2936	radeon_ring_write(ring, WAIT_CP_DMA_IDLE_bit);
3192	Serge	2937
5271	serge	2938	r = radeon_fence_emit(rdev, &fence, ring->idx);
3192	Serge	2939	if (r) {
		2940	radeon_ring_unlock_undo(rdev, ring);
5271	serge	2941	radeon_sync_free(rdev, &sync, NULL);
		2942	return ERR_PTR(r);
3192	Serge	2943	}
		2944
5078	serge	2945	radeon_ring_unlock_commit(rdev, ring, false);
5271	serge	2946	radeon_sync_free(rdev, &sync, fence);
3192	Serge	2947
5271	serge	2948	return fence;
3192	Serge	2949	}
		2950
1221	serge	2951	int r600_set_surface_reg(struct radeon_device *rdev, int reg,
		2952	uint32_t tiling_flags, uint32_t pitch,
		2953	uint32_t offset, uint32_t obj_size)
		2954	{
		2955	/* FIXME: implement */
		2956	return 0;
		2957	}
		2958
		2959	void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
		2960	{
		2961	/* FIXME: implement */
		2962	}
		2963
2997	Serge	2964	static int r600_startup(struct radeon_device *rdev)
1221	serge	2965	{
3192	Serge	2966	struct radeon_ring *ring;
1221	serge	2967	int r;
		2968
1963	serge	2969	/* enable pcie gen2 link */
		2970	r600_pcie_gen2_enable(rdev);
		2971
5078	serge	2972	/* scratch needs to be initialized before MC */
3764	Serge	2973	r = r600_vram_scratch_init(rdev);
		2974	if (r)
		2975	return r;
		2976
1221	serge	2977	r600_mc_program(rdev);
5078	serge	2978
1221	serge	2979	if (rdev->flags & RADEON_IS_AGP) {
		2980	r600_agp_enable(rdev);
		2981	} else {
		2982	r = r600_pcie_gart_enable(rdev);
		2983	if (r)
		2984	return r;
		2985	}
		2986	r600_gpu_init(rdev);
		2987
2005	serge	2988	/* allocate wb buffer */
		2989	r = radeon_wb_init(rdev);
		2990	if (r)
		2991	return r;
		2992
3192	Serge	2993	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
		2994	if (r) {
		2995	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		2996	return r;
		2997	}
		2998
2005	serge	2999	/* Enable IRQ */
3764	Serge	3000	if (!rdev->irq.installed) {
		3001	r = radeon_irq_kms_init(rdev);
		3002	if (r)
		3003	return r;
		3004	}
		3005
2005	serge	3006	r = r600_irq_init(rdev);
		3007	if (r) {
		3008	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		3009	// radeon_irq_kms_fini(rdev);
		3010	return r;
		3011	}
		3012	r600_irq_set(rdev);
		3013
3192	Serge	3014	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2997	Serge	3015	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5078	serge	3016	RADEON_CP_PACKET2);
3192	Serge	3017	if (r)
		3018	return r;
2997	Serge	3019
1413	serge	3020	r = r600_cp_load_microcode(rdev);
		3021	if (r)
		3022	return r;
		3023	r = r600_cp_resume(rdev);
		3024	if (r)
		3025	return r;
1963	serge	3026
3192	Serge	3027	r = radeon_ib_pool_init(rdev);
		3028	if (r) {
		3029	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		3030	return r;
		3031	}
5078	serge	3032
		3033
1221	serge	3034	return 0;
		3035	}
		3036
		3037	void r600_vga_set_state(struct radeon_device *rdev, bool state)
		3038	{
		3039	uint32_t temp;
		3040
		3041	temp = RREG32(CONFIG_CNTL);
		3042	if (state == false) {
		3043	temp &= ~(1<<0);
		3044	temp \|= (1<<1);
		3045	} else {
		3046	temp &= ~(1<<1);
		3047	}
		3048	WREG32(CONFIG_CNTL, temp);
		3049	}
		3050
		3051
		3052
		3053
		3054
		3055	/* Plan is to move initialization in that function and use
		3056	* helper function so that radeon_device_init pretty much
		3057	* do nothing more than calling asic specific function. This
		3058	* should also allow to remove a bunch of callback function
		3059	* like vram_info.
		3060	*/
		3061	int r600_init(struct radeon_device *rdev)
		3062	{
		3063	int r;
		3064
		3065	if (r600_debugfs_mc_info_init(rdev)) {
		3066	DRM_ERROR("Failed to register debugfs file for mc !\n");
		3067	}
		3068	/* Read BIOS */
		3069	if (!radeon_get_bios(rdev)) {
		3070	if (ASIC_IS_AVIVO(rdev))
		3071	return -EINVAL;
		3072	}
		3073	/* Must be an ATOMBIOS */
		3074	if (!rdev->is_atom_bios) {
		3075	dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
		3076	return -EINVAL;
		3077	}
		3078	r = radeon_atombios_init(rdev);
		3079	if (r)
		3080	return r;
		3081	/* Post card if necessary */
1963	serge	3082	if (!radeon_card_posted(rdev)) {
1321	serge	3083	if (!rdev->bios) {
		3084	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		3085	return -EINVAL;
		3086	}
1221	serge	3087	DRM_INFO("GPU not posted. posting now...\n");
		3088	atom_asic_init(rdev->mode_info.atom_context);
		3089	}
		3090	/* Initialize scratch registers */
		3091	r600_scratch_init(rdev);
		3092	/* Initialize surface registers */
		3093	radeon_surface_init(rdev);
1268	serge	3094	/* Initialize clocks */
1221	serge	3095	radeon_get_clock_info(rdev->ddev);
		3096	/* Fence driver */
2004	serge	3097	r = radeon_fence_driver_init(rdev);
		3098	if (r)
		3099	return r;
1403	serge	3100	if (rdev->flags & RADEON_IS_AGP) {
		3101	r = radeon_agp_init(rdev);
		3102	if (r)
		3103	radeon_agp_disable(rdev);
		3104	}
1221	serge	3105	r = r600_mc_init(rdev);
		3106	if (r)
		3107	return r;
		3108	/* Memory manager */
1321	serge	3109	r = radeon_bo_init(rdev);
1221	serge	3110	if (r)
		3111	return r;
1321	serge	3112
5078	serge	3113	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		3114	r = r600_init_microcode(rdev);
		3115	if (r) {
		3116	DRM_ERROR("Failed to load firmware!\n");
		3117	return r;
		3118	}
		3119	}
		3120
		3121	/* Initialize power management */
		3122	radeon_pm_init(rdev);
		3123
2997	Serge	3124	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
		3125	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1221	serge	3126
2004	serge	3127	rdev->ih.ring_obj = NULL;
		3128	r600_ih_ring_init(rdev, 64 * 1024);
1221	serge	3129
		3130	r = r600_pcie_gart_init(rdev);
		3131	if (r)
		3132	return r;
		3133
1321	serge	3134	rdev->accel_working = true;
1221	serge	3135	r = r600_startup(rdev);
		3136	if (r) {
1428	serge	3137	dev_err(rdev->dev, "disabling GPU acceleration\n");
1221	serge	3138	r600_pcie_gart_fini(rdev);
		3139	rdev->accel_working = false;
		3140	}
2005	serge	3141
1221	serge	3142	return 0;
		3143	}
		3144
2004	serge	3145	/*
		3146	* CS stuff
		3147	*/
		3148	void r600_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		3149	{
2997	Serge	3150	struct radeon_ring *ring = &rdev->ring[ib->ring];
		3151	u32 next_rptr;
		3152
		3153	if (ring->rptr_save_reg) {
		3154	next_rptr = ring->wptr + 3 + 4;
		3155	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		3156	radeon_ring_write(ring, ((ring->rptr_save_reg -
		3157	PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		3158	radeon_ring_write(ring, next_rptr);
		3159	} else if (rdev->wb.enabled) {
		3160	next_rptr = ring->wptr + 5 + 4;
		3161	radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
		3162	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		3163	radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) \| (1 << 18));
		3164	radeon_ring_write(ring, next_rptr);
		3165	radeon_ring_write(ring, 0);
		3166	}
		3167
		3168	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		3169	radeon_ring_write(ring,
2004	serge	3170	#ifdef __BIG_ENDIAN
		3171	(2 << 0) \|
		3172	#endif
		3173	(ib->gpu_addr & 0xFFFFFFFC));
2997	Serge	3174	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
		3175	radeon_ring_write(ring, ib->length_dw);
2004	serge	3176	}
		3177
2997	Serge	3178	int r600_ib_test(struct radeon_device rdev, struct radeon_ring ring)
2004	serge	3179	{
2997	Serge	3180	struct radeon_ib ib;
2004	serge	3181	uint32_t scratch;
		3182	uint32_t tmp = 0;
		3183	unsigned i;
		3184	int r;
		3185
		3186	r = radeon_scratch_get(rdev, &scratch);
		3187	if (r) {
		3188	DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
		3189	return r;
		3190	}
		3191	WREG32(scratch, 0xCAFEDEAD);
2997	Serge	3192	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
2004	serge	3193	if (r) {
		3194	DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2997	Serge	3195	goto free_scratch;
2004	serge	3196	}
2997	Serge	3197	ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
		3198	ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		3199	ib.ptr[2] = 0xDEADBEEF;
		3200	ib.length_dw = 3;
5078	serge	3201	r = radeon_ib_schedule(rdev, &ib, NULL, false);
2004	serge	3202	if (r) {
		3203	DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2997	Serge	3204	goto free_ib;
2004	serge	3205	}
2997	Serge	3206	r = radeon_fence_wait(ib.fence, false);
2004	serge	3207	if (r) {
		3208	DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2997	Serge	3209	goto free_ib;
2004	serge	3210	}
		3211	for (i = 0; i < rdev->usec_timeout; i++) {
		3212	tmp = RREG32(scratch);
		3213	if (tmp == 0xDEADBEEF)
		3214	break;
		3215	DRM_UDELAY(1);
		3216	}
		3217	if (i < rdev->usec_timeout) {
2997	Serge	3218	DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
2004	serge	3219	} else {
		3220	DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
		3221	scratch, tmp);
		3222	r = -EINVAL;
		3223	}
2997	Serge	3224	free_ib:
		3225	radeon_ib_free(rdev, &ib);
		3226	free_scratch:
2004	serge	3227	radeon_scratch_free(rdev, scratch);
		3228	return r;
		3229	}
		3230
		3231	/*
		3232	* Interrupts
		3233	*
		3234	* Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
		3235	* the same as the CP ring buffer, but in reverse. Rather than the CPU
		3236	* writing to the ring and the GPU consuming, the GPU writes to the ring
		3237	* and host consumes. As the host irq handler processes interrupts, it
		3238	* increments the rptr. When the rptr catches up with the wptr, all the
		3239	* current interrupts have been processed.
		3240	*/
		3241
		3242	void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
		3243	{
		3244	u32 rb_bufsz;
		3245
		3246	/* Align ring size */
5078	serge	3247	rb_bufsz = order_base_2(ring_size / 4);
2004	serge	3248	ring_size = (1 << rb_bufsz) * 4;
		3249	rdev->ih.ring_size = ring_size;
		3250	rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
		3251	rdev->ih.rptr = 0;
		3252	}
		3253
2997	Serge	3254	int r600_ih_ring_alloc(struct radeon_device *rdev)
2004	serge	3255	{
		3256	int r;
		3257
		3258	/* Allocate ring buffer */
		3259	if (rdev->ih.ring_obj == NULL) {
		3260	r = radeon_bo_create(rdev, rdev->ih.ring_size,
		3261	PAGE_SIZE, true,
5078	serge	3262	RADEON_GEM_DOMAIN_GTT, 0,
5271	serge	3263	NULL, NULL, &rdev->ih.ring_obj);
2004	serge	3264	if (r) {
		3265	DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
		3266	return r;
		3267	}
		3268	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		3269	if (unlikely(r != 0))
		3270	return r;
		3271	r = radeon_bo_pin(rdev->ih.ring_obj,
		3272	RADEON_GEM_DOMAIN_GTT,
		3273	&rdev->ih.gpu_addr);
		3274	if (r) {
		3275	radeon_bo_unreserve(rdev->ih.ring_obj);
		3276	DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
		3277	return r;
		3278	}
		3279	r = radeon_bo_kmap(rdev->ih.ring_obj,
		3280	(void **)&rdev->ih.ring);
		3281	radeon_bo_unreserve(rdev->ih.ring_obj);
		3282	if (r) {
		3283	DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
		3284	return r;
		3285	}
		3286	}
		3287	return 0;
		3288	}
		3289
2997	Serge	3290	void r600_ih_ring_fini(struct radeon_device *rdev)
2004	serge	3291	{
		3292	int r;
		3293	if (rdev->ih.ring_obj) {
		3294	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		3295	if (likely(r == 0)) {
		3296	radeon_bo_kunmap(rdev->ih.ring_obj);
		3297	radeon_bo_unpin(rdev->ih.ring_obj);
		3298	radeon_bo_unreserve(rdev->ih.ring_obj);
		3299	}
		3300	radeon_bo_unref(&rdev->ih.ring_obj);
		3301	rdev->ih.ring = NULL;
		3302	rdev->ih.ring_obj = NULL;
		3303	}
		3304	}
		3305
		3306	void r600_rlc_stop(struct radeon_device *rdev)
		3307	{
		3308
		3309	if ((rdev->family >= CHIP_RV770) &&
		3310	(rdev->family <= CHIP_RV740)) {
		3311	/* r7xx asics need to soft reset RLC before halting */
		3312	WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
		3313	RREG32(SRBM_SOFT_RESET);
2997	Serge	3314	mdelay(15);
2004	serge	3315	WREG32(SRBM_SOFT_RESET, 0);
		3316	RREG32(SRBM_SOFT_RESET);
		3317	}
		3318
		3319	WREG32(RLC_CNTL, 0);
		3320	}
		3321
		3322	static void r600_rlc_start(struct radeon_device *rdev)
		3323	{
		3324	WREG32(RLC_CNTL, RLC_ENABLE);
		3325	}
		3326
5078	serge	3327	static int r600_rlc_resume(struct radeon_device *rdev)
2004	serge	3328	{
		3329	u32 i;
		3330	const __be32 *fw_data;
		3331
		3332	if (!rdev->rlc_fw)
		3333	return -EINVAL;
		3334
		3335	r600_rlc_stop(rdev);
		3336
2997	Serge	3337	WREG32(RLC_HB_CNTL, 0);
		3338
2004	serge	3339	WREG32(RLC_HB_BASE, 0);
		3340	WREG32(RLC_HB_RPTR, 0);
		3341	WREG32(RLC_HB_WPTR, 0);
		3342	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
		3343	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
		3344	WREG32(RLC_MC_CNTL, 0);
		3345	WREG32(RLC_UCODE_CNTL, 0);
		3346
		3347	fw_data = (const __be32 *)rdev->rlc_fw->data;
5078	serge	3348	if (rdev->family >= CHIP_RV770) {
2004	serge	3349	for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
		3350	WREG32(RLC_UCODE_ADDR, i);
		3351	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		3352	}
		3353	} else {
5078	serge	3354	for (i = 0; i < R600_RLC_UCODE_SIZE; i++) {
2004	serge	3355	WREG32(RLC_UCODE_ADDR, i);
		3356	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		3357	}
		3358	}
		3359	WREG32(RLC_UCODE_ADDR, 0);
		3360
		3361	r600_rlc_start(rdev);
		3362
		3363	return 0;
		3364	}
		3365
		3366	static void r600_enable_interrupts(struct radeon_device *rdev)
		3367	{
		3368	u32 ih_cntl = RREG32(IH_CNTL);
		3369	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		3370
		3371	ih_cntl \|= ENABLE_INTR;
		3372	ih_rb_cntl \|= IH_RB_ENABLE;
		3373	WREG32(IH_CNTL, ih_cntl);
		3374	WREG32(IH_RB_CNTL, ih_rb_cntl);
		3375	rdev->ih.enabled = true;
		3376	}
		3377
		3378	void r600_disable_interrupts(struct radeon_device *rdev)
		3379	{
		3380	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		3381	u32 ih_cntl = RREG32(IH_CNTL);
		3382
		3383	ih_rb_cntl &= ~IH_RB_ENABLE;
		3384	ih_cntl &= ~ENABLE_INTR;
		3385	WREG32(IH_RB_CNTL, ih_rb_cntl);
		3386	WREG32(IH_CNTL, ih_cntl);
		3387	/* set rptr, wptr to 0 */
		3388	WREG32(IH_RB_RPTR, 0);
		3389	WREG32(IH_RB_WPTR, 0);
		3390	rdev->ih.enabled = false;
		3391	rdev->ih.rptr = 0;
		3392	}
		3393
1963	serge	3394	static void r600_disable_interrupt_state(struct radeon_device *rdev)
		3395	{
		3396	u32 tmp;
1221	serge	3397
1963	serge	3398	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
3192	Serge	3399	tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
		3400	WREG32(DMA_CNTL, tmp);
1963	serge	3401	WREG32(GRBM_INT_CNTL, 0);
		3402	WREG32(DxMODE_INT_MASK, 0);
		3403	WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
		3404	WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
		3405	if (ASIC_IS_DCE3(rdev)) {
		3406	WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
		3407	WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
		3408	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3409	WREG32(DC_HPD1_INT_CONTROL, tmp);
		3410	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3411	WREG32(DC_HPD2_INT_CONTROL, tmp);
		3412	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3413	WREG32(DC_HPD3_INT_CONTROL, tmp);
		3414	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3415	WREG32(DC_HPD4_INT_CONTROL, tmp);
		3416	if (ASIC_IS_DCE32(rdev)) {
		3417	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3418	WREG32(DC_HPD5_INT_CONTROL, tmp);
		3419	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3420	WREG32(DC_HPD6_INT_CONTROL, tmp);
2997	Serge	3421	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3422	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
		3423	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3424	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
		3425	} else {
		3426	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3427	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
		3428	tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3429	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
1963	serge	3430	}
		3431	} else {
		3432	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		3433	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		3434	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		3435	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		3436	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		3437	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		3438	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		3439	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2997	Serge	3440	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3441	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
		3442	tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3443	WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
1963	serge	3444	}
		3445	}
1221	serge	3446
2004	serge	3447	int r600_irq_init(struct radeon_device *rdev)
		3448	{
		3449	int ret = 0;
		3450	int rb_bufsz;
		3451	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
1221	serge	3452
2004	serge	3453	/* allocate ring */
		3454	ret = r600_ih_ring_alloc(rdev);
		3455	if (ret)
		3456	return ret;
1221	serge	3457
2004	serge	3458	/* disable irqs */
		3459	r600_disable_interrupts(rdev);
1221	serge	3460
2004	serge	3461	/* init rlc */
5078	serge	3462	if (rdev->family >= CHIP_CEDAR)
		3463	ret = evergreen_rlc_resume(rdev);
		3464	else
		3465	ret = r600_rlc_resume(rdev);
2004	serge	3466	if (ret) {
		3467	r600_ih_ring_fini(rdev);
		3468	return ret;
		3469	}
1221	serge	3470
2004	serge	3471	/* setup interrupt control */
		3472	/* set dummy read address to ring address */
		3473	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
		3474	interrupt_cntl = RREG32(INTERRUPT_CNTL);
		3475	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
		3476	* IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
		3477	*/
		3478	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
		3479	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
		3480	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
		3481	WREG32(INTERRUPT_CNTL, interrupt_cntl);
1221	serge	3482
2004	serge	3483	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5078	serge	3484	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
1221	serge	3485
2004	serge	3486	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE \|
		3487	IH_WPTR_OVERFLOW_CLEAR \|
		3488	(rb_bufsz << 1));
1963	serge	3489
2004	serge	3490	if (rdev->wb.enabled)
		3491	ih_rb_cntl \|= IH_WPTR_WRITEBACK_ENABLE;
		3492
		3493	/* set the writeback address whether it's enabled or not */
		3494	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
		3495	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
		3496
		3497	WREG32(IH_RB_CNTL, ih_rb_cntl);
		3498
		3499	/* set rptr, wptr to 0 */
		3500	WREG32(IH_RB_RPTR, 0);
		3501	WREG32(IH_RB_WPTR, 0);
		3502
		3503	/* Default settings for IH_CNTL (disabled at first) */
		3504	ih_cntl = MC_WRREQ_CREDIT(0x10) \| MC_WR_CLEAN_CNT(0x10);
		3505	/* RPTR_REARM only works if msi's are enabled */
		3506	if (rdev->msi_enabled)
		3507	ih_cntl \|= RPTR_REARM;
		3508	WREG32(IH_CNTL, ih_cntl);
		3509
		3510	/* force the active interrupt state to all disabled */
		3511	if (rdev->family >= CHIP_CEDAR)
		3512	evergreen_disable_interrupt_state(rdev);
		3513	else
		3514	r600_disable_interrupt_state(rdev);
		3515
2997	Serge	3516	/* at this point everything should be setup correctly to enable master */
		3517	pci_set_master(rdev->pdev);
		3518
2004	serge	3519	/* enable irqs */
		3520	r600_enable_interrupts(rdev);
		3521
		3522	return ret;
		3523	}
		3524	int r600_irq_set(struct radeon_device *rdev)
		3525	{
		3526	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
		3527	u32 mode_int = 0;
		3528	u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
		3529	u32 grbm_int_cntl = 0;
2997	Serge	3530	u32 hdmi0, hdmi1;
3192	Serge	3531	u32 dma_cntl;
5078	serge	3532	u32 thermal_int = 0;
2004	serge	3533
		3534	if (!rdev->irq.installed) {
		3535	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		3536	return -EINVAL;
		3537	}
		3538	/* don't enable anything if the ih is disabled */
		3539	if (!rdev->ih.enabled) {
		3540	r600_disable_interrupts(rdev);
		3541	/* force the active interrupt state to all disabled */
		3542	r600_disable_interrupt_state(rdev);
		3543	return 0;
		3544	}
		3545
		3546	if (ASIC_IS_DCE3(rdev)) {
		3547	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3548	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3549	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3550	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3551	if (ASIC_IS_DCE32(rdev)) {
		3552	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3553	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2997	Serge	3554	hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		3555	hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		3556	} else {
		3557	hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3558	hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3559	}
		3560	} else {
		3561	hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3562	hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3563	hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2997	Serge	3564	hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3565	hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3566	}
5078	serge	3567
3192	Serge	3568	dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
2004	serge	3569
5078	serge	3570	if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
		3571	thermal_int = RREG32(CG_THERMAL_INT) &
		3572	~(THERM_INT_MASK_HIGH \| THERM_INT_MASK_LOW);
		3573	} else if (rdev->family >= CHIP_RV770) {
		3574	thermal_int = RREG32(RV770_CG_THERMAL_INT) &
		3575	~(THERM_INT_MASK_HIGH \| THERM_INT_MASK_LOW);
		3576	}
		3577	if (rdev->irq.dpm_thermal) {
		3578	DRM_DEBUG("dpm thermal\n");
		3579	thermal_int \|= THERM_INT_MASK_HIGH \| THERM_INT_MASK_LOW;
		3580	}
		3581
2997	Serge	3582	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
2004	serge	3583	DRM_DEBUG("r600_irq_set: sw int\n");
		3584	cp_int_cntl \|= RB_INT_ENABLE;
		3585	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		3586	}
3192	Serge	3587
		3588	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
		3589	DRM_DEBUG("r600_irq_set: sw int dma\n");
		3590	dma_cntl \|= TRAP_ENABLE;
		3591	}
		3592
2004	serge	3593	if (rdev->irq.crtc_vblank_int[0] \|\|
2997	Serge	3594	atomic_read(&rdev->irq.pflip[0])) {
2004	serge	3595	DRM_DEBUG("r600_irq_set: vblank 0\n");
		3596	mode_int \|= D1MODE_VBLANK_INT_MASK;
		3597	}
		3598	if (rdev->irq.crtc_vblank_int[1] \|\|
2997	Serge	3599	atomic_read(&rdev->irq.pflip[1])) {
2004	serge	3600	DRM_DEBUG("r600_irq_set: vblank 1\n");
		3601	mode_int \|= D2MODE_VBLANK_INT_MASK;
		3602	}
		3603	if (rdev->irq.hpd[0]) {
		3604	DRM_DEBUG("r600_irq_set: hpd 1\n");
		3605	hpd1 \|= DC_HPDx_INT_EN;
		3606	}
		3607	if (rdev->irq.hpd[1]) {
		3608	DRM_DEBUG("r600_irq_set: hpd 2\n");
		3609	hpd2 \|= DC_HPDx_INT_EN;
		3610	}
		3611	if (rdev->irq.hpd[2]) {
		3612	DRM_DEBUG("r600_irq_set: hpd 3\n");
		3613	hpd3 \|= DC_HPDx_INT_EN;
		3614	}
		3615	if (rdev->irq.hpd[3]) {
		3616	DRM_DEBUG("r600_irq_set: hpd 4\n");
		3617	hpd4 \|= DC_HPDx_INT_EN;
		3618	}
		3619	if (rdev->irq.hpd[4]) {
		3620	DRM_DEBUG("r600_irq_set: hpd 5\n");
		3621	hpd5 \|= DC_HPDx_INT_EN;
		3622	}
		3623	if (rdev->irq.hpd[5]) {
		3624	DRM_DEBUG("r600_irq_set: hpd 6\n");
		3625	hpd6 \|= DC_HPDx_INT_EN;
		3626	}
2997	Serge	3627	if (rdev->irq.afmt[0]) {
		3628	DRM_DEBUG("r600_irq_set: hdmi 0\n");
		3629	hdmi0 \|= HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3630	}
2997	Serge	3631	if (rdev->irq.afmt[1]) {
		3632	DRM_DEBUG("r600_irq_set: hdmi 0\n");
		3633	hdmi1 \|= HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3634	}
		3635
		3636	WREG32(CP_INT_CNTL, cp_int_cntl);
3192	Serge	3637	WREG32(DMA_CNTL, dma_cntl);
2004	serge	3638	WREG32(DxMODE_INT_MASK, mode_int);
5078	serge	3639	WREG32(D1GRPH_INTERRUPT_CONTROL, DxGRPH_PFLIP_INT_MASK);
		3640	WREG32(D2GRPH_INTERRUPT_CONTROL, DxGRPH_PFLIP_INT_MASK);
2004	serge	3641	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		3642	if (ASIC_IS_DCE3(rdev)) {
		3643	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		3644	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		3645	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		3646	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		3647	if (ASIC_IS_DCE32(rdev)) {
		3648	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		3649	WREG32(DC_HPD6_INT_CONTROL, hpd6);
2997	Serge	3650	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
		3651	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
		3652	} else {
		3653	WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
		3654	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
2004	serge	3655	}
		3656	} else {
		3657	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
		3658	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
		3659	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
2997	Serge	3660	WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
		3661	WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
2004	serge	3662	}
5078	serge	3663	if ((rdev->family > CHIP_R600) && (rdev->family < CHIP_RV770)) {
		3664	WREG32(CG_THERMAL_INT, thermal_int);
		3665	} else if (rdev->family >= CHIP_RV770) {
		3666	WREG32(RV770_CG_THERMAL_INT, thermal_int);
		3667	}
2004	serge	3668
		3669	return 0;
		3670	}
		3671
2997	Serge	3672	static void r600_irq_ack(struct radeon_device *rdev)
2004	serge	3673	{
		3674	u32 tmp;
		3675
		3676	if (ASIC_IS_DCE3(rdev)) {
		3677	rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
		3678	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
		3679	rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
2997	Serge	3680	if (ASIC_IS_DCE32(rdev)) {
		3681	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
		3682	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
		3683	} else {
		3684	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
		3685	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
		3686	}
2004	serge	3687	} else {
		3688	rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		3689	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		3690	rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
2997	Serge	3691	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
		3692	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
2004	serge	3693	}
		3694	rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
		3695	rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
		3696
		3697	if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		3698	WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		3699	if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		3700	WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		3701	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
		3702	WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		3703	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
		3704	WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		3705	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
		3706	WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		3707	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
		3708	WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		3709	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		3710	if (ASIC_IS_DCE3(rdev)) {
		3711	tmp = RREG32(DC_HPD1_INT_CONTROL);
		3712	tmp \|= DC_HPDx_INT_ACK;
		3713	WREG32(DC_HPD1_INT_CONTROL, tmp);
		3714	} else {
		3715	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		3716	tmp \|= DC_HPDx_INT_ACK;
		3717	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		3718	}
		3719	}
		3720	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		3721	if (ASIC_IS_DCE3(rdev)) {
		3722	tmp = RREG32(DC_HPD2_INT_CONTROL);
		3723	tmp \|= DC_HPDx_INT_ACK;
		3724	WREG32(DC_HPD2_INT_CONTROL, tmp);
		3725	} else {
		3726	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		3727	tmp \|= DC_HPDx_INT_ACK;
		3728	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		3729	}
		3730	}
		3731	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		3732	if (ASIC_IS_DCE3(rdev)) {
		3733	tmp = RREG32(DC_HPD3_INT_CONTROL);
		3734	tmp \|= DC_HPDx_INT_ACK;
		3735	WREG32(DC_HPD3_INT_CONTROL, tmp);
		3736	} else {
		3737	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		3738	tmp \|= DC_HPDx_INT_ACK;
		3739	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		3740	}
		3741	}
		3742	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		3743	tmp = RREG32(DC_HPD4_INT_CONTROL);
		3744	tmp \|= DC_HPDx_INT_ACK;
		3745	WREG32(DC_HPD4_INT_CONTROL, tmp);
		3746	}
		3747	if (ASIC_IS_DCE32(rdev)) {
		3748	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		3749	tmp = RREG32(DC_HPD5_INT_CONTROL);
		3750	tmp \|= DC_HPDx_INT_ACK;
		3751	WREG32(DC_HPD5_INT_CONTROL, tmp);
		3752	}
		3753	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		3754	tmp = RREG32(DC_HPD5_INT_CONTROL);
		3755	tmp \|= DC_HPDx_INT_ACK;
		3756	WREG32(DC_HPD6_INT_CONTROL, tmp);
		3757	}
2997	Serge	3758	if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
		3759	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
		3760	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		3761	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
		3762	}
		3763	if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
		3764	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
		3765	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		3766	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
2004	serge	3767	}
2997	Serge	3768	} else {
		3769	if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
		3770	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
		3771	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		3772	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
2004	serge	3773	}
2997	Serge	3774	if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
2004	serge	3775	if (ASIC_IS_DCE3(rdev)) {
2997	Serge	3776	tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
		3777	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		3778	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
		3779	} else {
		3780	tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
		3781	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		3782	WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
2004	serge	3783	}
		3784	}
		3785	}
		3786	}
		3787
3192	Serge	3788	void r600_irq_disable(struct radeon_device *rdev)
		3789	{
		3790	r600_disable_interrupts(rdev);
		3791	/* Wait and acknowledge irq */
		3792	mdelay(1);
		3793	r600_irq_ack(rdev);
		3794	r600_disable_interrupt_state(rdev);
		3795	}
		3796
2997	Serge	3797	static u32 r600_get_ih_wptr(struct radeon_device *rdev)
2004	serge	3798	{
		3799	u32 wptr, tmp;
		3800
		3801	if (rdev->wb.enabled)
		3802	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		3803	else
		3804	wptr = RREG32(IH_RB_WPTR);
		3805
		3806	if (wptr & RB_OVERFLOW) {
5179	serge	3807	wptr &= ~RB_OVERFLOW;
2004	serge	3808	/* When a ring buffer overflow happen start parsing interrupt
		3809	* from the last not overwritten vector (wptr + 16). Hopefully
		3810	* this should allow us to catchup.
		3811	*/
5179	serge	3812	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
		3813	wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
2004	serge	3814	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		3815	tmp = RREG32(IH_RB_CNTL);
		3816	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		3817	WREG32(IH_RB_CNTL, tmp);
		3818	}
		3819	return (wptr & rdev->ih.ptr_mask);
		3820	}
		3821
		3822	/* r600 IV Ring
		3823	* Each IV ring entry is 128 bits:
		3824	* [7:0] - interrupt source id
		3825	* [31:8] - reserved
		3826	* [59:32] - interrupt source data
		3827	* [127:60] - reserved
		3828	*
		3829	* The basic interrupt vector entries
		3830	* are decoded as follows:
		3831	* src_id src_data description
		3832	* 1 0 D1 Vblank
		3833	* 1 1 D1 Vline
		3834	* 5 0 D2 Vblank
		3835	* 5 1 D2 Vline
		3836	* 19 0 FP Hot plug detection A
		3837	* 19 1 FP Hot plug detection B
		3838	* 19 2 DAC A auto-detection
		3839	* 19 3 DAC B auto-detection
		3840	* 21 4 HDMI block A
		3841	* 21 5 HDMI block B
		3842	* 176 - CP_INT RB
		3843	* 177 - CP_INT IB1
		3844	* 178 - CP_INT IB2
		3845	* 181 - EOP Interrupt
		3846	* 233 - GUI Idle
		3847	*
		3848	* Note, these are based on r600 and may need to be
		3849	* adjusted or added to on newer asics
		3850	*/
3764	Serge	3851	#undef DRM_DEBUG
2160	serge	3852	#define DRM_DEBUG(...)
		3853
2004	serge	3854	int r600_irq_process(struct radeon_device *rdev)
		3855	{
		3856	u32 wptr;
		3857	u32 rptr;
		3858	u32 src_id, src_data;
		3859	u32 ring_index;
		3860	bool queue_hotplug = false;
2997	Serge	3861	bool queue_hdmi = false;
5078	serge	3862	bool queue_thermal = false;
2004	serge	3863
		3864	if (!rdev->ih.enabled \|\| rdev->shutdown)
		3865	return IRQ_NONE;
		3866
2160	serge	3867	/* No MSIs, need a dummy read to flush PCI DMAs */
		3868	if (!rdev->msi_enabled)
		3869	RREG32(IH_RB_WPTR);
		3870
2004	serge	3871	wptr = r600_get_ih_wptr(rdev);
		3872
2997	Serge	3873	restart_ih:
		3874	/* is somebody else already processing irqs? */
		3875	if (atomic_xchg(&rdev->ih.lock, 1))
2004	serge	3876	return IRQ_NONE;
		3877
2997	Serge	3878	rptr = rdev->ih.rptr;
		3879	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		3880
2160	serge	3881	/* Order reading of wptr vs. reading of IH ring data */
		3882	rmb();
		3883
2004	serge	3884	/* display interrupts */
		3885	r600_irq_ack(rdev);
		3886
		3887	while (rptr != wptr) {
		3888	/* wptr/rptr are in bytes! */
		3889	ring_index = rptr / 4;
		3890	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		3891	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		3892
		3893	switch (src_id) {
		3894	case 1: /* D1 vblank/vline */
		3895	switch (src_data) {
		3896	case 0: /* D1 vblank */
		3897	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
		3898	if (rdev->irq.crtc_vblank_int[0]) {
		3899	// drm_handle_vblank(rdev->ddev, 0);
		3900	rdev->pm.vblank_sync = true;
		3901	// wake_up(&rdev->irq.vblank_queue);
		3902	}
		3903	// if (rdev->irq.pflip[0])
		3904	// radeon_crtc_handle_flip(rdev, 0);
		3905	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		3906	DRM_DEBUG("IH: D1 vblank\n");
		3907	}
		3908	break;
		3909	case 1: /* D1 vline */
		3910	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
		3911	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		3912	DRM_DEBUG("IH: D1 vline\n");
		3913	}
		3914	break;
		3915	default:
		3916	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3917	break;
		3918	}
		3919	break;
		3920	case 5: /* D2 vblank/vline */
		3921	switch (src_data) {
		3922	case 0: /* D2 vblank */
		3923	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
		3924	if (rdev->irq.crtc_vblank_int[1]) {
		3925	// drm_handle_vblank(rdev->ddev, 1);
		3926	rdev->pm.vblank_sync = true;
		3927	// wake_up(&rdev->irq.vblank_queue);
		3928	}
		3929	// if (rdev->irq.pflip[1])
		3930	// radeon_crtc_handle_flip(rdev, 1);
		3931	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
		3932	DRM_DEBUG("IH: D2 vblank\n");
		3933	}
		3934	break;
		3935	case 1: /* D1 vline */
		3936	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
		3937	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
		3938	DRM_DEBUG("IH: D2 vline\n");
		3939	}
		3940	break;
		3941	default:
		3942	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3943	break;
		3944	}
		3945	break;
		3946	case 19: /* HPD/DAC hotplug */
		3947	switch (src_data) {
		3948	case 0:
		3949	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		3950	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
		3951	queue_hotplug = true;
		3952	DRM_DEBUG("IH: HPD1\n");
		3953	}
		3954	break;
		3955	case 1:
		3956	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		3957	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
		3958	queue_hotplug = true;
		3959	DRM_DEBUG("IH: HPD2\n");
		3960	}
		3961	break;
		3962	case 4:
		3963	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		3964	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
		3965	queue_hotplug = true;
		3966	DRM_DEBUG("IH: HPD3\n");
		3967	}
		3968	break;
		3969	case 5:
		3970	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		3971	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
		3972	queue_hotplug = true;
		3973	DRM_DEBUG("IH: HPD4\n");
		3974	}
		3975	break;
		3976	case 10:
		3977	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		3978	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
		3979	queue_hotplug = true;
		3980	DRM_DEBUG("IH: HPD5\n");
		3981	}
		3982	break;
		3983	case 12:
		3984	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		3985	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
		3986	queue_hotplug = true;
		3987	DRM_DEBUG("IH: HPD6\n");
		3988	}
		3989	break;
		3990	default:
		3991	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3992	break;
		3993	}
		3994	break;
2997	Serge	3995	case 21: /* hdmi */
		3996	switch (src_data) {
		3997	case 4:
		3998	if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
		3999	rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
		4000	queue_hdmi = true;
		4001	DRM_DEBUG("IH: HDMI0\n");
		4002	}
		4003	break;
		4004	case 5:
		4005	if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
		4006	rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
		4007	queue_hdmi = true;
		4008	DRM_DEBUG("IH: HDMI1\n");
		4009	}
		4010	break;
		4011	default:
		4012	DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
		4013	break;
		4014	}
2004	serge	4015	break;
5078	serge	4016	case 124: /* UVD */
		4017	DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
		4018	radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
		4019	break;
2004	serge	4020	case 176: /* CP_INT in ring buffer */
		4021	case 177: /* CP_INT in IB1 */
		4022	case 178: /* CP_INT in IB2 */
		4023	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2997	Serge	4024	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2004	serge	4025	break;
		4026	case 181: /* CP EOP event */
		4027	DRM_DEBUG("IH: CP EOP\n");
2997	Serge	4028	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2004	serge	4029	break;
3192	Serge	4030	case 224: /* DMA trap event */
		4031	DRM_DEBUG("IH: DMA trap\n");
		4032	radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
		4033	break;
5078	serge	4034	case 230: /* thermal low to high */
		4035	DRM_DEBUG("IH: thermal low to high\n");
		4036	rdev->pm.dpm.thermal.high_to_low = false;
		4037	queue_thermal = true;
		4038	break;
		4039	case 231: /* thermal high to low */
		4040	DRM_DEBUG("IH: thermal high to low\n");
		4041	rdev->pm.dpm.thermal.high_to_low = true;
		4042	queue_thermal = true;
		4043	break;
2004	serge	4044	case 233: /* GUI IDLE */
		4045	DRM_DEBUG("IH: GUI idle\n");
		4046	break;
		4047	default:
		4048	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4049	break;
		4050	}
		4051
		4052	/* wptr/rptr are in bytes! */
		4053	rptr += 16;
		4054	rptr &= rdev->ih.ptr_mask;
5179	serge	4055	WREG32(IH_RB_RPTR, rptr);
2004	serge	4056	}
2997	Serge	4057	rdev->ih.rptr = rptr;
		4058	atomic_set(&rdev->ih.lock, 0);
		4059
2004	serge	4060	/* make sure wptr hasn't changed while processing */
		4061	wptr = r600_get_ih_wptr(rdev);
2997	Serge	4062	if (wptr != rptr)
2004	serge	4063	goto restart_ih;
2997	Serge	4064
2004	serge	4065	return IRQ_HANDLED;
		4066	}
		4067
1221	serge	4068	/*
		4069	* Debugfs info
		4070	*/
		4071	#if defined(CONFIG_DEBUG_FS)
		4072
		4073	static int r600_debugfs_mc_info(struct seq_file m, void data)
		4074	{
		4075	struct drm_info_node node = (struct drm_info_node ) m->private;
		4076	struct drm_device *dev = node->minor->dev;
		4077	struct radeon_device *rdev = dev->dev_private;
		4078
		4079	DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
		4080	DREG32_SYS(m, rdev, VM_L2_STATUS);
		4081	return 0;
		4082	}
		4083
		4084	static struct drm_info_list r600_mc_info_list[] = {
		4085	{"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
		4086	};
		4087	#endif
		4088
		4089	int r600_debugfs_mc_info_init(struct radeon_device *rdev)
		4090	{
		4091	#if defined(CONFIG_DEBUG_FS)
		4092	return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
		4093	#else
		4094	return 0;
		4095	#endif
		4096	}
1404	serge	4097
		4098	/**
5078	serge	4099	* r600_mmio_hdp_flush - flush Host Data Path cache via MMIO
1404	serge	4100	* rdev: radeon device structure
		4101	*
5078	serge	4102	* Some R6XX/R7XX don't seem to take into account HDP flushes performed
		4103	* through the ring buffer. This leads to corruption in rendering, see
		4104	* http://bugzilla.kernel.org/show_bug.cgi?id=15186 . To avoid this, we
		4105	* directly perform the HDP flush by writing the register through MMIO.
1404	serge	4106	*/
5078	serge	4107	void r600_mmio_hdp_flush(struct radeon_device *rdev)
1404	serge	4108	{
1963	serge	4109	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		4110	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
		4111	* This seems to cause problems on some AGP cards. Just use the old
		4112	* method for them.
		4113	*/
		4114	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		4115	rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
		4116	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		4117	u32 tmp;
		4118
		4119	WREG32(HDP_DEBUG1, 0);
		4120	tmp = readl((void __iomem *)ptr);
		4121	} else
1404	serge	4122	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		4123	}
1963	serge	4124
		4125	void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
		4126	{
3764	Serge	4127	u32 link_width_cntl, mask;
1963	serge	4128
		4129	if (rdev->flags & RADEON_IS_IGP)
		4130	return;
		4131
		4132	if (!(rdev->flags & RADEON_IS_PCIE))
		4133	return;
		4134
		4135	/* x2 cards have a special sequence */
		4136	if (ASIC_IS_X2(rdev))
		4137	return;
		4138
3764	Serge	4139	radeon_gui_idle(rdev);
1963	serge	4140
		4141	switch (lanes) {
		4142	case 0:
		4143	mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
		4144	break;
		4145	case 1:
		4146	mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
		4147	break;
		4148	case 2:
		4149	mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
		4150	break;
		4151	case 4:
		4152	mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
		4153	break;
		4154	case 8:
		4155	mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
		4156	break;
		4157	case 12:
3764	Serge	4158	/* not actually supported */
1963	serge	4159	mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
		4160	break;
		4161	case 16:
		4162	mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
		4163	break;
3764	Serge	4164	default:
		4165	DRM_ERROR("invalid pcie lane request: %d\n", lanes);
		4166	return;
1963	serge	4167	}
		4168
3764	Serge	4169	link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		4170	link_width_cntl &= ~RADEON_PCIE_LC_LINK_WIDTH_MASK;
		4171	link_width_cntl \|= mask << RADEON_PCIE_LC_LINK_WIDTH_SHIFT;
		4172	link_width_cntl \|= (RADEON_PCIE_LC_RECONFIG_NOW \|
1963	serge	4173	R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
		4174
3764	Serge	4175	WREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1963	serge	4176	}
		4177
		4178	int r600_get_pcie_lanes(struct radeon_device *rdev)
		4179	{
		4180	u32 link_width_cntl;
		4181
		4182	if (rdev->flags & RADEON_IS_IGP)
		4183	return 0;
		4184
		4185	if (!(rdev->flags & RADEON_IS_PCIE))
		4186	return 0;
		4187
		4188	/* x2 cards have a special sequence */
		4189	if (ASIC_IS_X2(rdev))
		4190	return 0;
		4191
3764	Serge	4192	radeon_gui_idle(rdev);
1963	serge	4193
3764	Serge	4194	link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
1963	serge	4195
		4196	switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
		4197	case RADEON_PCIE_LC_LINK_WIDTH_X1:
		4198	return 1;
		4199	case RADEON_PCIE_LC_LINK_WIDTH_X2:
		4200	return 2;
		4201	case RADEON_PCIE_LC_LINK_WIDTH_X4:
		4202	return 4;
		4203	case RADEON_PCIE_LC_LINK_WIDTH_X8:
		4204	return 8;
3764	Serge	4205	case RADEON_PCIE_LC_LINK_WIDTH_X12:
		4206	/* not actually supported */
		4207	return 12;
		4208	case RADEON_PCIE_LC_LINK_WIDTH_X0:
1963	serge	4209	case RADEON_PCIE_LC_LINK_WIDTH_X16:
		4210	default:
		4211	return 16;
		4212	}
		4213	}
		4214
		4215	static void r600_pcie_gen2_enable(struct radeon_device *rdev)
		4216	{
		4217	u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
		4218	u16 link_cntl2;
		4219
		4220	if (radeon_pcie_gen2 == 0)
		4221	return;
		4222
		4223	if (rdev->flags & RADEON_IS_IGP)
		4224	return;
		4225
		4226	if (!(rdev->flags & RADEON_IS_PCIE))
		4227	return;
		4228
		4229	/* x2 cards have a special sequence */
		4230	if (ASIC_IS_X2(rdev))
		4231	return;
		4232
		4233	/* only RV6xx+ chips are supported */
		4234	if (rdev->family <= CHIP_R600)
		4235	return;
		4236
3764	Serge	4237	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
		4238	(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
2997	Serge	4239	return;
		4240
3764	Serge	4241	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2997	Serge	4242	if (speed_cntl & LC_CURRENT_DATA_RATE) {
		4243	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
		4244	return;
		4245	}
		4246
		4247	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
		4248
1963	serge	4249	/* 55 nm r6xx asics */
		4250	if ((rdev->family == CHIP_RV670) \|\|
		4251	(rdev->family == CHIP_RV620) \|\|
		4252	(rdev->family == CHIP_RV635)) {
		4253	/* advertise upconfig capability */
3764	Serge	4254	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1963	serge	4255	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3764	Serge	4256	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		4257	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1963	serge	4258	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
		4259	lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
		4260	link_width_cntl &= ~(LC_LINK_WIDTH_MASK \|
		4261	LC_RECONFIG_ARC_MISSING_ESCAPE);
		4262	link_width_cntl \|= lanes \| LC_RECONFIG_NOW \| LC_RENEGOTIATE_EN;
3764	Serge	4263	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1963	serge	4264	} else {
		4265	link_width_cntl \|= LC_UPCONFIGURE_DIS;
3764	Serge	4266	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1963	serge	4267	}
		4268	}
		4269
3764	Serge	4270	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1963	serge	4271	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
		4272	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		4273
		4274	/* 55 nm r6xx asics */
		4275	if ((rdev->family == CHIP_RV670) \|\|
		4276	(rdev->family == CHIP_RV620) \|\|
		4277	(rdev->family == CHIP_RV635)) {
		4278	WREG32(MM_CFGREGS_CNTL, 0x8);
		4279	link_cntl2 = RREG32(0x4088);
		4280	WREG32(MM_CFGREGS_CNTL, 0);
		4281	/* not supported yet */
		4282	if (link_cntl2 & SELECTABLE_DEEMPHASIS)
		4283	return;
		4284	}
		4285
		4286	speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
		4287	speed_cntl \|= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
		4288	speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
		4289	speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
		4290	speed_cntl \|= LC_FORCE_EN_HW_SPEED_CHANGE;
3764	Serge	4291	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1963	serge	4292
		4293	tmp = RREG32(0x541c);
		4294	WREG32(0x541c, tmp \| 0x8);
		4295	WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
		4296	link_cntl2 = RREG16(0x4088);
		4297	link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
		4298	link_cntl2 \|= 0x2;
		4299	WREG16(0x4088, link_cntl2);
		4300	WREG32(MM_CFGREGS_CNTL, 0);
		4301
		4302	if ((rdev->family == CHIP_RV670) \|\|
		4303	(rdev->family == CHIP_RV620) \|\|
		4304	(rdev->family == CHIP_RV635)) {
3764	Serge	4305	training_cntl = RREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL);
1963	serge	4306	training_cntl &= ~LC_POINT_7_PLUS_EN;
3764	Serge	4307	WREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL, training_cntl);
1963	serge	4308	} else {
3764	Serge	4309	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1963	serge	4310	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3764	Serge	4311	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1963	serge	4312	}
		4313
3764	Serge	4314	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1963	serge	4315	speed_cntl \|= LC_GEN2_EN_STRAP;
3764	Serge	4316	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1963	serge	4317
		4318	} else {
3764	Serge	4319	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1963	serge	4320	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		4321	if (1)
		4322	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		4323	else
		4324	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3764	Serge	4325	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1963	serge	4326	}
		4327	}
2997	Serge	4328
		4329	/**
3764	Serge	4330	* r600_get_gpu_clock_counter - return GPU clock counter snapshot
2997	Serge	4331	*
		4332	* @rdev: radeon_device pointer
		4333	*
		4334	* Fetches a GPU clock counter snapshot (R6xx-cayman).
		4335	* Returns the 64 bit clock counter snapshot.
		4336	*/
3764	Serge	4337	uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)
2997	Serge	4338	{
		4339	uint64_t clock;
		4340
		4341	mutex_lock(&rdev->gpu_clock_mutex);
		4342	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
		4343	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) \|
		4344	((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
		4345	mutex_unlock(&rdev->gpu_clock_mutex);
		4346	return clock;
		4347	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/r600.c @ 5179 – Rev 5271