WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/evergreen.c

Rev	Author	Line No.	Line
1430	serge	1	/*
		2	* Copyright 2010 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	//#include
1963	serge	26	#include
2997	Serge	27	#include
1430	serge	28	#include "radeon.h"
1963	serge	29	#include "radeon_asic.h"
2997	Serge	30	#include
1963	serge	31	#include "evergreend.h"
1430	serge	32	#include "atom.h"
		33	#include "avivod.h"
		34	#include "evergreen_reg.h"
1986	serge	35	#include "evergreen_blit_shaders.h"
1430	serge	36
1963	serge	37	#define EVERGREEN_PFP_UCODE_SIZE 1120
		38	#define EVERGREEN_PM4_UCODE_SIZE 1376
		39
2997	Serge	40	static const u32 crtc_offsets[6] =
		41	{
		42	EVERGREEN_CRTC0_REGISTER_OFFSET,
		43	EVERGREEN_CRTC1_REGISTER_OFFSET,
		44	EVERGREEN_CRTC2_REGISTER_OFFSET,
		45	EVERGREEN_CRTC3_REGISTER_OFFSET,
		46	EVERGREEN_CRTC4_REGISTER_OFFSET,
		47	EVERGREEN_CRTC5_REGISTER_OFFSET
		48	};
		49
1430	serge	50	static void evergreen_gpu_init(struct radeon_device *rdev);
		51	void evergreen_fini(struct radeon_device *rdev);
2997	Serge	52	void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
		53	extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
		54	int ring, u32 cp_int_cntl);
1430	serge	55
2997	Serge	56	void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
		57	unsigned bankh, unsigned mtaspect,
		58	unsigned *tile_split)
		59	{
		60	*bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
		61	*bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
		62	*mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
		63	*tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
		64	switch (*bankw) {
		65	default:
		66	case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
		67	case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
		68	case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
		69	case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
		70	}
		71	switch (*bankh) {
		72	default:
		73	case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
		74	case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
		75	case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
		76	case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
		77	}
		78	switch (*mtaspect) {
		79	default:
		80	case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
		81	case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
		82	case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
		83	case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
		84	}
		85	}
1990	serge	86
2997	Serge	87	void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
		88	{
		89	u16 ctl, v;
		90	int err;
1990	serge	91
2997	Serge	92	err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
		93	if (err)
		94	return;
		95
		96	v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
		97
		98	/* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
		99	* to avoid hangs or perfomance issues
		100	*/
		101	if ((v == 0) \|\| (v == 6) \|\| (v == 7)) {
		102	ctl &= ~PCI_EXP_DEVCTL_READRQ;
		103	ctl \|= (2 << 12);
		104	pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
		105	}
		106	}
		107
		108	/**
		109	* dce4_wait_for_vblank - vblank wait asic callback.
		110	*
		111	* @rdev: radeon_device pointer
		112	* @crtc: crtc to wait for vblank on
		113	*
		114	* Wait for vblank on the requested crtc (evergreen+).
		115	*/
		116	void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
		117	{
		118	int i;
		119
		120	if (crtc >= rdev->num_crtc)
		121	return;
		122
		123	if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN) {
		124	for (i = 0; i < rdev->usec_timeout; i++) {
		125	if (!(RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK))
		126	break;
		127	udelay(1);
		128	}
		129	for (i = 0; i < rdev->usec_timeout; i++) {
		130	if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
		131	break;
		132	udelay(1);
		133	}
		134	}
		135	}
		136
		137
		138	/**
		139	* evergreen_page_flip - pageflip callback.
		140	*
		141	* @rdev: radeon_device pointer
		142	* @crtc_id: crtc to cleanup pageflip on
		143	* @crtc_base: new address of the crtc (GPU MC address)
		144	*
		145	* Does the actual pageflip (evergreen+).
		146	* During vblank we take the crtc lock and wait for the update_pending
		147	* bit to go high, when it does, we release the lock, and allow the
		148	* double buffered update to take place.
		149	* Returns the current update pending status.
		150	*/
1990	serge	151	u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
		152	{
		153	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
		154	u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
2997	Serge	155	int i;
1990	serge	156
		157	/* Lock the graphics update lock */
		158	tmp \|= EVERGREEN_GRPH_UPDATE_LOCK;
		159	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		160
		161	/* update the scanout addresses */
		162	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		163	upper_32_bits(crtc_base));
		164	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		165	(u32)crtc_base);
		166
		167	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		168	upper_32_bits(crtc_base));
		169	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		170	(u32)crtc_base);
		171
		172	/* Wait for update_pending to go high. */
2997	Serge	173	for (i = 0; i < rdev->usec_timeout; i++) {
		174	if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
		175	break;
		176	udelay(1);
		177	}
1990	serge	178	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
		179
		180	/* Unlock the lock, so double-buffering can take place inside vblank */
		181	tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
		182	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		183
		184	/* Return current update_pending status: */
		185	return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
		186	}
		187
		188	/* get temperature in millidegrees */
		189	int evergreen_get_temp(struct radeon_device *rdev)
		190	{
		191	u32 temp, toffset;
		192	int actual_temp = 0;
		193
		194	if (rdev->family == CHIP_JUNIPER) {
		195	toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
		196	TOFFSET_SHIFT;
		197	temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
		198	TS0_ADC_DOUT_SHIFT;
		199
		200	if (toffset & 0x100)
		201	actual_temp = temp / 2 - (0x200 - toffset);
		202	else
		203	actual_temp = temp / 2 + toffset;
		204
		205	actual_temp = actual_temp * 1000;
		206
		207	} else {
		208	temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
		209	ASIC_T_SHIFT;
		210
		211	if (temp & 0x400)
		212	actual_temp = -256;
		213	else if (temp & 0x200)
		214	actual_temp = 255;
		215	else if (temp & 0x100) {
		216	actual_temp = temp & 0x1ff;
		217	actual_temp \|= ~0x1ff;
		218	} else
		219	actual_temp = temp & 0xff;
		220
		221	actual_temp = (actual_temp * 1000) / 2;
		222	}
		223
		224	return actual_temp;
		225	}
		226
		227	int sumo_get_temp(struct radeon_device *rdev)
		228	{
		229	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
		230	int actual_temp = temp - 49;
		231
		232	return actual_temp * 1000;
		233	}
		234
2997	Serge	235	/**
		236	* sumo_pm_init_profile - Initialize power profiles callback.
		237	*
		238	* @rdev: radeon_device pointer
		239	*
		240	* Initialize the power states used in profile mode
		241	* (sumo, trinity, SI).
		242	* Used for profile mode only.
		243	*/
		244	void sumo_pm_init_profile(struct radeon_device *rdev)
		245	{
		246	int idx;
		247
		248	/* default */
		249	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		250	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		251	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		252	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
		253
		254	/* low,mid sh/mh */
		255	if (rdev->flags & RADEON_IS_MOBILITY)
		256	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
		257	else
		258	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		259
		260	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
		261	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
		262	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		263	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		264
		265	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
		266	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
		267	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		268	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		269
		270	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
		271	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
		272	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		273	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
		274
		275	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
		276	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
		277	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		278	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
		279
		280	/* high sh/mh */
		281	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		282	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
		283	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
		284	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		285	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
		286	rdev->pm.power_state[idx].num_clock_modes - 1;
		287
		288	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
		289	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
		290	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		291	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
		292	rdev->pm.power_state[idx].num_clock_modes - 1;
		293	}
		294
		295	/**
		296	* evergreen_pm_misc - set additional pm hw parameters callback.
		297	*
		298	* @rdev: radeon_device pointer
		299	*
		300	* Set non-clock parameters associated with a power state
		301	* (voltage, etc.) (evergreen+).
		302	*/
1990	serge	303	void evergreen_pm_misc(struct radeon_device *rdev)
		304	{
		305	int req_ps_idx = rdev->pm.requested_power_state_index;
		306	int req_cm_idx = rdev->pm.requested_clock_mode_index;
		307	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
		308	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
		309
		310	if (voltage->type == VOLTAGE_SW) {
		311	/* 0xff01 is a flag rather then an actual voltage */
		312	if (voltage->voltage == 0xff01)
		313	return;
		314	if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
		315	radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
		316	rdev->pm.current_vddc = voltage->voltage;
		317	DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
		318	}
		319	/* 0xff01 is a flag rather then an actual voltage */
		320	if (voltage->vddci == 0xff01)
		321	return;
		322	if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
		323	radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
		324	rdev->pm.current_vddci = voltage->vddci;
		325	DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
		326	}
		327	}
		328	}
		329
2997	Serge	330	/**
		331	* evergreen_pm_prepare - pre-power state change callback.
		332	*
		333	* @rdev: radeon_device pointer
		334	*
		335	* Prepare for a power state change (evergreen+).
		336	*/
1990	serge	337	void evergreen_pm_prepare(struct radeon_device *rdev)
		338	{
		339	struct drm_device *ddev = rdev->ddev;
		340	struct drm_crtc *crtc;
		341	struct radeon_crtc *radeon_crtc;
		342	u32 tmp;
		343
		344	/* disable any active CRTCs */
		345	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		346	radeon_crtc = to_radeon_crtc(crtc);
		347	if (radeon_crtc->enabled) {
		348	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		349	tmp \|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		350	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		351	}
		352	}
		353	}
		354
2997	Serge	355	/**
		356	* evergreen_pm_finish - post-power state change callback.
		357	*
		358	* @rdev: radeon_device pointer
		359	*
		360	* Clean up after a power state change (evergreen+).
		361	*/
1990	serge	362	void evergreen_pm_finish(struct radeon_device *rdev)
		363	{
		364	struct drm_device *ddev = rdev->ddev;
		365	struct drm_crtc *crtc;
		366	struct radeon_crtc *radeon_crtc;
		367	u32 tmp;
		368
		369	/* enable any active CRTCs */
		370	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		371	radeon_crtc = to_radeon_crtc(crtc);
		372	if (radeon_crtc->enabled) {
		373	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		374	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		375	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		376	}
		377	}
		378	}
		379
2997	Serge	380	/**
		381	* evergreen_hpd_sense - hpd sense callback.
		382	*
		383	* @rdev: radeon_device pointer
		384	* @hpd: hpd (hotplug detect) pin
		385	*
		386	* Checks if a digital monitor is connected (evergreen+).
		387	* Returns true if connected, false if not connected.
		388	*/
1430	serge	389	bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		390	{
		391	bool connected = false;
1963	serge	392
		393	switch (hpd) {
		394	case RADEON_HPD_1:
		395	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		396	connected = true;
		397	break;
		398	case RADEON_HPD_2:
		399	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		400	connected = true;
		401	break;
		402	case RADEON_HPD_3:
		403	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		404	connected = true;
		405	break;
		406	case RADEON_HPD_4:
		407	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		408	connected = true;
		409	break;
		410	case RADEON_HPD_5:
		411	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		412	connected = true;
		413	break;
		414	case RADEON_HPD_6:
		415	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		416	connected = true;
		417	break;
		418	default:
		419	break;
		420	}
		421
1430	serge	422	return connected;
		423	}
		424
2997	Serge	425	/**
		426	* evergreen_hpd_set_polarity - hpd set polarity callback.
		427	*
		428	* @rdev: radeon_device pointer
		429	* @hpd: hpd (hotplug detect) pin
		430	*
		431	* Set the polarity of the hpd pin (evergreen+).
		432	*/
1430	serge	433	void evergreen_hpd_set_polarity(struct radeon_device *rdev,
		434	enum radeon_hpd_id hpd)
		435	{
1963	serge	436	u32 tmp;
		437	bool connected = evergreen_hpd_sense(rdev, hpd);
		438
		439	switch (hpd) {
		440	case RADEON_HPD_1:
		441	tmp = RREG32(DC_HPD1_INT_CONTROL);
		442	if (connected)
		443	tmp &= ~DC_HPDx_INT_POLARITY;
		444	else
		445	tmp \|= DC_HPDx_INT_POLARITY;
		446	WREG32(DC_HPD1_INT_CONTROL, tmp);
		447	break;
		448	case RADEON_HPD_2:
		449	tmp = RREG32(DC_HPD2_INT_CONTROL);
		450	if (connected)
		451	tmp &= ~DC_HPDx_INT_POLARITY;
		452	else
		453	tmp \|= DC_HPDx_INT_POLARITY;
		454	WREG32(DC_HPD2_INT_CONTROL, tmp);
		455	break;
		456	case RADEON_HPD_3:
		457	tmp = RREG32(DC_HPD3_INT_CONTROL);
		458	if (connected)
		459	tmp &= ~DC_HPDx_INT_POLARITY;
		460	else
		461	tmp \|= DC_HPDx_INT_POLARITY;
		462	WREG32(DC_HPD3_INT_CONTROL, tmp);
		463	break;
		464	case RADEON_HPD_4:
		465	tmp = RREG32(DC_HPD4_INT_CONTROL);
		466	if (connected)
		467	tmp &= ~DC_HPDx_INT_POLARITY;
		468	else
		469	tmp \|= DC_HPDx_INT_POLARITY;
		470	WREG32(DC_HPD4_INT_CONTROL, tmp);
		471	break;
		472	case RADEON_HPD_5:
		473	tmp = RREG32(DC_HPD5_INT_CONTROL);
		474	if (connected)
		475	tmp &= ~DC_HPDx_INT_POLARITY;
		476	else
		477	tmp \|= DC_HPDx_INT_POLARITY;
		478	WREG32(DC_HPD5_INT_CONTROL, tmp);
		479	break;
		480	case RADEON_HPD_6:
		481	tmp = RREG32(DC_HPD6_INT_CONTROL);
		482	if (connected)
		483	tmp &= ~DC_HPDx_INT_POLARITY;
		484	else
		485	tmp \|= DC_HPDx_INT_POLARITY;
		486	WREG32(DC_HPD6_INT_CONTROL, tmp);
		487	break;
		488	default:
		489	break;
		490	}
1430	serge	491	}
		492
2997	Serge	493	/**
		494	* evergreen_hpd_init - hpd setup callback.
		495	*
		496	* @rdev: radeon_device pointer
		497	*
		498	* Setup the hpd pins used by the card (evergreen+).
		499	* Enable the pin, set the polarity, and enable the hpd interrupts.
		500	*/
1430	serge	501	void evergreen_hpd_init(struct radeon_device *rdev)
		502	{
1963	serge	503	struct drm_device *dev = rdev->ddev;
		504	struct drm_connector *connector;
2997	Serge	505	unsigned enabled = 0;
1963	serge	506	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \|
		507	DC_HPDx_RX_INT_TIMER(0xfa) \| DC_HPDx_EN;
		508
		509	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		510	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		511	switch (radeon_connector->hpd.hpd) {
		512	case RADEON_HPD_1:
		513	WREG32(DC_HPD1_CONTROL, tmp);
		514	break;
		515	case RADEON_HPD_2:
		516	WREG32(DC_HPD2_CONTROL, tmp);
		517	break;
		518	case RADEON_HPD_3:
		519	WREG32(DC_HPD3_CONTROL, tmp);
		520	break;
		521	case RADEON_HPD_4:
		522	WREG32(DC_HPD4_CONTROL, tmp);
		523	break;
		524	case RADEON_HPD_5:
		525	WREG32(DC_HPD5_CONTROL, tmp);
		526	break;
		527	case RADEON_HPD_6:
		528	WREG32(DC_HPD6_CONTROL, tmp);
		529	break;
		530	default:
		531	break;
		532	}
2997	Serge	533	radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
		534	enabled \|= 1 << radeon_connector->hpd.hpd;
1963	serge	535	}
2997	Serge	536	// radeon_irq_kms_enable_hpd(rdev, enabled);
1430	serge	537	}
		538
2997	Serge	539	/**
		540	* evergreen_hpd_fini - hpd tear down callback.
		541	*
		542	* @rdev: radeon_device pointer
		543	*
		544	* Tear down the hpd pins used by the card (evergreen+).
		545	* Disable the hpd interrupts.
		546	*/
1963	serge	547	void evergreen_hpd_fini(struct radeon_device *rdev)
		548	{
		549	struct drm_device *dev = rdev->ddev;
		550	struct drm_connector *connector;
2997	Serge	551	unsigned disabled = 0;
1430	serge	552
1963	serge	553	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		554	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		555	switch (radeon_connector->hpd.hpd) {
		556	case RADEON_HPD_1:
		557	WREG32(DC_HPD1_CONTROL, 0);
		558	break;
		559	case RADEON_HPD_2:
		560	WREG32(DC_HPD2_CONTROL, 0);
		561	break;
		562	case RADEON_HPD_3:
		563	WREG32(DC_HPD3_CONTROL, 0);
		564	break;
		565	case RADEON_HPD_4:
		566	WREG32(DC_HPD4_CONTROL, 0);
		567	break;
		568	case RADEON_HPD_5:
		569	WREG32(DC_HPD5_CONTROL, 0);
		570	break;
		571	case RADEON_HPD_6:
		572	WREG32(DC_HPD6_CONTROL, 0);
		573	break;
		574	default:
		575	break;
		576	}
2997	Serge	577	disabled \|= 1 << radeon_connector->hpd.hpd;
1963	serge	578	}
2997	Serge	579	// radeon_irq_kms_disable_hpd(rdev, disabled);
1430	serge	580	}
		581
1986	serge	582	/* watermark setup */
1963	serge	583
1986	serge	584	static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
		585	struct radeon_crtc *radeon_crtc,
		586	struct drm_display_mode *mode,
		587	struct drm_display_mode *other_mode)
		588	{
		589	u32 tmp;
		590	/*
		591	* Line Buffer Setup
		592	* There are 3 line buffers, each one shared by 2 display controllers.
		593	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
		594	* the display controllers. The paritioning is done via one of four
		595	* preset allocations specified in bits 2:0:
		596	* first display controller
		597	* 0 - first half of lb (3840 * 2)
		598	* 1 - first 3/4 of lb (5760 * 2)
		599	* 2 - whole lb (7680 * 2), other crtc must be disabled
		600	* 3 - first 1/4 of lb (1920 * 2)
		601	* second display controller
		602	* 4 - second half of lb (3840 * 2)
		603	* 5 - second 3/4 of lb (5760 * 2)
		604	* 6 - whole lb (7680 * 2), other crtc must be disabled
		605	* 7 - last 1/4 of lb (1920 * 2)
		606	*/
		607	/* this can get tricky if we have two large displays on a paired group
		608	* of crtcs. Ideally for multiple large displays we'd assign them to
		609	* non-linked crtcs for maximum line buffer allocation.
		610	*/
		611	if (radeon_crtc->base.enabled && mode) {
		612	if (other_mode)
		613	tmp = 0; /* 1/2 */
		614	else
		615	tmp = 2; /* whole */
		616	} else
		617	tmp = 0;
1963	serge	618
1986	serge	619	/* second controller of the pair uses second half of the lb */
		620	if (radeon_crtc->crtc_id % 2)
		621	tmp += 4;
		622	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
		623
		624	if (radeon_crtc->base.enabled && mode) {
		625	switch (tmp) {
		626	case 0:
		627	case 4:
		628	default:
		629	if (ASIC_IS_DCE5(rdev))
		630	return 4096 * 2;
		631	else
		632	return 3840 * 2;
		633	case 1:
		634	case 5:
		635	if (ASIC_IS_DCE5(rdev))
		636	return 6144 * 2;
		637	else
		638	return 5760 * 2;
		639	case 2:
		640	case 6:
		641	if (ASIC_IS_DCE5(rdev))
		642	return 8192 * 2;
		643	else
		644	return 7680 * 2;
		645	case 3:
		646	case 7:
		647	if (ASIC_IS_DCE5(rdev))
		648	return 2048 * 2;
		649	else
		650	return 1920 * 2;
		651	}
		652	}
		653
		654	/* controller not enabled, so no lb used */
		655	return 0;
		656	}
		657
2997	Serge	658	u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1986	serge	659	{
		660	u32 tmp = RREG32(MC_SHARED_CHMAP);
		661
		662	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		663	case 0:
		664	default:
		665	return 1;
		666	case 1:
		667	return 2;
		668	case 2:
		669	return 4;
		670	case 3:
		671	return 8;
		672	}
		673	}
		674
		675	struct evergreen_wm_params {
		676	u32 dram_channels; /* number of dram channels */
		677	u32 yclk; /* bandwidth per dram data pin in kHz */
		678	u32 sclk; /* engine clock in kHz */
		679	u32 disp_clk; /* display clock in kHz */
		680	u32 src_width; /* viewport width */
		681	u32 active_time; /* active display time in ns */
		682	u32 blank_time; /* blank time in ns */
		683	bool interlaced; /* mode is interlaced */
		684	fixed20_12 vsc; /* vertical scale ratio */
		685	u32 num_heads; /* number of active crtcs */
		686	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
		687	u32 lb_size; /* line buffer allocated to pipe */
		688	u32 vtaps; /* vertical scaler taps */
		689	};
		690
		691	static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
		692	{
		693	/* Calculate DRAM Bandwidth and the part allocated to display. */
		694	fixed20_12 dram_efficiency; /* 0.7 */
		695	fixed20_12 yclk, dram_channels, bandwidth;
		696	fixed20_12 a;
		697
		698	a.full = dfixed_const(1000);
		699	yclk.full = dfixed_const(wm->yclk);
		700	yclk.full = dfixed_div(yclk, a);
		701	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		702	a.full = dfixed_const(10);
		703	dram_efficiency.full = dfixed_const(7);
		704	dram_efficiency.full = dfixed_div(dram_efficiency, a);
		705	bandwidth.full = dfixed_mul(dram_channels, yclk);
		706	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
		707
		708	return dfixed_trunc(bandwidth);
		709	}
		710
		711	static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		712	{
		713	/* Calculate DRAM Bandwidth and the part allocated to display. */
		714	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
		715	fixed20_12 yclk, dram_channels, bandwidth;
		716	fixed20_12 a;
		717
		718	a.full = dfixed_const(1000);
		719	yclk.full = dfixed_const(wm->yclk);
		720	yclk.full = dfixed_div(yclk, a);
		721	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		722	a.full = dfixed_const(10);
		723	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
		724	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
		725	bandwidth.full = dfixed_mul(dram_channels, yclk);
		726	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
		727
		728	return dfixed_trunc(bandwidth);
		729	}
		730
		731	static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
		732	{
		733	/* Calculate the display Data return Bandwidth */
		734	fixed20_12 return_efficiency; /* 0.8 */
		735	fixed20_12 sclk, bandwidth;
		736	fixed20_12 a;
		737
		738	a.full = dfixed_const(1000);
		739	sclk.full = dfixed_const(wm->sclk);
		740	sclk.full = dfixed_div(sclk, a);
		741	a.full = dfixed_const(10);
		742	return_efficiency.full = dfixed_const(8);
		743	return_efficiency.full = dfixed_div(return_efficiency, a);
		744	a.full = dfixed_const(32);
		745	bandwidth.full = dfixed_mul(a, sclk);
		746	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
		747
		748	return dfixed_trunc(bandwidth);
		749	}
		750
		751	static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
		752	{
		753	/* Calculate the DMIF Request Bandwidth */
		754	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
		755	fixed20_12 disp_clk, bandwidth;
		756	fixed20_12 a;
		757
		758	a.full = dfixed_const(1000);
		759	disp_clk.full = dfixed_const(wm->disp_clk);
		760	disp_clk.full = dfixed_div(disp_clk, a);
		761	a.full = dfixed_const(10);
		762	disp_clk_request_efficiency.full = dfixed_const(8);
		763	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
		764	a.full = dfixed_const(32);
		765	bandwidth.full = dfixed_mul(a, disp_clk);
		766	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
		767
		768	return dfixed_trunc(bandwidth);
		769	}
		770
		771	static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
		772	{
		773	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
		774	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
		775	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
		776	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
		777
		778	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
		779	}
		780
		781	static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
		782	{
		783	/* Calculate the display mode Average Bandwidth
		784	* DisplayMode should contain the source and destination dimensions,
		785	* timing, etc.
		786	*/
		787	fixed20_12 bpp;
		788	fixed20_12 line_time;
		789	fixed20_12 src_width;
		790	fixed20_12 bandwidth;
		791	fixed20_12 a;
		792
		793	a.full = dfixed_const(1000);
		794	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
		795	line_time.full = dfixed_div(line_time, a);
		796	bpp.full = dfixed_const(wm->bytes_per_pixel);
		797	src_width.full = dfixed_const(wm->src_width);
		798	bandwidth.full = dfixed_mul(src_width, bpp);
		799	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
		800	bandwidth.full = dfixed_div(bandwidth, line_time);
		801
		802	return dfixed_trunc(bandwidth);
		803	}
		804
		805	static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
		806	{
		807	/* First calcualte the latency in ns */
		808	u32 mc_latency = 2000; /* 2000 ns. */
		809	u32 available_bandwidth = evergreen_available_bandwidth(wm);
		810	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
		811	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
		812	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
		813	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
		814	(wm->num_heads * cursor_line_pair_return_time);
		815	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
		816	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
		817	fixed20_12 a, b, c;
		818
		819	if (wm->num_heads == 0)
		820	return 0;
		821
		822	a.full = dfixed_const(2);
		823	b.full = dfixed_const(1);
		824	if ((wm->vsc.full > a.full) \|\|
		825	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) \|\|
		826	(wm->vtaps >= 5) \|\|
		827	((wm->vsc.full >= a.full) && wm->interlaced))
		828	max_src_lines_per_dst_line = 4;
		829	else
		830	max_src_lines_per_dst_line = 2;
		831
		832	a.full = dfixed_const(available_bandwidth);
		833	b.full = dfixed_const(wm->num_heads);
		834	a.full = dfixed_div(a, b);
		835
		836	b.full = dfixed_const(1000);
		837	c.full = dfixed_const(wm->disp_clk);
		838	b.full = dfixed_div(c, b);
		839	c.full = dfixed_const(wm->bytes_per_pixel);
		840	b.full = dfixed_mul(b, c);
		841
		842	lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
		843
		844	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
		845	b.full = dfixed_const(1000);
		846	c.full = dfixed_const(lb_fill_bw);
		847	b.full = dfixed_div(c, b);
		848	a.full = dfixed_div(a, b);
		849	line_fill_time = dfixed_trunc(a);
		850
		851	if (line_fill_time < wm->active_time)
		852	return latency;
		853	else
		854	return latency + (line_fill_time - wm->active_time);
		855
		856	}
		857
		858	static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		859	{
		860	if (evergreen_average_bandwidth(wm) <=
		861	(evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
		862	return true;
		863	else
		864	return false;
		865	};
		866
		867	static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
		868	{
		869	if (evergreen_average_bandwidth(wm) <=
		870	(evergreen_available_bandwidth(wm) / wm->num_heads))
		871	return true;
		872	else
		873	return false;
		874	};
		875
		876	static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
		877	{
		878	u32 lb_partitions = wm->lb_size / wm->src_width;
		879	u32 line_time = wm->active_time + wm->blank_time;
		880	u32 latency_tolerant_lines;
		881	u32 latency_hiding;
		882	fixed20_12 a;
		883
		884	a.full = dfixed_const(1);
		885	if (wm->vsc.full > a.full)
		886	latency_tolerant_lines = 1;
		887	else {
		888	if (lb_partitions <= (wm->vtaps + 1))
		889	latency_tolerant_lines = 1;
		890	else
		891	latency_tolerant_lines = 2;
		892	}
		893
		894	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
		895
		896	if (evergreen_latency_watermark(wm) <= latency_hiding)
		897	return true;
		898	else
		899	return false;
		900	}
		901
		902	static void evergreen_program_watermarks(struct radeon_device *rdev,
		903	struct radeon_crtc *radeon_crtc,
		904	u32 lb_size, u32 num_heads)
		905	{
		906	struct drm_display_mode *mode = &radeon_crtc->base.mode;
		907	struct evergreen_wm_params wm;
		908	u32 pixel_period;
		909	u32 line_time = 0;
		910	u32 latency_watermark_a = 0, latency_watermark_b = 0;
		911	u32 priority_a_mark = 0, priority_b_mark = 0;
		912	u32 priority_a_cnt = PRIORITY_OFF;
		913	u32 priority_b_cnt = PRIORITY_OFF;
		914	u32 pipe_offset = radeon_crtc->crtc_id * 16;
		915	u32 tmp, arb_control3;
		916	fixed20_12 a, b, c;
		917
		918	if (radeon_crtc->base.enabled && num_heads && mode) {
		919	pixel_period = 1000000 / (u32)mode->clock;
		920	line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
		921	priority_a_cnt = 0;
		922	priority_b_cnt = 0;
		923
		924	wm.yclk = rdev->pm.current_mclk * 10;
		925	wm.sclk = rdev->pm.current_sclk * 10;
		926	wm.disp_clk = mode->clock;
		927	wm.src_width = mode->crtc_hdisplay;
		928	wm.active_time = mode->crtc_hdisplay * pixel_period;
		929	wm.blank_time = line_time - wm.active_time;
		930	wm.interlaced = false;
		931	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		932	wm.interlaced = true;
		933	wm.vsc = radeon_crtc->vsc;
		934	wm.vtaps = 1;
		935	if (radeon_crtc->rmx_type != RMX_OFF)
		936	wm.vtaps = 2;
		937	wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
		938	wm.lb_size = lb_size;
		939	wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
		940	wm.num_heads = num_heads;
		941
		942	/* set for high clocks */
		943	latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
		944	/* set for low clocks */
		945	/* wm.yclk = low clk; wm.sclk = low clk */
		946	latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
		947
		948	/* possibly force display priority to high */
		949	/* should really do this at mode validation time... */
		950	if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) \|\|
		951	!evergreen_average_bandwidth_vs_available_bandwidth(&wm) \|\|
		952	!evergreen_check_latency_hiding(&wm) \|\|
		953	(rdev->disp_priority == 2)) {
2160	serge	954	DRM_DEBUG_KMS("force priority to high\n");
1986	serge	955	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		956	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		957	}
		958
		959	a.full = dfixed_const(1000);
		960	b.full = dfixed_const(mode->clock);
		961	b.full = dfixed_div(b, a);
		962	c.full = dfixed_const(latency_watermark_a);
		963	c.full = dfixed_mul(c, b);
		964	c.full = dfixed_mul(c, radeon_crtc->hsc);
		965	c.full = dfixed_div(c, a);
		966	a.full = dfixed_const(16);
		967	c.full = dfixed_div(c, a);
		968	priority_a_mark = dfixed_trunc(c);
		969	priority_a_cnt \|= priority_a_mark & PRIORITY_MARK_MASK;
		970
		971	a.full = dfixed_const(1000);
		972	b.full = dfixed_const(mode->clock);
		973	b.full = dfixed_div(b, a);
		974	c.full = dfixed_const(latency_watermark_b);
		975	c.full = dfixed_mul(c, b);
		976	c.full = dfixed_mul(c, radeon_crtc->hsc);
		977	c.full = dfixed_div(c, a);
		978	a.full = dfixed_const(16);
		979	c.full = dfixed_div(c, a);
		980	priority_b_mark = dfixed_trunc(c);
		981	priority_b_cnt \|= priority_b_mark & PRIORITY_MARK_MASK;
		982	}
		983
		984	/* select wm A */
		985	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		986	tmp = arb_control3;
		987	tmp &= ~LATENCY_WATERMARK_MASK(3);
		988	tmp \|= LATENCY_WATERMARK_MASK(1);
		989	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		990	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		991	(LATENCY_LOW_WATERMARK(latency_watermark_a) \|
		992	LATENCY_HIGH_WATERMARK(line_time)));
		993	/* select wm B */
		994	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		995	tmp &= ~LATENCY_WATERMARK_MASK(3);
		996	tmp \|= LATENCY_WATERMARK_MASK(2);
		997	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		998	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		999	(LATENCY_LOW_WATERMARK(latency_watermark_b) \|
		1000	LATENCY_HIGH_WATERMARK(line_time)));
		1001	/* restore original selection */
		1002	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
		1003
		1004	/* write the priority marks */
		1005	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
		1006	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
		1007
		1008	}
		1009
2997	Serge	1010	/**
		1011	* evergreen_bandwidth_update - update display watermarks callback.
		1012	*
		1013	* @rdev: radeon_device pointer
		1014	*
		1015	* Update the display watermarks based on the requested mode(s)
		1016	* (evergreen+).
		1017	*/
1963	serge	1018	void evergreen_bandwidth_update(struct radeon_device *rdev)
1430	serge	1019	{
1986	serge	1020	struct drm_display_mode *mode0 = NULL;
		1021	struct drm_display_mode *mode1 = NULL;
		1022	u32 num_heads = 0, lb_size;
		1023	int i;
		1024
		1025	radeon_update_display_priority(rdev);
		1026
		1027	for (i = 0; i < rdev->num_crtc; i++) {
		1028	if (rdev->mode_info.crtcs[i]->base.enabled)
		1029	num_heads++;
		1030	}
		1031	for (i = 0; i < rdev->num_crtc; i += 2) {
		1032	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
		1033	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
		1034	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
		1035	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
		1036	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
		1037	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
		1038	}
1430	serge	1039	}
		1040
2997	Serge	1041	/**
		1042	* evergreen_mc_wait_for_idle - wait for MC idle callback.
		1043	*
		1044	* @rdev: radeon_device pointer
		1045	*
		1046	* Wait for the MC (memory controller) to be idle.
		1047	* (evergreen+).
		1048	* Returns 0 if the MC is idle, -1 if not.
		1049	*/
1963	serge	1050	int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1430	serge	1051	{
		1052	unsigned i;
		1053	u32 tmp;
		1054
		1055	for (i = 0; i < rdev->usec_timeout; i++) {
		1056	/* read MC_STATUS */
		1057	tmp = RREG32(SRBM_STATUS) & 0x1F00;
		1058	if (!tmp)
		1059	return 0;
		1060	udelay(1);
		1061	}
		1062	return -1;
		1063	}
		1064
		1065	/*
		1066	* GART
		1067	*/
1963	serge	1068	void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
		1069	{
		1070	unsigned i;
		1071	u32 tmp;
		1072
		1073	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		1074
		1075	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		1076	for (i = 0; i < rdev->usec_timeout; i++) {
		1077	/* read MC_STATUS */
		1078	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		1079	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		1080	if (tmp == 2) {
		1081	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		1082	return;
		1083	}
		1084	if (tmp) {
		1085	return;
		1086	}
		1087	udelay(1);
		1088	}
		1089	}
		1090
2997	Serge	1091	static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
1430	serge	1092	{
		1093	u32 tmp;
1963	serge	1094	int r;
1430	serge	1095
2997	Serge	1096	if (rdev->gart.robj == NULL) {
1430	serge	1097	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		1098	return -EINVAL;
		1099	}
		1100	r = radeon_gart_table_vram_pin(rdev);
		1101	if (r)
		1102	return r;
		1103	radeon_gart_restore(rdev);
		1104	/* Setup L2 cache */
		1105	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		1106	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1107	EFFECTIVE_L2_QUEUE_SIZE(7));
		1108	WREG32(VM_L2_CNTL2, 0);
		1109	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		1110	/* Setup TLB control */
		1111	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		1112	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		1113	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		1114	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
1963	serge	1115	if (rdev->flags & RADEON_IS_IGP) {
		1116	WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
		1117	WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
		1118	WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
		1119	} else {
3031	serge	1120	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		1121	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		1122	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2997	Serge	1123	if ((rdev->family == CHIP_JUNIPER) \|\|
		1124	(rdev->family == CHIP_CYPRESS) \|\|
		1125	(rdev->family == CHIP_HEMLOCK) \|\|
		1126	(rdev->family == CHIP_BARTS))
		1127	WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
1963	serge	1128	}
1430	serge	1129	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		1130	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		1131	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		1132	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
		1133	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		1134	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		1135	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		1136	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		1137	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		1138	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		1139	(u32)(rdev->dummy_page.addr >> 12));
1963	serge	1140	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	1141
1963	serge	1142	evergreen_pcie_gart_tlb_flush(rdev);
2997	Serge	1143	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		1144	(unsigned)(rdev->mc.gtt_size >> 20),
		1145	(unsigned long long)rdev->gart.table_addr);
1430	serge	1146	rdev->gart.ready = true;
		1147	return 0;
		1148	}
		1149
2997	Serge	1150	static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
1430	serge	1151	{
		1152	u32 tmp;
		1153
		1154	/* Disable all tables */
1963	serge	1155	WREG32(VM_CONTEXT0_CNTL, 0);
		1156	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	1157
		1158	/* Setup L2 cache */
		1159	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		1160	EFFECTIVE_L2_QUEUE_SIZE(7));
		1161	WREG32(VM_L2_CNTL2, 0);
		1162	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		1163	/* Setup TLB control */
		1164	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		1165	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		1166	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		1167	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		1168	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		1169	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		1170	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		1171	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2997	Serge	1172	radeon_gart_table_vram_unpin(rdev);
1430	serge	1173	}
		1174
2997	Serge	1175	static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
1430	serge	1176	{
		1177	evergreen_pcie_gart_disable(rdev);
		1178	radeon_gart_table_vram_free(rdev);
		1179	radeon_gart_fini(rdev);
		1180	}
		1181
		1182
2997	Serge	1183	static void evergreen_agp_enable(struct radeon_device *rdev)
1430	serge	1184	{
		1185	u32 tmp;
		1186
		1187	/* Setup L2 cache */
		1188	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		1189	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1190	EFFECTIVE_L2_QUEUE_SIZE(7));
		1191	WREG32(VM_L2_CNTL2, 0);
		1192	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		1193	/* Setup TLB control */
		1194	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		1195	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		1196	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		1197	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		1198	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		1199	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		1200	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		1201	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		1202	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		1203	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		1204	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1963	serge	1205	WREG32(VM_CONTEXT0_CNTL, 0);
		1206	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	1207	}
		1208
1963	serge	1209	void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	1210	{
2997	Serge	1211	u32 crtc_enabled, tmp, frame_count, blackout;
		1212	int i, j;
		1213
1430	serge	1214	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
		1215	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
		1216
2997	Serge	1217	/* disable VGA render */
1430	serge	1218	WREG32(VGA_RENDER_CONTROL, 0);
2997	Serge	1219	/* blank the display controllers */
		1220	for (i = 0; i < rdev->num_crtc; i++) {
		1221	crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
		1222	if (crtc_enabled) {
		1223	save->crtc_enabled[i] = true;
		1224	if (ASIC_IS_DCE6(rdev)) {
		1225	tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
		1226	if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
		1227	radeon_wait_for_vblank(rdev, i);
		1228	tmp \|= EVERGREEN_CRTC_BLANK_DATA_EN;
		1229	WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
3031	serge	1230	}
2997	Serge	1231	} else {
		1232	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
		1233	if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
		1234	radeon_wait_for_vblank(rdev, i);
		1235	tmp \|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		1236	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
3031	serge	1237	}
		1238	}
2997	Serge	1239	/* wait for the next frame */
		1240	frame_count = radeon_get_vblank_counter(rdev, i);
		1241	for (j = 0; j < rdev->usec_timeout; j++) {
		1242	if (radeon_get_vblank_counter(rdev, i) != frame_count)
		1243	break;
		1244	udelay(1);
3031	serge	1245	}
3120	serge	1246	} else {
		1247	save->crtc_enabled[i] = false;
3031	serge	1248	}
1963	serge	1249	}
1430	serge	1250
2997	Serge	1251	radeon_mc_wait_for_idle(rdev);
		1252
		1253	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
		1254	if ((blackout & BLACKOUT_MODE_MASK) != 1) {
		1255	/* Block CPU access */
		1256	WREG32(BIF_FB_EN, 0);
		1257	/* blackout the MC */
		1258	blackout &= ~BLACKOUT_MODE_MASK;
		1259	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout \| 1);
2005	serge	1260	}
1430	serge	1261	}
		1262
1963	serge	1263	void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	1264	{
2997	Serge	1265	u32 tmp, frame_count;
		1266	int i, j;
1430	serge	1267
2997	Serge	1268	/* update crtc base addresses */
		1269	for (i = 0; i < rdev->num_crtc; i++) {
		1270	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
3031	serge	1271	upper_32_bits(rdev->mc.vram_start));
2997	Serge	1272	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
3031	serge	1273	upper_32_bits(rdev->mc.vram_start));
2997	Serge	1274	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
3031	serge	1275	(u32)rdev->mc.vram_start);
2997	Serge	1276	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
3031	serge	1277	(u32)rdev->mc.vram_start);
2997	Serge	1278	}
		1279	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
		1280	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
1430	serge	1281
2997	Serge	1282	/* unblackout the MC */
		1283	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
		1284	tmp &= ~BLACKOUT_MODE_MASK;
		1285	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
		1286	/* allow CPU access */
		1287	WREG32(BIF_FB_EN, FB_READ_EN \| FB_WRITE_EN);
1430	serge	1288
2997	Serge	1289	for (i = 0; i < rdev->num_crtc; i++) {
3031	serge	1290	if (save->crtc_enabled[i]) {
2997	Serge	1291	if (ASIC_IS_DCE6(rdev)) {
		1292	tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
		1293	tmp \|= EVERGREEN_CRTC_BLANK_DATA_EN;
		1294	WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
		1295	} else {
		1296	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
		1297	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		1298	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
		1299	}
		1300	/* wait for the next frame */
		1301	frame_count = radeon_get_vblank_counter(rdev, i);
		1302	for (j = 0; j < rdev->usec_timeout; j++) {
		1303	if (radeon_get_vblank_counter(rdev, i) != frame_count)
		1304	break;
		1305	udelay(1);
		1306	}
3031	serge	1307	}
2005	serge	1308	}
2997	Serge	1309	/* Unlock vga access */
1430	serge	1310	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
		1311	mdelay(1);
		1312	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
		1313	}
		1314
1963	serge	1315	void evergreen_mc_program(struct radeon_device *rdev)
1430	serge	1316	{
		1317	struct evergreen_mc_save save;
		1318	u32 tmp;
		1319	int i, j;
		1320
		1321	/* Initialize HDP */
		1322	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1323	WREG32((0x2c14 + j), 0x00000000);
		1324	WREG32((0x2c18 + j), 0x00000000);
		1325	WREG32((0x2c1c + j), 0x00000000);
		1326	WREG32((0x2c20 + j), 0x00000000);
		1327	WREG32((0x2c24 + j), 0x00000000);
		1328	}
		1329	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		1330
		1331	evergreen_mc_stop(rdev, &save);
		1332	if (evergreen_mc_wait_for_idle(rdev)) {
		1333	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1334	}
		1335	/* Lockout access through VGA aperture*/
		1336	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		1337	/* Update configuration */
		1338	if (rdev->flags & RADEON_IS_AGP) {
		1339	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		1340	/* VRAM before AGP */
		1341	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1342	rdev->mc.vram_start >> 12);
		1343	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1344	rdev->mc.gtt_end >> 12);
		1345	} else {
		1346	/* VRAM after AGP */
		1347	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1348	rdev->mc.gtt_start >> 12);
		1349	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1350	rdev->mc.vram_end >> 12);
		1351	}
		1352	} else {
		1353	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1354	rdev->mc.vram_start >> 12);
		1355	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1356	rdev->mc.vram_end >> 12);
		1357	}
2997	Serge	1358	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
		1359	/* llano/ontario only */
		1360	if ((rdev->family == CHIP_PALM) \|\|
		1361	(rdev->family == CHIP_SUMO) \|\|
		1362	(rdev->family == CHIP_SUMO2)) {
1963	serge	1363	tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
		1364	tmp \|= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
		1365	tmp \|= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
		1366	WREG32(MC_FUS_VM_FB_OFFSET, tmp);
		1367	}
1430	serge	1368	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		1369	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		1370	WREG32(MC_VM_FB_LOCATION, tmp);
		1371	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1963	serge	1372	WREG32(HDP_NONSURFACE_INFO, (2 << 7) \| (1 << 30));
		1373	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1430	serge	1374	if (rdev->flags & RADEON_IS_AGP) {
		1375	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
		1376	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
		1377	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		1378	} else {
		1379	WREG32(MC_VM_AGP_BASE, 0);
		1380	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		1381	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		1382	}
		1383	if (evergreen_mc_wait_for_idle(rdev)) {
		1384	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1385	}
		1386	evergreen_mc_resume(rdev, &save);
		1387	/* we need to own VRAM, so turn off the VGA renderer here
		1388	* to stop it overwriting our objects */
		1389	rv515_vga_render_disable(rdev);
		1390	}
		1391
		1392	/*
		1393	* CP.
		1394	*/
1986	serge	1395	void evergreen_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		1396	{
2997	Serge	1397	struct radeon_ring *ring = &rdev->ring[ib->ring];
		1398	u32 next_rptr;
		1399
1986	serge	1400	/* set to DX10/11 mode */
2997	Serge	1401	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
		1402	radeon_ring_write(ring, 1);
		1403
		1404	if (ring->rptr_save_reg) {
		1405	next_rptr = ring->wptr + 3 + 4;
		1406	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3031	serge	1407	radeon_ring_write(ring, ((ring->rptr_save_reg -
2997	Serge	1408	PACKET3_SET_CONFIG_REG_START) >> 2));
		1409	radeon_ring_write(ring, next_rptr);
		1410	} else if (rdev->wb.enabled) {
		1411	next_rptr = ring->wptr + 5 + 4;
		1412	radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
		1413	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		1414	radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) \| (1 << 18));
		1415	radeon_ring_write(ring, next_rptr);
		1416	radeon_ring_write(ring, 0);
		1417	}
		1418
		1419	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		1420	radeon_ring_write(ring,
1986	serge	1421	#ifdef __BIG_ENDIAN
		1422	(2 << 0) \|
		1423	#endif
		1424	(ib->gpu_addr & 0xFFFFFFFC));
2997	Serge	1425	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
		1426	radeon_ring_write(ring, ib->length_dw);
1986	serge	1427	}
1963	serge	1428
1986	serge	1429
1963	serge	1430	static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1430	serge	1431	{
1963	serge	1432	const __be32 *fw_data;
		1433	int i;
		1434
		1435	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1436	return -EINVAL;
		1437
		1438	r700_cp_stop(rdev);
		1439	WREG32(CP_RB_CNTL,
		1440	#ifdef __BIG_ENDIAN
		1441	BUF_SWAP_32BIT \|
		1442	#endif
		1443	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
		1444
		1445	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1446	WREG32(CP_PFP_UCODE_ADDR, 0);
		1447	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
		1448	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		1449	WREG32(CP_PFP_UCODE_ADDR, 0);
		1450
		1451	fw_data = (const __be32 *)rdev->me_fw->data;
		1452	WREG32(CP_ME_RAM_WADDR, 0);
		1453	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
		1454	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		1455
		1456	WREG32(CP_PFP_UCODE_ADDR, 0);
		1457	WREG32(CP_ME_RAM_WADDR, 0);
		1458	WREG32(CP_ME_RAM_RADDR, 0);
		1459	return 0;
1430	serge	1460	}
		1461
1963	serge	1462	static int evergreen_cp_start(struct radeon_device *rdev)
		1463	{
2997	Serge	1464	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	1465	int r, i;
		1466	uint32_t cp_me;
1430	serge	1467
2997	Serge	1468	r = radeon_ring_lock(rdev, ring, 7);
1963	serge	1469	if (r) {
		1470	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1471	return r;
		1472	}
2997	Serge	1473	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1474	radeon_ring_write(ring, 0x1);
		1475	radeon_ring_write(ring, 0x0);
		1476	radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
		1477	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1478	radeon_ring_write(ring, 0);
		1479	radeon_ring_write(ring, 0);
		1480	radeon_ring_unlock_commit(rdev, ring);
1963	serge	1481
		1482	cp_me = 0xff;
		1483	WREG32(CP_ME_CNTL, cp_me);
		1484
2997	Serge	1485	r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
1963	serge	1486	if (r) {
		1487	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1488	return r;
		1489	}
		1490
		1491	/* setup clear context state */
2997	Serge	1492	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1493	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1963	serge	1494
		1495	for (i = 0; i < evergreen_default_size; i++)
2997	Serge	1496	radeon_ring_write(ring, evergreen_default_state[i]);
1963	serge	1497
2997	Serge	1498	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1499	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1963	serge	1500
		1501	/* set clear context state */
2997	Serge	1502	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
		1503	radeon_ring_write(ring, 0);
1963	serge	1504
		1505	/* SQ_VTX_BASE_VTX_LOC */
2997	Serge	1506	radeon_ring_write(ring, 0xc0026f00);
		1507	radeon_ring_write(ring, 0x00000000);
		1508	radeon_ring_write(ring, 0x00000000);
		1509	radeon_ring_write(ring, 0x00000000);
1963	serge	1510
		1511	/* Clear consts */
2997	Serge	1512	radeon_ring_write(ring, 0xc0036f00);
		1513	radeon_ring_write(ring, 0x00000bc4);
		1514	radeon_ring_write(ring, 0xffffffff);
		1515	radeon_ring_write(ring, 0xffffffff);
		1516	radeon_ring_write(ring, 0xffffffff);
1963	serge	1517
2997	Serge	1518	radeon_ring_write(ring, 0xc0026900);
		1519	radeon_ring_write(ring, 0x00000316);
		1520	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		1521	radeon_ring_write(ring, 0x00000010); /* */
1963	serge	1522
2997	Serge	1523	radeon_ring_unlock_commit(rdev, ring);
1963	serge	1524
		1525	return 0;
		1526	}
		1527
2997	Serge	1528	static int evergreen_cp_resume(struct radeon_device *rdev)
1430	serge	1529	{
2997	Serge	1530	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	1531	u32 tmp;
		1532	u32 rb_bufsz;
		1533	int r;
1430	serge	1534
1963	serge	1535	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
		1536	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP \|
		1537	SOFT_RESET_PA \|
		1538	SOFT_RESET_SH \|
		1539	SOFT_RESET_VGT \|
2160	serge	1540	SOFT_RESET_SPI \|
1963	serge	1541	SOFT_RESET_SX));
		1542	RREG32(GRBM_SOFT_RESET);
		1543	mdelay(15);
		1544	WREG32(GRBM_SOFT_RESET, 0);
		1545	RREG32(GRBM_SOFT_RESET);
		1546
		1547	/* Set ring buffer size */
2997	Serge	1548	rb_bufsz = drm_order(ring->ring_size / 8);
1963	serge	1549	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
		1550	#ifdef __BIG_ENDIAN
		1551	tmp \|= BUF_SWAP_32BIT;
		1552	#endif
		1553	WREG32(CP_RB_CNTL, tmp);
2997	Serge	1554	WREG32(CP_SEM_WAIT_TIMER, 0x0);
		1555	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1963	serge	1556
		1557	/* Set the write pointer delay */
		1558	WREG32(CP_RB_WPTR_DELAY, 0);
		1559
		1560	/* Initialize the ring buffer's read and write pointers */
		1561	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1562	WREG32(CP_RB_RPTR_WR, 0);
2997	Serge	1563	ring->wptr = 0;
		1564	WREG32(CP_RB_WPTR, ring->wptr);
1963	serge	1565
3120	serge	1566	/* set the wb address whether it's enabled or not */
1963	serge	1567	WREG32(CP_RB_RPTR_ADDR,
		1568	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1569	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1570	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1571
		1572	if (rdev->wb.enabled)
		1573	WREG32(SCRATCH_UMSK, 0xff);
		1574	else {
		1575	tmp \|= RB_NO_UPDATE;
		1576	WREG32(SCRATCH_UMSK, 0);
		1577	}
		1578
		1579	mdelay(1);
		1580	WREG32(CP_RB_CNTL, tmp);
		1581
2997	Serge	1582	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1963	serge	1583	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1584
2997	Serge	1585	ring->rptr = RREG32(CP_RB_RPTR);
1963	serge	1586
		1587	evergreen_cp_start(rdev);
2997	Serge	1588	ring->ready = true;
		1589	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
1963	serge	1590	if (r) {
2997	Serge	1591	ring->ready = false;
1963	serge	1592	return r;
		1593	}
1430	serge	1594	return 0;
		1595	}
		1596
		1597	/*
		1598	* Core functions
		1599	*/
		1600	static void evergreen_gpu_init(struct radeon_device *rdev)
		1601	{
2997	Serge	1602	u32 gb_addr_config;
1963	serge	1603	u32 mc_shared_chmap, mc_arb_ramcfg;
		1604	u32 sx_debug_1;
		1605	u32 smx_dc_ctl0;
		1606	u32 sq_config;
		1607	u32 sq_lds_resource_mgmt;
		1608	u32 sq_gpr_resource_mgmt_1;
		1609	u32 sq_gpr_resource_mgmt_2;
		1610	u32 sq_gpr_resource_mgmt_3;
		1611	u32 sq_thread_resource_mgmt;
		1612	u32 sq_thread_resource_mgmt_2;
		1613	u32 sq_stack_resource_mgmt_1;
		1614	u32 sq_stack_resource_mgmt_2;
		1615	u32 sq_stack_resource_mgmt_3;
		1616	u32 vgt_cache_invalidation;
		1617	u32 hdp_host_path_cntl, tmp;
2997	Serge	1618	u32 disabled_rb_mask;
1963	serge	1619	int i, j, num_shader_engines, ps_thread_count;
		1620
		1621	switch (rdev->family) {
		1622	case CHIP_CYPRESS:
		1623	case CHIP_HEMLOCK:
		1624	rdev->config.evergreen.num_ses = 2;
		1625	rdev->config.evergreen.max_pipes = 4;
		1626	rdev->config.evergreen.max_tile_pipes = 8;
		1627	rdev->config.evergreen.max_simds = 10;
		1628	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1629	rdev->config.evergreen.max_gprs = 256;
		1630	rdev->config.evergreen.max_threads = 248;
		1631	rdev->config.evergreen.max_gs_threads = 32;
		1632	rdev->config.evergreen.max_stack_entries = 512;
		1633	rdev->config.evergreen.sx_num_of_sets = 4;
		1634	rdev->config.evergreen.sx_max_export_size = 256;
		1635	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1636	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1637	rdev->config.evergreen.max_hw_contexts = 8;
		1638	rdev->config.evergreen.sq_num_cf_insts = 2;
		1639
		1640	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1641	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1642	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1643	gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1644	break;
		1645	case CHIP_JUNIPER:
		1646	rdev->config.evergreen.num_ses = 1;
		1647	rdev->config.evergreen.max_pipes = 4;
		1648	rdev->config.evergreen.max_tile_pipes = 4;
		1649	rdev->config.evergreen.max_simds = 10;
		1650	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1651	rdev->config.evergreen.max_gprs = 256;
		1652	rdev->config.evergreen.max_threads = 248;
		1653	rdev->config.evergreen.max_gs_threads = 32;
		1654	rdev->config.evergreen.max_stack_entries = 512;
		1655	rdev->config.evergreen.sx_num_of_sets = 4;
		1656	rdev->config.evergreen.sx_max_export_size = 256;
		1657	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1658	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1659	rdev->config.evergreen.max_hw_contexts = 8;
		1660	rdev->config.evergreen.sq_num_cf_insts = 2;
		1661
		1662	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1663	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1664	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1665	gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1666	break;
		1667	case CHIP_REDWOOD:
		1668	rdev->config.evergreen.num_ses = 1;
		1669	rdev->config.evergreen.max_pipes = 4;
		1670	rdev->config.evergreen.max_tile_pipes = 4;
		1671	rdev->config.evergreen.max_simds = 5;
		1672	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1673	rdev->config.evergreen.max_gprs = 256;
		1674	rdev->config.evergreen.max_threads = 248;
		1675	rdev->config.evergreen.max_gs_threads = 32;
		1676	rdev->config.evergreen.max_stack_entries = 256;
		1677	rdev->config.evergreen.sx_num_of_sets = 4;
		1678	rdev->config.evergreen.sx_max_export_size = 256;
		1679	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1680	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1681	rdev->config.evergreen.max_hw_contexts = 8;
		1682	rdev->config.evergreen.sq_num_cf_insts = 2;
		1683
		1684	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1685	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1686	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1687	gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1688	break;
		1689	case CHIP_CEDAR:
		1690	default:
		1691	rdev->config.evergreen.num_ses = 1;
		1692	rdev->config.evergreen.max_pipes = 2;
		1693	rdev->config.evergreen.max_tile_pipes = 2;
		1694	rdev->config.evergreen.max_simds = 2;
		1695	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1696	rdev->config.evergreen.max_gprs = 256;
		1697	rdev->config.evergreen.max_threads = 192;
		1698	rdev->config.evergreen.max_gs_threads = 16;
		1699	rdev->config.evergreen.max_stack_entries = 256;
		1700	rdev->config.evergreen.sx_num_of_sets = 4;
		1701	rdev->config.evergreen.sx_max_export_size = 128;
		1702	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1703	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1704	rdev->config.evergreen.max_hw_contexts = 4;
		1705	rdev->config.evergreen.sq_num_cf_insts = 1;
		1706
		1707	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1708	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1709	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1710	gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1711	break;
		1712	case CHIP_PALM:
		1713	rdev->config.evergreen.num_ses = 1;
		1714	rdev->config.evergreen.max_pipes = 2;
		1715	rdev->config.evergreen.max_tile_pipes = 2;
		1716	rdev->config.evergreen.max_simds = 2;
		1717	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1718	rdev->config.evergreen.max_gprs = 256;
		1719	rdev->config.evergreen.max_threads = 192;
		1720	rdev->config.evergreen.max_gs_threads = 16;
		1721	rdev->config.evergreen.max_stack_entries = 256;
		1722	rdev->config.evergreen.sx_num_of_sets = 4;
		1723	rdev->config.evergreen.sx_max_export_size = 128;
		1724	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1725	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1726	rdev->config.evergreen.max_hw_contexts = 4;
		1727	rdev->config.evergreen.sq_num_cf_insts = 1;
		1728
		1729	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1730	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1731	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1732	gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1733	break;
		1734	case CHIP_SUMO:
		1735	rdev->config.evergreen.num_ses = 1;
		1736	rdev->config.evergreen.max_pipes = 4;
3192	Serge	1737	rdev->config.evergreen.max_tile_pipes = 4;
1963	serge	1738	if (rdev->pdev->device == 0x9648)
		1739	rdev->config.evergreen.max_simds = 3;
		1740	else if ((rdev->pdev->device == 0x9647) \|\|
		1741	(rdev->pdev->device == 0x964a))
		1742	rdev->config.evergreen.max_simds = 4;
		1743	else
		1744	rdev->config.evergreen.max_simds = 5;
		1745	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1746	rdev->config.evergreen.max_gprs = 256;
		1747	rdev->config.evergreen.max_threads = 248;
		1748	rdev->config.evergreen.max_gs_threads = 32;
		1749	rdev->config.evergreen.max_stack_entries = 256;
		1750	rdev->config.evergreen.sx_num_of_sets = 4;
		1751	rdev->config.evergreen.sx_max_export_size = 256;
		1752	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1753	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1754	rdev->config.evergreen.max_hw_contexts = 8;
		1755	rdev->config.evergreen.sq_num_cf_insts = 2;
		1756
		1757	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1758	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1759	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3192	Serge	1760	gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1761	break;
		1762	case CHIP_SUMO2:
		1763	rdev->config.evergreen.num_ses = 1;
		1764	rdev->config.evergreen.max_pipes = 4;
		1765	rdev->config.evergreen.max_tile_pipes = 4;
		1766	rdev->config.evergreen.max_simds = 2;
		1767	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1768	rdev->config.evergreen.max_gprs = 256;
		1769	rdev->config.evergreen.max_threads = 248;
		1770	rdev->config.evergreen.max_gs_threads = 32;
		1771	rdev->config.evergreen.max_stack_entries = 512;
		1772	rdev->config.evergreen.sx_num_of_sets = 4;
		1773	rdev->config.evergreen.sx_max_export_size = 256;
		1774	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1775	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1776	rdev->config.evergreen.max_hw_contexts = 8;
		1777	rdev->config.evergreen.sq_num_cf_insts = 2;
		1778
		1779	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1780	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1781	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3192	Serge	1782	gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1783	break;
		1784	case CHIP_BARTS:
		1785	rdev->config.evergreen.num_ses = 2;
		1786	rdev->config.evergreen.max_pipes = 4;
		1787	rdev->config.evergreen.max_tile_pipes = 8;
		1788	rdev->config.evergreen.max_simds = 7;
		1789	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1790	rdev->config.evergreen.max_gprs = 256;
		1791	rdev->config.evergreen.max_threads = 248;
		1792	rdev->config.evergreen.max_gs_threads = 32;
		1793	rdev->config.evergreen.max_stack_entries = 512;
		1794	rdev->config.evergreen.sx_num_of_sets = 4;
		1795	rdev->config.evergreen.sx_max_export_size = 256;
		1796	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1797	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1798	rdev->config.evergreen.max_hw_contexts = 8;
		1799	rdev->config.evergreen.sq_num_cf_insts = 2;
		1800
		1801	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1802	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1803	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1804	gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1805	break;
		1806	case CHIP_TURKS:
		1807	rdev->config.evergreen.num_ses = 1;
		1808	rdev->config.evergreen.max_pipes = 4;
		1809	rdev->config.evergreen.max_tile_pipes = 4;
		1810	rdev->config.evergreen.max_simds = 6;
		1811	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1812	rdev->config.evergreen.max_gprs = 256;
		1813	rdev->config.evergreen.max_threads = 248;
		1814	rdev->config.evergreen.max_gs_threads = 32;
		1815	rdev->config.evergreen.max_stack_entries = 256;
		1816	rdev->config.evergreen.sx_num_of_sets = 4;
		1817	rdev->config.evergreen.sx_max_export_size = 256;
		1818	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1819	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1820	rdev->config.evergreen.max_hw_contexts = 8;
		1821	rdev->config.evergreen.sq_num_cf_insts = 2;
		1822
		1823	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1824	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1825	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1826	gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1827	break;
		1828	case CHIP_CAICOS:
		1829	rdev->config.evergreen.num_ses = 1;
3192	Serge	1830	rdev->config.evergreen.max_pipes = 2;
1963	serge	1831	rdev->config.evergreen.max_tile_pipes = 2;
		1832	rdev->config.evergreen.max_simds = 2;
		1833	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1834	rdev->config.evergreen.max_gprs = 256;
		1835	rdev->config.evergreen.max_threads = 192;
		1836	rdev->config.evergreen.max_gs_threads = 16;
		1837	rdev->config.evergreen.max_stack_entries = 256;
		1838	rdev->config.evergreen.sx_num_of_sets = 4;
		1839	rdev->config.evergreen.sx_max_export_size = 128;
		1840	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1841	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1842	rdev->config.evergreen.max_hw_contexts = 4;
		1843	rdev->config.evergreen.sq_num_cf_insts = 1;
		1844
		1845	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1846	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1847	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1848	gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1849	break;
		1850	}
		1851
		1852	/* Initialize HDP */
		1853	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1854	WREG32((0x2c14 + j), 0x00000000);
		1855	WREG32((0x2c18 + j), 0x00000000);
		1856	WREG32((0x2c1c + j), 0x00000000);
		1857	WREG32((0x2c20 + j), 0x00000000);
		1858	WREG32((0x2c24 + j), 0x00000000);
		1859	}
		1860
		1861	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1862
2997	Serge	1863	evergreen_fix_pci_max_read_req_size(rdev);
1963	serge	1864
		1865	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2997	Serge	1866	if ((rdev->family == CHIP_PALM) \|\|
		1867	(rdev->family == CHIP_SUMO) \|\|
		1868	(rdev->family == CHIP_SUMO2))
1963	serge	1869	mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
		1870	else
3031	serge	1871	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1963	serge	1872
		1873	/* setup tiling info dword. gb_addr_config is not adequate since it does
		1874	* not have bank info, so create a custom tiling dword.
		1875	* bits 3:0 num_pipes
		1876	* bits 7:4 num_banks
		1877	* bits 11:8 group_size
		1878	* bits 15:12 row_size
		1879	*/
		1880	rdev->config.evergreen.tile_config = 0;
		1881	switch (rdev->config.evergreen.max_tile_pipes) {
		1882	case 1:
		1883	default:
		1884	rdev->config.evergreen.tile_config \|= (0 << 0);
		1885	break;
		1886	case 2:
		1887	rdev->config.evergreen.tile_config \|= (1 << 0);
		1888	break;
		1889	case 4:
		1890	rdev->config.evergreen.tile_config \|= (2 << 0);
		1891	break;
		1892	case 8:
		1893	rdev->config.evergreen.tile_config \|= (3 << 0);
		1894	break;
		1895	}
1986	serge	1896	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1963	serge	1897	if (rdev->flags & RADEON_IS_IGP)
1986	serge	1898	rdev->config.evergreen.tile_config \|= 1 << 4;
2997	Serge	1899	else {
		1900	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
		1901	case 0: /* four banks */
		1902	rdev->config.evergreen.tile_config \|= 0 << 4;
		1903	break;
		1904	case 1: /* eight banks */
		1905	rdev->config.evergreen.tile_config \|= 1 << 4;
		1906	break;
		1907	case 2: /* sixteen banks */
		1908	default:
		1909	rdev->config.evergreen.tile_config \|= 2 << 4;
		1910	break;
		1911	}
		1912	}
		1913	rdev->config.evergreen.tile_config \|= 0 << 8;
1963	serge	1914	rdev->config.evergreen.tile_config \|=
		1915	((gb_addr_config & 0x30000000) >> 28) << 12;
		1916
2997	Serge	1917	num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
1963	serge	1918
2997	Serge	1919	if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
		1920	u32 efuse_straps_4;
		1921	u32 efuse_straps_3;
1963	serge	1922
2997	Serge	1923	WREG32(RCU_IND_INDEX, 0x204);
		1924	efuse_straps_4 = RREG32(RCU_IND_DATA);
		1925	WREG32(RCU_IND_INDEX, 0x203);
		1926	efuse_straps_3 = RREG32(RCU_IND_DATA);
		1927	tmp = (((efuse_straps_4 & 0xf) << 4) \|
		1928	((efuse_straps_3 & 0xf0000000) >> 28));
		1929	} else {
		1930	tmp = 0;
		1931	for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
		1932	u32 rb_disable_bitmap;
1963	serge	1933
2997	Serge	1934	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		1935	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		1936	rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
		1937	tmp <<= 4;
		1938	tmp \|= rb_disable_bitmap;
1963	serge	1939	}
2997	Serge	1940	}
		1941	/* enabled rb are just the one not disabled :) */
		1942	disabled_rb_mask = tmp;
1963	serge	1943
2997	Serge	1944	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
		1945	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
1963	serge	1946
2997	Serge	1947	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		1948	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
		1949	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3192	Serge	1950	WREG32(DMA_TILING_CONFIG, gb_addr_config);
1963	serge	1951
2997	Serge	1952	tmp = gb_addr_config & NUM_PIPES_MASK;
		1953	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
		1954	EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
		1955	WREG32(GB_BACKEND_MAP, tmp);
1963	serge	1956
		1957	WREG32(CGTS_SYS_TCC_DISABLE, 0);
		1958	WREG32(CGTS_TCC_DISABLE, 0);
		1959	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
		1960	WREG32(CGTS_USER_TCC_DISABLE, 0);
		1961
		1962	/* set HW defaults for 3D engine */
		1963	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \|
		1964	ROQ_IB2_START(0x2b)));
		1965
		1966	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
		1967
		1968	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \|
		1969	SYNC_GRADIENT \|
		1970	SYNC_WALKER \|
		1971	SYNC_ALIGNER));
		1972
		1973	sx_debug_1 = RREG32(SX_DEBUG_1);
		1974	sx_debug_1 \|= ENABLE_NEW_SMX_ADDRESS;
		1975	WREG32(SX_DEBUG_1, sx_debug_1);
		1976
		1977
		1978	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
		1979	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
		1980	smx_dc_ctl0 \|= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
		1981	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
		1982
2997	Serge	1983	if (rdev->family <= CHIP_SUMO2)
		1984	WREG32(SMX_SAR_CTL0, 0x00010000);
		1985
1963	serge	1986	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) \|
		1987	POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) \|
		1988	SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
		1989
		1990	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) \|
		1991	SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) \|
		1992	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
		1993
		1994	WREG32(VGT_NUM_INSTANCES, 1);
		1995	WREG32(SPI_CONFIG_CNTL, 0);
		1996	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
		1997	WREG32(CP_PERFMON_CNTL, 0);
		1998
		1999	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) \|
		2000	FETCH_FIFO_HIWATER(0x4) \|
		2001	DONE_FIFO_HIWATER(0xe0) \|
		2002	ALU_UPDATE_FIFO_HIWATER(0x8)));
		2003
		2004	sq_config = RREG32(SQ_CONFIG);
		2005	sq_config &= ~(PS_PRIO(3) \|
		2006	VS_PRIO(3) \|
		2007	GS_PRIO(3) \|
		2008	ES_PRIO(3));
		2009	sq_config \|= (VC_ENABLE \|
		2010	EXPORT_SRC_C \|
		2011	PS_PRIO(0) \|
		2012	VS_PRIO(1) \|
		2013	GS_PRIO(2) \|
		2014	ES_PRIO(3));
		2015
		2016	switch (rdev->family) {
		2017	case CHIP_CEDAR:
		2018	case CHIP_PALM:
		2019	case CHIP_SUMO:
		2020	case CHIP_SUMO2:
		2021	case CHIP_CAICOS:
		2022	/* no vertex cache */
		2023	sq_config &= ~VC_ENABLE;
		2024	break;
		2025	default:
		2026	break;
		2027	}
		2028
		2029	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
		2030
		2031	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
		2032	sq_gpr_resource_mgmt_1 \|= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
		2033	sq_gpr_resource_mgmt_1 \|= NUM_CLAUSE_TEMP_GPRS(4);
		2034	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		2035	sq_gpr_resource_mgmt_2 \|= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		2036	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		2037	sq_gpr_resource_mgmt_3 \|= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		2038
		2039	switch (rdev->family) {
		2040	case CHIP_CEDAR:
		2041	case CHIP_PALM:
		2042	case CHIP_SUMO:
		2043	case CHIP_SUMO2:
		2044	ps_thread_count = 96;
		2045	break;
		2046	default:
		2047	ps_thread_count = 128;
		2048	break;
		2049	}
		2050
		2051	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
		2052	sq_thread_resource_mgmt \|= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2053	sq_thread_resource_mgmt \|= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2054	sq_thread_resource_mgmt \|= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2055	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2056	sq_thread_resource_mgmt_2 \|= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2057
		2058	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2059	sq_stack_resource_mgmt_1 \|= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2060	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2061	sq_stack_resource_mgmt_2 \|= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2062	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2063	sq_stack_resource_mgmt_3 \|= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2064
		2065	WREG32(SQ_CONFIG, sq_config);
		2066	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		2067	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		2068	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
		2069	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		2070	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
		2071	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		2072	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		2073	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
		2074	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
		2075	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
		2076
		2077	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		2078	FORCE_EOV_MAX_REZ_CNT(255)));
		2079
		2080	switch (rdev->family) {
		2081	case CHIP_CEDAR:
		2082	case CHIP_PALM:
		2083	case CHIP_SUMO:
		2084	case CHIP_SUMO2:
		2085	case CHIP_CAICOS:
		2086	vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
		2087	break;
		2088	default:
		2089	vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
		2090	break;
		2091	}
		2092	vgt_cache_invalidation \|= AUTO_INVLD_EN(ES_AND_GS_AUTO);
		2093	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
		2094
		2095	WREG32(VGT_GS_VERTEX_REUSE, 16);
		2096	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
		2097	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		2098
		2099	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
		2100	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
		2101
		2102	WREG32(CB_PERF_CTR0_SEL_0, 0);
		2103	WREG32(CB_PERF_CTR0_SEL_1, 0);
		2104	WREG32(CB_PERF_CTR1_SEL_0, 0);
		2105	WREG32(CB_PERF_CTR1_SEL_1, 0);
		2106	WREG32(CB_PERF_CTR2_SEL_0, 0);
		2107	WREG32(CB_PERF_CTR2_SEL_1, 0);
		2108	WREG32(CB_PERF_CTR3_SEL_0, 0);
		2109	WREG32(CB_PERF_CTR3_SEL_1, 0);
		2110
		2111	/* clear render buffer base addresses */
		2112	WREG32(CB_COLOR0_BASE, 0);
		2113	WREG32(CB_COLOR1_BASE, 0);
		2114	WREG32(CB_COLOR2_BASE, 0);
		2115	WREG32(CB_COLOR3_BASE, 0);
		2116	WREG32(CB_COLOR4_BASE, 0);
		2117	WREG32(CB_COLOR5_BASE, 0);
		2118	WREG32(CB_COLOR6_BASE, 0);
		2119	WREG32(CB_COLOR7_BASE, 0);
		2120	WREG32(CB_COLOR8_BASE, 0);
		2121	WREG32(CB_COLOR9_BASE, 0);
		2122	WREG32(CB_COLOR10_BASE, 0);
		2123	WREG32(CB_COLOR11_BASE, 0);
		2124
		2125	/* set the shader const cache sizes to 0 */
		2126	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
		2127	WREG32(i, 0);
		2128	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
		2129	WREG32(i, 0);
		2130
		2131	tmp = RREG32(HDP_MISC_CNTL);
		2132	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		2133	WREG32(HDP_MISC_CNTL, tmp);
		2134
		2135	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		2136	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		2137
		2138	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		2139
		2140	udelay(50);
		2141
1430	serge	2142	}
		2143
		2144	int evergreen_mc_init(struct radeon_device *rdev)
		2145	{
		2146	u32 tmp;
		2147	int chansize, numchan;
		2148
		2149	/* Get VRAM informations */
		2150	rdev->mc.vram_is_ddr = true;
2997	Serge	2151	if ((rdev->family == CHIP_PALM) \|\|
		2152	(rdev->family == CHIP_SUMO) \|\|
		2153	(rdev->family == CHIP_SUMO2))
2004	serge	2154	tmp = RREG32(FUS_MC_ARB_RAMCFG);
		2155	else
3031	serge	2156	tmp = RREG32(MC_ARB_RAMCFG);
1430	serge	2157	if (tmp & CHANSIZE_OVERRIDE) {
		2158	chansize = 16;
		2159	} else if (tmp & CHANSIZE_MASK) {
		2160	chansize = 64;
		2161	} else {
		2162	chansize = 32;
		2163	}
		2164	tmp = RREG32(MC_SHARED_CHMAP);
		2165	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		2166	case 0:
		2167	default:
		2168	numchan = 1;
		2169	break;
		2170	case 1:
		2171	numchan = 2;
		2172	break;
		2173	case 2:
		2174	numchan = 4;
		2175	break;
		2176	case 3:
		2177	numchan = 8;
		2178	break;
		2179	}
		2180	rdev->mc.vram_width = numchan * chansize;
		2181	/* Could aper size report 0 ? */
1963	serge	2182	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		2183	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1430	serge	2184	/* Setup GPU memory space */
2997	Serge	2185	if ((rdev->family == CHIP_PALM) \|\|
		2186	(rdev->family == CHIP_SUMO) \|\|
		2187	(rdev->family == CHIP_SUMO2)) {
1963	serge	2188	/* size in bytes on fusion */
		2189	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		2190	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
		2191	} else {
2997	Serge	2192	/* size in MB on evergreen/cayman/tn */
3031	serge	2193	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
		2194	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
1963	serge	2195	}
1430	serge	2196	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1963	serge	2197	r700_vram_gtt_location(rdev, &rdev->mc);
		2198	radeon_update_bandwidth_info(rdev);
		2199
1430	serge	2200	return 0;
		2201	}
		2202
2997	Serge	2203	bool evergreen_gpu_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
1430	serge	2204	{
1986	serge	2205	u32 srbm_status;
		2206	u32 grbm_status;
		2207	u32 grbm_status_se0, grbm_status_se1;
		2208
		2209	srbm_status = RREG32(SRBM_STATUS);
		2210	grbm_status = RREG32(GRBM_STATUS);
		2211	grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
		2212	grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
		2213	if (!(grbm_status & GUI_ACTIVE)) {
2997	Serge	2214	radeon_ring_lockup_update(ring);
3031	serge	2215	return false;
1986	serge	2216	}
		2217	/* force CP activities */
2997	Serge	2218	radeon_ring_force_activity(rdev, ring);
		2219	return radeon_ring_test_lockup(rdev, ring);
1963	serge	2220	}
		2221
		2222	static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
		2223	{
		2224	struct evergreen_mc_save save;
		2225	u32 grbm_reset = 0;
		2226
		2227	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
		2228	return 0;
		2229
		2230	dev_info(rdev->dev, "GPU softreset \n");
		2231	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2232	RREG32(GRBM_STATUS));
		2233	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2234	RREG32(GRBM_STATUS_SE0));
		2235	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2236	RREG32(GRBM_STATUS_SE1));
		2237	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2238	RREG32(SRBM_STATUS));
2997	Serge	2239	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		2240	RREG32(CP_STALLED_STAT1));
		2241	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		2242	RREG32(CP_STALLED_STAT2));
		2243	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		2244	RREG32(CP_BUSY_STAT));
		2245	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		2246	RREG32(CP_STAT));
1963	serge	2247	evergreen_mc_stop(rdev, &save);
		2248	if (evergreen_mc_wait_for_idle(rdev)) {
		2249	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		2250	}
		2251	/* Disable CP parsing/prefetching */
		2252	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT);
		2253
		2254	/* reset all the gfx blocks */
		2255	grbm_reset = (SOFT_RESET_CP \|
		2256	SOFT_RESET_CB \|
		2257	SOFT_RESET_DB \|
		2258	SOFT_RESET_PA \|
		2259	SOFT_RESET_SC \|
		2260	SOFT_RESET_SPI \|
		2261	SOFT_RESET_SH \|
		2262	SOFT_RESET_SX \|
		2263	SOFT_RESET_TC \|
		2264	SOFT_RESET_TA \|
		2265	SOFT_RESET_VC \|
		2266	SOFT_RESET_VGT);
		2267
		2268	dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
		2269	WREG32(GRBM_SOFT_RESET, grbm_reset);
		2270	(void)RREG32(GRBM_SOFT_RESET);
		2271	udelay(50);
		2272	WREG32(GRBM_SOFT_RESET, 0);
		2273	(void)RREG32(GRBM_SOFT_RESET);
		2274	/* Wait a little for things to settle down */
		2275	udelay(50);
		2276	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2277	RREG32(GRBM_STATUS));
		2278	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2279	RREG32(GRBM_STATUS_SE0));
		2280	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2281	RREG32(GRBM_STATUS_SE1));
		2282	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2283	RREG32(SRBM_STATUS));
2997	Serge	2284	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		2285	RREG32(CP_STALLED_STAT1));
		2286	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		2287	RREG32(CP_STALLED_STAT2));
		2288	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		2289	RREG32(CP_BUSY_STAT));
		2290	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		2291	RREG32(CP_STAT));
1963	serge	2292	evergreen_mc_resume(rdev, &save);
1430	serge	2293	return 0;
		2294	}
		2295
1963	serge	2296	int evergreen_asic_reset(struct radeon_device *rdev)
		2297	{
		2298	return evergreen_gpu_soft_reset(rdev);
		2299	}
		2300
		2301	/* Interrupts */
		2302
		2303	u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
		2304	{
2997	Serge	2305	if (crtc >= rdev->num_crtc)
3031	serge	2306	return 0;
2997	Serge	2307	else
		2308	return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
1963	serge	2309	}
		2310
		2311	void evergreen_disable_interrupt_state(struct radeon_device *rdev)
		2312	{
		2313	u32 tmp;
		2314
2997	Serge	2315	if (rdev->family >= CHIP_CAYMAN) {
		2316	cayman_cp_int_cntl_setup(rdev, 0,
		2317	CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		2318	cayman_cp_int_cntl_setup(rdev, 1, 0);
		2319	cayman_cp_int_cntl_setup(rdev, 2, 0);
3192	Serge	2320	tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
		2321	WREG32(CAYMAN_DMA1_CNTL, tmp);
2997	Serge	2322	} else
3031	serge	2323	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
3192	Serge	2324	tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
		2325	WREG32(DMA_CNTL, tmp);
1963	serge	2326	WREG32(GRBM_INT_CNTL, 0);
		2327	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2328	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005	serge	2329	if (rdev->num_crtc >= 4) {
3031	serge	2330	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2331	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005	serge	2332	}
		2333	if (rdev->num_crtc >= 6) {
3031	serge	2334	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2335	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	2336	}
		2337
		2338	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2339	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005	serge	2340	if (rdev->num_crtc >= 4) {
3031	serge	2341	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2342	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005	serge	2343	}
		2344	if (rdev->num_crtc >= 6) {
3031	serge	2345	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2346	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	2347	}
		2348
2997	Serge	2349	/* only one DAC on DCE6 */
		2350	if (!ASIC_IS_DCE6(rdev))
3031	serge	2351	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
1963	serge	2352	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2353
		2354	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2355	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2356	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2357	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2358	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2359	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2360	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2361	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2362	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2363	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2364	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2365	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2366
		2367	}
2005	serge	2368
		2369	int evergreen_irq_set(struct radeon_device *rdev)
		2370	{
		2371	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
2997	Serge	2372	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
2005	serge	2373	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
		2374	u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
		2375	u32 grbm_int_cntl = 0;
		2376	u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2997	Serge	2377	u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
3192	Serge	2378	u32 dma_cntl, dma_cntl1 = 0;
2005	serge	2379
		2380	if (!rdev->irq.installed) {
		2381	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		2382	return -EINVAL;
		2383	}
		2384	/* don't enable anything if the ih is disabled */
		2385	if (!rdev->ih.enabled) {
		2386	r600_disable_interrupts(rdev);
		2387	/* force the active interrupt state to all disabled */
		2388	evergreen_disable_interrupt_state(rdev);
		2389	return 0;
		2390	}
		2391
		2392	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2393	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2394	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2395	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2396	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2397	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2398
2997	Serge	2399	afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2400	afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2401	afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2402	afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2403	afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2404	afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2405
3192	Serge	2406	dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
		2407
2997	Serge	2408	if (rdev->family >= CHIP_CAYMAN) {
		2409	/* enable CP interrupts on all rings */
		2410	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		2411	DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
		2412	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		2413	}
		2414	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
		2415	DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
		2416	cp_int_cntl1 \|= TIME_STAMP_INT_ENABLE;
		2417	}
		2418	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
		2419	DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
		2420	cp_int_cntl2 \|= TIME_STAMP_INT_ENABLE;
		2421	}
		2422	} else {
		2423	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		2424	DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
3031	serge	2425	cp_int_cntl \|= RB_INT_ENABLE;
		2426	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		2427	}
2005	serge	2428	}
2997	Serge	2429
3192	Serge	2430	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
		2431	DRM_DEBUG("r600_irq_set: sw int dma\n");
		2432	dma_cntl \|= TRAP_ENABLE;
		2433	}
		2434
		2435	if (rdev->family >= CHIP_CAYMAN) {
		2436	dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
		2437	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
		2438	DRM_DEBUG("r600_irq_set: sw int dma1\n");
		2439	dma_cntl1 \|= TRAP_ENABLE;
		2440	}
		2441	}
		2442
2005	serge	2443	if (rdev->irq.crtc_vblank_int[0] \|\|
2997	Serge	2444	atomic_read(&rdev->irq.pflip[0])) {
2005	serge	2445	DRM_DEBUG("evergreen_irq_set: vblank 0\n");
		2446	crtc1 \|= VBLANK_INT_MASK;
		2447	}
		2448	if (rdev->irq.crtc_vblank_int[1] \|\|
2997	Serge	2449	atomic_read(&rdev->irq.pflip[1])) {
2005	serge	2450	DRM_DEBUG("evergreen_irq_set: vblank 1\n");
		2451	crtc2 \|= VBLANK_INT_MASK;
		2452	}
		2453	if (rdev->irq.crtc_vblank_int[2] \|\|
2997	Serge	2454	atomic_read(&rdev->irq.pflip[2])) {
2005	serge	2455	DRM_DEBUG("evergreen_irq_set: vblank 2\n");
		2456	crtc3 \|= VBLANK_INT_MASK;
		2457	}
		2458	if (rdev->irq.crtc_vblank_int[3] \|\|
2997	Serge	2459	atomic_read(&rdev->irq.pflip[3])) {
2005	serge	2460	DRM_DEBUG("evergreen_irq_set: vblank 3\n");
		2461	crtc4 \|= VBLANK_INT_MASK;
		2462	}
		2463	if (rdev->irq.crtc_vblank_int[4] \|\|
2997	Serge	2464	atomic_read(&rdev->irq.pflip[4])) {
2005	serge	2465	DRM_DEBUG("evergreen_irq_set: vblank 4\n");
		2466	crtc5 \|= VBLANK_INT_MASK;
		2467	}
		2468	if (rdev->irq.crtc_vblank_int[5] \|\|
2997	Serge	2469	atomic_read(&rdev->irq.pflip[5])) {
2005	serge	2470	DRM_DEBUG("evergreen_irq_set: vblank 5\n");
		2471	crtc6 \|= VBLANK_INT_MASK;
		2472	}
		2473	if (rdev->irq.hpd[0]) {
		2474	DRM_DEBUG("evergreen_irq_set: hpd 1\n");
		2475	hpd1 \|= DC_HPDx_INT_EN;
		2476	}
		2477	if (rdev->irq.hpd[1]) {
		2478	DRM_DEBUG("evergreen_irq_set: hpd 2\n");
		2479	hpd2 \|= DC_HPDx_INT_EN;
		2480	}
		2481	if (rdev->irq.hpd[2]) {
		2482	DRM_DEBUG("evergreen_irq_set: hpd 3\n");
		2483	hpd3 \|= DC_HPDx_INT_EN;
		2484	}
		2485	if (rdev->irq.hpd[3]) {
		2486	DRM_DEBUG("evergreen_irq_set: hpd 4\n");
		2487	hpd4 \|= DC_HPDx_INT_EN;
		2488	}
		2489	if (rdev->irq.hpd[4]) {
		2490	DRM_DEBUG("evergreen_irq_set: hpd 5\n");
		2491	hpd5 \|= DC_HPDx_INT_EN;
		2492	}
		2493	if (rdev->irq.hpd[5]) {
		2494	DRM_DEBUG("evergreen_irq_set: hpd 6\n");
		2495	hpd6 \|= DC_HPDx_INT_EN;
		2496	}
2997	Serge	2497	if (rdev->irq.afmt[0]) {
		2498	DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
		2499	afmt1 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
2005	serge	2500	}
2997	Serge	2501	if (rdev->irq.afmt[1]) {
		2502	DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
		2503	afmt2 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2504	}
		2505	if (rdev->irq.afmt[2]) {
		2506	DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
		2507	afmt3 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2508	}
		2509	if (rdev->irq.afmt[3]) {
		2510	DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
		2511	afmt4 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2512	}
		2513	if (rdev->irq.afmt[4]) {
		2514	DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
		2515	afmt5 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2516	}
		2517	if (rdev->irq.afmt[5]) {
		2518	DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
		2519	afmt6 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2520	}
2005	serge	2521
2997	Serge	2522	if (rdev->family >= CHIP_CAYMAN) {
		2523	cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
		2524	cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
		2525	cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
		2526	} else
3031	serge	2527	WREG32(CP_INT_CNTL, cp_int_cntl);
3192	Serge	2528
		2529	WREG32(DMA_CNTL, dma_cntl);
		2530
		2531	if (rdev->family >= CHIP_CAYMAN)
		2532	WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
		2533
2005	serge	2534	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		2535
		2536	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
		2537	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
		2538	if (rdev->num_crtc >= 4) {
		2539	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
		2540	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
		2541	}
		2542	if (rdev->num_crtc >= 6) {
		2543	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
		2544	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
		2545	}
		2546
		2547	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
		2548	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
		2549	if (rdev->num_crtc >= 4) {
		2550	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
		2551	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
		2552	}
		2553	if (rdev->num_crtc >= 6) {
		2554	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
		2555	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
		2556	}
		2557
		2558	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		2559	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		2560	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		2561	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		2562	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		2563	WREG32(DC_HPD6_INT_CONTROL, hpd6);
		2564
2997	Serge	2565	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
		2566	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
		2567	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
		2568	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
		2569	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
		2570	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
		2571
2005	serge	2572	return 0;
		2573	}
		2574
2997	Serge	2575	static void evergreen_irq_ack(struct radeon_device *rdev)
2005	serge	2576	{
		2577	u32 tmp;
		2578
		2579	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		2580	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		2581	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
		2582	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
		2583	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
		2584	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
		2585	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2586	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2587	if (rdev->num_crtc >= 4) {
		2588	rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2589	rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2590	}
		2591	if (rdev->num_crtc >= 6) {
		2592	rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2593	rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2594	}
		2595
2997	Serge	2596	rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2597	rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2598	rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2599	rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2600	rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2601	rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2602
2005	serge	2603	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
		2604	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2605	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
		2606	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2607	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
		2608	WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
		2609	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
		2610	WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
		2611	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
		2612	WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
		2613	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
		2614	WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
		2615
		2616	if (rdev->num_crtc >= 4) {
		2617	if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
		2618	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2619	if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
		2620	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2621	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
		2622	WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
		2623	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
		2624	WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
		2625	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
		2626	WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
		2627	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
		2628	WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
		2629	}
		2630
		2631	if (rdev->num_crtc >= 6) {
		2632	if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
		2633	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2634	if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
		2635	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2636	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
		2637	WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
		2638	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
		2639	WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
		2640	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
		2641	WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
		2642	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
		2643	WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
		2644	}
		2645
		2646	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		2647	tmp = RREG32(DC_HPD1_INT_CONTROL);
		2648	tmp \|= DC_HPDx_INT_ACK;
		2649	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2650	}
		2651	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		2652	tmp = RREG32(DC_HPD2_INT_CONTROL);
		2653	tmp \|= DC_HPDx_INT_ACK;
		2654	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2655	}
		2656	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		2657	tmp = RREG32(DC_HPD3_INT_CONTROL);
		2658	tmp \|= DC_HPDx_INT_ACK;
		2659	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2660	}
		2661	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		2662	tmp = RREG32(DC_HPD4_INT_CONTROL);
		2663	tmp \|= DC_HPDx_INT_ACK;
		2664	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2665	}
		2666	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		2667	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2668	tmp \|= DC_HPDx_INT_ACK;
		2669	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2670	}
		2671	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		2672	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2673	tmp \|= DC_HPDx_INT_ACK;
		2674	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2675	}
2997	Serge	2676	if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
		2677	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2678	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2679	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
		2680	}
		2681	if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
		2682	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2683	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2684	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
		2685	}
		2686	if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
		2687	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2688	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2689	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
		2690	}
		2691	if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
		2692	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2693	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2694	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
		2695	}
		2696	if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
		2697	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2698	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2699	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
		2700	}
		2701	if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
		2702	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2703	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2704	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
		2705	}
2005	serge	2706	}
2997	Serge	2707
		2708	static void evergreen_irq_disable(struct radeon_device *rdev)
2005	serge	2709	{
2997	Serge	2710	r600_disable_interrupts(rdev);
		2711	/* Wait and acknowledge irq */
		2712	mdelay(1);
		2713	evergreen_irq_ack(rdev);
		2714	evergreen_disable_interrupt_state(rdev);
		2715	}
		2716
		2717	void evergreen_irq_suspend(struct radeon_device *rdev)
		2718	{
		2719	evergreen_irq_disable(rdev);
		2720	r600_rlc_stop(rdev);
		2721	}
		2722
		2723	static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
		2724	{
2005	serge	2725	u32 wptr, tmp;
		2726
		2727	if (rdev->wb.enabled)
		2728	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		2729	else
		2730	wptr = RREG32(IH_RB_WPTR);
		2731
		2732	if (wptr & RB_OVERFLOW) {
		2733	/* When a ring buffer overflow happen start parsing interrupt
		2734	* from the last not overwritten vector (wptr + 16). Hopefully
		2735	* this should allow us to catchup.
		2736	*/
		2737	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
		2738	wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
		2739	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		2740	tmp = RREG32(IH_RB_CNTL);
		2741	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		2742	WREG32(IH_RB_CNTL, tmp);
		2743	}
		2744	return (wptr & rdev->ih.ptr_mask);
		2745	}
		2746
		2747	int evergreen_irq_process(struct radeon_device *rdev)
		2748	{
		2749	u32 wptr;
		2750	u32 rptr;
		2751	u32 src_id, src_data;
		2752	u32 ring_index;
		2753	bool queue_hotplug = false;
2997	Serge	2754	bool queue_hdmi = false;
2005	serge	2755
		2756	if (!rdev->ih.enabled \|\| rdev->shutdown)
		2757	return IRQ_NONE;
		2758
		2759	wptr = evergreen_get_ih_wptr(rdev);
2997	Serge	2760
		2761	restart_ih:
		2762	/* is somebody else already processing irqs? */
		2763	if (atomic_xchg(&rdev->ih.lock, 1))
		2764	return IRQ_NONE;
		2765
2005	serge	2766	rptr = rdev->ih.rptr;
		2767	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		2768
2175	serge	2769	/* Order reading of wptr vs. reading of IH ring data */
		2770	rmb();
		2771
2005	serge	2772	/* display interrupts */
		2773	evergreen_irq_ack(rdev);
		2774
		2775	while (rptr != wptr) {
		2776	/* wptr/rptr are in bytes! */
		2777	ring_index = rptr / 4;
		2778	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		2779	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		2780
		2781	switch (src_id) {
		2782	case 1: /* D1 vblank/vline */
		2783	switch (src_data) {
		2784	case 0: /* D1 vblank */
		2785	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
		2786	if (rdev->irq.crtc_vblank_int[0]) {
		2787	// drm_handle_vblank(rdev->ddev, 0);
		2788	rdev->pm.vblank_sync = true;
		2789	// wake_up(&rdev->irq.vblank_queue);
		2790	}
		2791	// if (rdev->irq.pflip[0])
		2792	// radeon_crtc_handle_flip(rdev, 0);
		2793	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		2794	DRM_DEBUG("IH: D1 vblank\n");
		2795	}
		2796	break;
		2797	case 1: /* D1 vline */
		2798	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
		2799	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		2800	DRM_DEBUG("IH: D1 vline\n");
		2801	}
		2802	break;
		2803	default:
		2804	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2805	break;
		2806	}
		2807	break;
		2808	case 2: /* D2 vblank/vline */
		2809	switch (src_data) {
		2810	case 0: /* D2 vblank */
		2811	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
		2812	if (rdev->irq.crtc_vblank_int[1]) {
		2813	// drm_handle_vblank(rdev->ddev, 1);
		2814	rdev->pm.vblank_sync = true;
		2815	// wake_up(&rdev->irq.vblank_queue);
		2816	}
		2817	// if (rdev->irq.pflip[1])
		2818	// radeon_crtc_handle_flip(rdev, 1);
		2819	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
		2820	DRM_DEBUG("IH: D2 vblank\n");
		2821	}
		2822	break;
		2823	case 1: /* D2 vline */
		2824	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
		2825	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
		2826	DRM_DEBUG("IH: D2 vline\n");
		2827	}
		2828	break;
		2829	default:
		2830	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2831	break;
		2832	}
		2833	break;
		2834	case 3: /* D3 vblank/vline */
		2835	switch (src_data) {
		2836	case 0: /* D3 vblank */
		2837	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
		2838	if (rdev->irq.crtc_vblank_int[2]) {
		2839	// drm_handle_vblank(rdev->ddev, 2);
		2840	rdev->pm.vblank_sync = true;
		2841	// wake_up(&rdev->irq.vblank_queue);
		2842	}
		2843	// if (rdev->irq.pflip[2])
		2844	// radeon_crtc_handle_flip(rdev, 2);
		2845	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
		2846	DRM_DEBUG("IH: D3 vblank\n");
		2847	}
		2848	break;
		2849	case 1: /* D3 vline */
		2850	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
		2851	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
		2852	DRM_DEBUG("IH: D3 vline\n");
		2853	}
		2854	break;
		2855	default:
		2856	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2857	break;
		2858	}
		2859	break;
		2860	case 4: /* D4 vblank/vline */
		2861	switch (src_data) {
		2862	case 0: /* D4 vblank */
		2863	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
		2864	if (rdev->irq.crtc_vblank_int[3]) {
		2865	// drm_handle_vblank(rdev->ddev, 3);
		2866	rdev->pm.vblank_sync = true;
		2867	// wake_up(&rdev->irq.vblank_queue);
		2868	}
		2869	// if (rdev->irq.pflip[3])
		2870	// radeon_crtc_handle_flip(rdev, 3);
		2871	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
		2872	DRM_DEBUG("IH: D4 vblank\n");
		2873	}
		2874	break;
		2875	case 1: /* D4 vline */
		2876	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
		2877	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
		2878	DRM_DEBUG("IH: D4 vline\n");
		2879	}
		2880	break;
		2881	default:
		2882	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2883	break;
		2884	}
		2885	break;
		2886	case 5: /* D5 vblank/vline */
		2887	switch (src_data) {
		2888	case 0: /* D5 vblank */
		2889	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
		2890	if (rdev->irq.crtc_vblank_int[4]) {
		2891	// drm_handle_vblank(rdev->ddev, 4);
		2892	rdev->pm.vblank_sync = true;
		2893	// wake_up(&rdev->irq.vblank_queue);
		2894	}
		2895	// if (rdev->irq.pflip[4])
		2896	// radeon_crtc_handle_flip(rdev, 4);
		2897	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
		2898	DRM_DEBUG("IH: D5 vblank\n");
		2899	}
		2900	break;
		2901	case 1: /* D5 vline */
		2902	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
		2903	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
		2904	DRM_DEBUG("IH: D5 vline\n");
		2905	}
		2906	break;
		2907	default:
		2908	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2909	break;
		2910	}
		2911	break;
		2912	case 6: /* D6 vblank/vline */
		2913	switch (src_data) {
		2914	case 0: /* D6 vblank */
		2915	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
		2916	if (rdev->irq.crtc_vblank_int[5]) {
		2917	// drm_handle_vblank(rdev->ddev, 5);
		2918	rdev->pm.vblank_sync = true;
		2919	// wake_up(&rdev->irq.vblank_queue);
		2920	}
		2921	// if (rdev->irq.pflip[5])
		2922	// radeon_crtc_handle_flip(rdev, 5);
		2923	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
		2924	DRM_DEBUG("IH: D6 vblank\n");
		2925	}
		2926	break;
		2927	case 1: /* D6 vline */
		2928	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
		2929	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
		2930	DRM_DEBUG("IH: D6 vline\n");
		2931	}
		2932	break;
		2933	default:
		2934	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2935	break;
		2936	}
		2937	break;
		2938	case 42: /* HPD hotplug */
		2939	switch (src_data) {
		2940	case 0:
		2941	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		2942	rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
		2943	queue_hotplug = true;
		2944	DRM_DEBUG("IH: HPD1\n");
		2945	}
		2946	break;
		2947	case 1:
		2948	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		2949	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
		2950	queue_hotplug = true;
		2951	DRM_DEBUG("IH: HPD2\n");
		2952	}
		2953	break;
		2954	case 2:
		2955	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		2956	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
		2957	queue_hotplug = true;
		2958	DRM_DEBUG("IH: HPD3\n");
		2959	}
		2960	break;
		2961	case 3:
		2962	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		2963	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
		2964	queue_hotplug = true;
		2965	DRM_DEBUG("IH: HPD4\n");
		2966	}
		2967	break;
		2968	case 4:
		2969	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		2970	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
		2971	queue_hotplug = true;
		2972	DRM_DEBUG("IH: HPD5\n");
		2973	}
		2974	break;
		2975	case 5:
		2976	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		2977	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
		2978	queue_hotplug = true;
		2979	DRM_DEBUG("IH: HPD6\n");
		2980	}
		2981	break;
		2982	default:
		2983	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2984	break;
		2985	}
		2986	break;
2997	Serge	2987	case 44: /* hdmi */
		2988	switch (src_data) {
		2989	case 0:
		2990	if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
		2991	rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
		2992	queue_hdmi = true;
		2993	DRM_DEBUG("IH: HDMI0\n");
		2994	}
		2995	break;
		2996	case 1:
		2997	if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
		2998	rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
		2999	queue_hdmi = true;
		3000	DRM_DEBUG("IH: HDMI1\n");
		3001	}
		3002	break;
		3003	case 2:
		3004	if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
		3005	rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
		3006	queue_hdmi = true;
		3007	DRM_DEBUG("IH: HDMI2\n");
		3008	}
		3009	break;
		3010	case 3:
		3011	if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
		3012	rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
		3013	queue_hdmi = true;
		3014	DRM_DEBUG("IH: HDMI3\n");
		3015	}
		3016	break;
		3017	case 4:
		3018	if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
		3019	rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
		3020	queue_hdmi = true;
		3021	DRM_DEBUG("IH: HDMI4\n");
		3022	}
		3023	break;
		3024	case 5:
		3025	if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
		3026	rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
		3027	queue_hdmi = true;
		3028	DRM_DEBUG("IH: HDMI5\n");
		3029	}
		3030	break;
		3031	default:
		3032	DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
		3033	break;
		3034	}
		3035	break;
3192	Serge	3036	case 146:
		3037	case 147:
		3038	dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
		3039	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
		3040	RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
		3041	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
		3042	RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
		3043	/* reset addr and status */
		3044	WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
		3045	break;
2005	serge	3046	case 176: /* CP_INT in ring buffer */
		3047	case 177: /* CP_INT in IB1 */
		3048	case 178: /* CP_INT in IB2 */
		3049	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2997	Serge	3050	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2005	serge	3051	break;
		3052	case 181: /* CP EOP event */
		3053	DRM_DEBUG("IH: CP EOP\n");
2997	Serge	3054	if (rdev->family >= CHIP_CAYMAN) {
		3055	switch (src_data) {
		3056	case 0:
		3057	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
		3058	break;
		3059	case 1:
		3060	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		3061	break;
		3062	case 2:
		3063	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		3064	break;
		3065	}
		3066	} else
		3067	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2005	serge	3068	break;
3192	Serge	3069	case 224: /* DMA trap event */
		3070	DRM_DEBUG("IH: DMA trap\n");
		3071	radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
		3072	break;
2005	serge	3073	case 233: /* GUI IDLE */
		3074	DRM_DEBUG("IH: GUI idle\n");
		3075	break;
3192	Serge	3076	case 244: /* DMA trap event */
		3077	if (rdev->family >= CHIP_CAYMAN) {
		3078	DRM_DEBUG("IH: DMA1 trap\n");
		3079	radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
		3080	}
		3081	break;
2005	serge	3082	default:
		3083	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3084	break;
		3085	}
		3086
		3087	/* wptr/rptr are in bytes! */
		3088	rptr += 16;
		3089	rptr &= rdev->ih.ptr_mask;
		3090	}
2997	Serge	3091	rdev->ih.rptr = rptr;
		3092	WREG32(IH_RB_RPTR, rdev->ih.rptr);
		3093	atomic_set(&rdev->ih.lock, 0);
		3094
2005	serge	3095	/* make sure wptr hasn't changed while processing */
		3096	wptr = evergreen_get_ih_wptr(rdev);
2997	Serge	3097	if (wptr != rptr)
2005	serge	3098	goto restart_ih;
2997	Serge	3099
2005	serge	3100	return IRQ_HANDLED;
		3101	}
		3102
3192	Serge	3103	/**
		3104	* evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
		3105	*
		3106	* @rdev: radeon_device pointer
		3107	* @fence: radeon fence object
		3108	*
		3109	* Add a DMA fence packet to the ring to write
		3110	* the fence seq number and DMA trap packet to generate
		3111	* an interrupt if needed (evergreen-SI).
		3112	*/
		3113	void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
		3114	struct radeon_fence *fence)
		3115	{
		3116	struct radeon_ring *ring = &rdev->ring[fence->ring];
		3117	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		3118	/* write the fence */
		3119	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0));
		3120	radeon_ring_write(ring, addr & 0xfffffffc);
		3121	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
		3122	radeon_ring_write(ring, fence->seq);
		3123	/* generate an interrupt */
		3124	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0));
		3125	/* flush HDP */
		3126	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
		3127	radeon_ring_write(ring, (0xf << 16) \| HDP_MEM_COHERENCY_FLUSH_CNTL);
		3128	radeon_ring_write(ring, 1);
		3129	}
		3130
		3131	/**
		3132	* evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
		3133	*
		3134	* @rdev: radeon_device pointer
		3135	* @ib: IB object to schedule
		3136	*
		3137	* Schedule an IB in the DMA ring (evergreen).
		3138	*/
		3139	void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
		3140	struct radeon_ib *ib)
		3141	{
		3142	struct radeon_ring *ring = &rdev->ring[ib->ring];
		3143
		3144	if (rdev->wb.enabled) {
		3145	u32 next_rptr = ring->wptr + 4;
		3146	while ((next_rptr & 7) != 5)
		3147	next_rptr++;
		3148	next_rptr += 3;
		3149	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
		3150	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		3151	radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
		3152	radeon_ring_write(ring, next_rptr);
		3153	}
		3154
		3155	/* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
		3156	* Pad as necessary with NOPs.
		3157	*/
		3158	while ((ring->wptr & 7) != 5)
		3159	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
		3160	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0));
		3161	radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
		3162	radeon_ring_write(ring, (ib->length_dw << 12) \| (upper_32_bits(ib->gpu_addr) & 0xFF));
		3163
		3164	}
		3165
		3166	/**
		3167	* evergreen_copy_dma - copy pages using the DMA engine
		3168	*
		3169	* @rdev: radeon_device pointer
		3170	* @src_offset: src GPU address
		3171	* @dst_offset: dst GPU address
		3172	* @num_gpu_pages: number of GPU pages to xfer
		3173	* @fence: radeon fence object
		3174	*
		3175	* Copy GPU paging using the DMA engine (evergreen-cayman).
		3176	* Used by the radeon ttm implementation to move pages if
		3177	* registered as the asic copy callback.
		3178	*/
		3179	int evergreen_copy_dma(struct radeon_device *rdev,
		3180	uint64_t src_offset, uint64_t dst_offset,
		3181	unsigned num_gpu_pages,
		3182	struct radeon_fence **fence)
		3183	{
		3184	struct radeon_semaphore *sem = NULL;
		3185	int ring_index = rdev->asic->copy.dma_ring_index;
		3186	struct radeon_ring *ring = &rdev->ring[ring_index];
		3187	u32 size_in_dw, cur_size_in_dw;
		3188	int i, num_loops;
		3189	int r = 0;
		3190
		3191	r = radeon_semaphore_create(rdev, &sem);
		3192	if (r) {
		3193	DRM_ERROR("radeon: moving bo (%d).\n", r);
		3194	return r;
		3195	}
		3196
		3197	size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
		3198	num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
		3199	r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
		3200	if (r) {
		3201	DRM_ERROR("radeon: moving bo (%d).\n", r);
		3202	radeon_semaphore_free(rdev, &sem, NULL);
		3203	return r;
		3204	}
		3205
		3206	if (radeon_fence_need_sync(*fence, ring->idx)) {
		3207	radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
		3208	ring->idx);
		3209	radeon_fence_note_sync(*fence, ring->idx);
		3210	} else {
		3211	radeon_semaphore_free(rdev, &sem, NULL);
		3212	}
		3213
		3214	for (i = 0; i < num_loops; i++) {
		3215	cur_size_in_dw = size_in_dw;
		3216	if (cur_size_in_dw > 0xFFFFF)
		3217	cur_size_in_dw = 0xFFFFF;
		3218	size_in_dw -= cur_size_in_dw;
		3219	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
		3220	radeon_ring_write(ring, dst_offset & 0xfffffffc);
		3221	radeon_ring_write(ring, src_offset & 0xfffffffc);
		3222	radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
		3223	radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
		3224	src_offset += cur_size_in_dw * 4;
		3225	dst_offset += cur_size_in_dw * 4;
		3226	}
		3227
		3228	r = radeon_fence_emit(rdev, fence, ring->idx);
		3229	if (r) {
		3230	radeon_ring_unlock_undo(rdev, ring);
		3231	return r;
		3232	}
		3233
		3234	radeon_ring_unlock_commit(rdev, ring);
		3235	radeon_semaphore_free(rdev, &sem, *fence);
		3236
		3237	return r;
		3238	}
		3239
1430	serge	3240	static int evergreen_startup(struct radeon_device *rdev)
		3241	{
2997	Serge	3242	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1430	serge	3243	int r;
		3244
1990	serge	3245	/* enable pcie gen2 link */
3031	serge	3246	evergreen_pcie_gen2_enable(rdev);
1990	serge	3247
		3248	if (ASIC_IS_DCE5(rdev)) {
		3249	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw \|\| !rdev->mc_fw) {
		3250	r = ni_init_microcode(rdev);
		3251	if (r) {
		3252	DRM_ERROR("Failed to load firmware!\n");
		3253	return r;
		3254	}
		3255	}
		3256	r = ni_mc_load_microcode(rdev);
		3257	if (r) {
		3258	DRM_ERROR("Failed to load MC firmware!\n");
		3259	return r;
		3260	}
		3261	} else {
3031	serge	3262	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		3263	r = r600_init_microcode(rdev);
		3264	if (r) {
		3265	DRM_ERROR("Failed to load firmware!\n");
		3266	return r;
		3267	}
1430	serge	3268	}
		3269	}
1963	serge	3270
2997	Serge	3271	r = r600_vram_scratch_init(rdev);
		3272	if (r)
		3273	return r;
		3274
1430	serge	3275	evergreen_mc_program(rdev);
		3276	if (rdev->flags & RADEON_IS_AGP) {
1963	serge	3277	evergreen_agp_enable(rdev);
1430	serge	3278	} else {
		3279	r = evergreen_pcie_gart_enable(rdev);
		3280	if (r)
		3281	return r;
		3282	}
		3283	evergreen_gpu_init(rdev);
2005	serge	3284
1963	serge	3285	r = evergreen_blit_init(rdev);
		3286	if (r) {
2997	Serge	3287	// r600_blit_fini(rdev);
		3288	rdev->asic->copy.copy = NULL;
1963	serge	3289	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		3290	}
1430	serge	3291
1963	serge	3292	/* allocate wb buffer */
		3293	r = radeon_wb_init(rdev);
		3294	if (r)
		3295	return r;
		3296
3192	Serge	3297	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
		3298	if (r) {
		3299	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		3300	return r;
		3301	}
		3302
		3303	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
		3304	if (r) {
		3305	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		3306	return r;
		3307	}
		3308
1963	serge	3309	/* Enable IRQ */
2005	serge	3310	r = r600_irq_init(rdev);
		3311	if (r) {
		3312	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		3313	// radeon_irq_kms_fini(rdev);
		3314	return r;
		3315	}
		3316	evergreen_irq_set(rdev);
1963	serge	3317
2997	Serge	3318	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
		3319	R600_CP_RB_RPTR, R600_CP_RB_WPTR,
		3320	0, 0xfffff, RADEON_CP_PACKET2);
1430	serge	3321	if (r)
		3322	return r;
3192	Serge	3323
		3324	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		3325	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
		3326	DMA_RB_RPTR, DMA_RB_WPTR,
		3327	2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
		3328	if (r)
		3329	return r;
		3330
1430	serge	3331	r = evergreen_cp_load_microcode(rdev);
		3332	if (r)
		3333	return r;
1963	serge	3334	r = evergreen_cp_resume(rdev);
1430	serge	3335	if (r)
		3336	return r;
3192	Serge	3337	r = r600_dma_resume(rdev);
		3338	if (r)
		3339	return r;
1963	serge	3340
3192	Serge	3341	r = radeon_ib_pool_init(rdev);
		3342	if (r) {
		3343	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		3344	return r;
		3345	}
		3346
1430	serge	3347	return 0;
		3348	}
		3349
		3350
		3351
2997	Serge	3352	#if 0
1430	serge	3353
2005	serge	3354	int evergreen_copy_blit(struct radeon_device *rdev,
		3355	uint64_t src_offset, uint64_t dst_offset,
		3356	unsigned num_pages, struct radeon_fence *fence)
		3357	{
		3358	int r;
1430	serge	3359
2005	serge	3360	mutex_lock(&rdev->r600_blit.mutex);
		3361	rdev->r600_blit.vb_ib = NULL;
		3362	r = evergreen_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
		3363	if (r) {
		3364	if (rdev->r600_blit.vb_ib)
		3365	radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
		3366	mutex_unlock(&rdev->r600_blit.mutex);
		3367	return r;
		3368	}
		3369	evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
		3370	evergreen_blit_done_copy(rdev, fence);
		3371	mutex_unlock(&rdev->r600_blit.mutex);
		3372	return 0;
		3373	}
2997	Serge	3374	#endif
1430	serge	3375
		3376	/* Plan is to move initialization in that function and use
		3377	* helper function so that radeon_device_init pretty much
		3378	* do nothing more than calling asic specific function. This
		3379	* should also allow to remove a bunch of callback function
		3380	* like vram_info.
		3381	*/
		3382	int evergreen_init(struct radeon_device *rdev)
		3383	{
		3384	int r;
		3385
		3386	/* Read BIOS */
		3387	if (!radeon_get_bios(rdev)) {
		3388	if (ASIC_IS_AVIVO(rdev))
		3389	return -EINVAL;
		3390	}
		3391	/* Must be an ATOMBIOS */
		3392	if (!rdev->is_atom_bios) {
1986	serge	3393	dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
1430	serge	3394	return -EINVAL;
		3395	}
		3396	r = radeon_atombios_init(rdev);
		3397	if (r)
		3398	return r;
1986	serge	3399	/* reset the asic, the gfx blocks are often in a bad state
		3400	* after the driver is unloaded or after a resume
		3401	*/
		3402	if (radeon_asic_reset(rdev))
		3403	dev_warn(rdev->dev, "GPU reset failed !\n");
1430	serge	3404	/* Post card if necessary */
1986	serge	3405	if (!radeon_card_posted(rdev)) {
1430	serge	3406	if (!rdev->bios) {
		3407	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		3408	return -EINVAL;
		3409	}
		3410	DRM_INFO("GPU not posted. posting now...\n");
		3411	atom_asic_init(rdev->mode_info.atom_context);
		3412	}
		3413	/* Initialize scratch registers */
		3414	r600_scratch_init(rdev);
		3415	/* Initialize surface registers */
		3416	radeon_surface_init(rdev);
		3417	/* Initialize clocks */
		3418	radeon_get_clock_info(rdev->ddev);
		3419	/* Fence driver */
2005	serge	3420	r = radeon_fence_driver_init(rdev);
		3421	if (r)
		3422	return r;
3031	serge	3423	/* initialize AGP */
1430	serge	3424	if (rdev->flags & RADEON_IS_AGP) {
		3425	r = radeon_agp_init(rdev);
		3426	if (r)
		3427	radeon_agp_disable(rdev);
		3428	}
		3429	/* initialize memory controller */
		3430	r = evergreen_mc_init(rdev);
		3431	if (r)
		3432	return r;
		3433	/* Memory manager */
		3434	r = radeon_bo_init(rdev);
		3435	if (r)
		3436	return r;
1963	serge	3437
2005	serge	3438	r = radeon_irq_kms_init(rdev);
		3439	if (r)
		3440	return r;
1430	serge	3441
2997	Serge	3442	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
		3443	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1430	serge	3444
3192	Serge	3445	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
		3446	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
		3447
2005	serge	3448	rdev->ih.ring_obj = NULL;
		3449	r600_ih_ring_init(rdev, 64 * 1024);
1430	serge	3450
		3451	r = r600_pcie_gart_init(rdev);
		3452	if (r)
		3453	return r;
1963	serge	3454
		3455	rdev->accel_working = true;
1430	serge	3456	r = evergreen_startup(rdev);
		3457	if (r) {
1963	serge	3458	dev_err(rdev->dev, "disabling GPU acceleration\n");
1430	serge	3459	rdev->accel_working = false;
		3460	}
2997	Serge	3461
		3462	/* Don't start up if the MC ucode is missing on BTC parts.
		3463	* The default clocks and voltages before the MC ucode
		3464	* is loaded are not suffient for advanced operations.
		3465	*/
		3466	if (ASIC_IS_DCE5(rdev)) {
		3467	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
		3468	DRM_ERROR("radeon: MC ucode required for NI+.\n");
		3469	return -EINVAL;
2005	serge	3470	}
1430	serge	3471	}
2997	Serge	3472
1430	serge	3473	return 0;
		3474	}
		3475
1986	serge	3476
2997	Serge	3477	void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
1430	serge	3478	{
2997	Serge	3479	u32 link_width_cntl, speed_cntl, mask;
		3480	int ret;
1986	serge	3481
		3482	if (radeon_pcie_gen2 == 0)
		3483	return;
		3484
		3485	if (rdev->flags & RADEON_IS_IGP)
		3486	return;
		3487
		3488	if (!(rdev->flags & RADEON_IS_PCIE))
		3489	return;
		3490
		3491	/* x2 cards have a special sequence */
		3492	if (ASIC_IS_X2(rdev))
		3493	return;
		3494
2997	Serge	3495	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
		3496	if (ret != 0)
		3497	return;
		3498
		3499	if (!(mask & DRM_PCIE_SPEED_50))
		3500	return;
		3501
1986	serge	3502	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
2997	Serge	3503	if (speed_cntl & LC_CURRENT_DATA_RATE) {
		3504	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
		3505	return;
		3506	}
		3507
		3508	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
		3509
1986	serge	3510	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) \|\|
		3511	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		3512
		3513	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3514	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3515	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3516
		3517	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3518	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		3519	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3520
		3521	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3522	speed_cntl \|= LC_CLR_FAILED_SPD_CHANGE_CNT;
		3523	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3524
		3525	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3526	speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
		3527	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3528
		3529	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3530	speed_cntl \|= LC_GEN2_EN_STRAP;
		3531	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3532
		3533	} else {
		3534	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3535	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		3536	if (1)
		3537	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3538	else
		3539	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3540	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3541	}
1430	serge	3542	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/evergreen.c @ 5179 – Rev 3192