WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/evergreen.c

Rev	Author	Line No.	Line
1430	serge	1	/*
		2	* Copyright 2010 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	//#include
1963	serge	26	#include
2997	Serge	27	#include
1430	serge	28	#include "radeon.h"
1963	serge	29	#include "radeon_asic.h"
2997	Serge	30	#include
1963	serge	31	#include "evergreend.h"
1430	serge	32	#include "atom.h"
		33	#include "avivod.h"
		34	#include "evergreen_reg.h"
1986	serge	35	#include "evergreen_blit_shaders.h"
1430	serge	36
1963	serge	37	#define EVERGREEN_PFP_UCODE_SIZE 1120
		38	#define EVERGREEN_PM4_UCODE_SIZE 1376
		39
2997	Serge	40	static const u32 crtc_offsets[6] =
		41	{
		42	EVERGREEN_CRTC0_REGISTER_OFFSET,
		43	EVERGREEN_CRTC1_REGISTER_OFFSET,
		44	EVERGREEN_CRTC2_REGISTER_OFFSET,
		45	EVERGREEN_CRTC3_REGISTER_OFFSET,
		46	EVERGREEN_CRTC4_REGISTER_OFFSET,
		47	EVERGREEN_CRTC5_REGISTER_OFFSET
		48	};
		49
1430	serge	50	static void evergreen_gpu_init(struct radeon_device *rdev);
		51	void evergreen_fini(struct radeon_device *rdev);
2997	Serge	52	void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
		53	extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
		54	int ring, u32 cp_int_cntl);
1430	serge	55
2997	Serge	56	void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
		57	unsigned bankh, unsigned mtaspect,
		58	unsigned *tile_split)
		59	{
		60	*bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
		61	*bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
		62	*mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
		63	*tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
		64	switch (*bankw) {
		65	default:
		66	case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
		67	case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
		68	case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
		69	case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
		70	}
		71	switch (*bankh) {
		72	default:
		73	case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
		74	case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
		75	case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
		76	case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
		77	}
		78	switch (*mtaspect) {
		79	default:
		80	case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
		81	case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
		82	case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
		83	case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
		84	}
		85	}
1990	serge	86
2997	Serge	87	void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
		88	{
		89	u16 ctl, v;
		90	int err;
1990	serge	91
2997	Serge	92	err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
		93	if (err)
		94	return;
		95
		96	v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
		97
		98	/* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
		99	* to avoid hangs or perfomance issues
		100	*/
		101	if ((v == 0) \|\| (v == 6) \|\| (v == 7)) {
		102	ctl &= ~PCI_EXP_DEVCTL_READRQ;
		103	ctl \|= (2 << 12);
		104	pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
		105	}
		106	}
		107
		108	/**
		109	* dce4_wait_for_vblank - vblank wait asic callback.
		110	*
		111	* @rdev: radeon_device pointer
		112	* @crtc: crtc to wait for vblank on
		113	*
		114	* Wait for vblank on the requested crtc (evergreen+).
		115	*/
		116	void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
		117	{
		118	int i;
		119
		120	if (crtc >= rdev->num_crtc)
		121	return;
		122
		123	if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN) {
		124	for (i = 0; i < rdev->usec_timeout; i++) {
		125	if (!(RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK))
		126	break;
		127	udelay(1);
		128	}
		129	for (i = 0; i < rdev->usec_timeout; i++) {
		130	if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
		131	break;
		132	udelay(1);
		133	}
		134	}
		135	}
		136
		137
		138	/**
		139	* evergreen_page_flip - pageflip callback.
		140	*
		141	* @rdev: radeon_device pointer
		142	* @crtc_id: crtc to cleanup pageflip on
		143	* @crtc_base: new address of the crtc (GPU MC address)
		144	*
		145	* Does the actual pageflip (evergreen+).
		146	* During vblank we take the crtc lock and wait for the update_pending
		147	* bit to go high, when it does, we release the lock, and allow the
		148	* double buffered update to take place.
		149	* Returns the current update pending status.
		150	*/
1990	serge	151	u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
		152	{
		153	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
		154	u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
2997	Serge	155	int i;
1990	serge	156
		157	/* Lock the graphics update lock */
		158	tmp \|= EVERGREEN_GRPH_UPDATE_LOCK;
		159	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		160
		161	/* update the scanout addresses */
		162	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		163	upper_32_bits(crtc_base));
		164	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		165	(u32)crtc_base);
		166
		167	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		168	upper_32_bits(crtc_base));
		169	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		170	(u32)crtc_base);
		171
		172	/* Wait for update_pending to go high. */
2997	Serge	173	for (i = 0; i < rdev->usec_timeout; i++) {
		174	if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
		175	break;
		176	udelay(1);
		177	}
1990	serge	178	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
		179
		180	/* Unlock the lock, so double-buffering can take place inside vblank */
		181	tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
		182	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		183
		184	/* Return current update_pending status: */
		185	return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
		186	}
		187
		188	/* get temperature in millidegrees */
		189	int evergreen_get_temp(struct radeon_device *rdev)
		190	{
		191	u32 temp, toffset;
		192	int actual_temp = 0;
		193
		194	if (rdev->family == CHIP_JUNIPER) {
		195	toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
		196	TOFFSET_SHIFT;
		197	temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
		198	TS0_ADC_DOUT_SHIFT;
		199
		200	if (toffset & 0x100)
		201	actual_temp = temp / 2 - (0x200 - toffset);
		202	else
		203	actual_temp = temp / 2 + toffset;
		204
		205	actual_temp = actual_temp * 1000;
		206
		207	} else {
		208	temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
		209	ASIC_T_SHIFT;
		210
		211	if (temp & 0x400)
		212	actual_temp = -256;
		213	else if (temp & 0x200)
		214	actual_temp = 255;
		215	else if (temp & 0x100) {
		216	actual_temp = temp & 0x1ff;
		217	actual_temp \|= ~0x1ff;
		218	} else
		219	actual_temp = temp & 0xff;
		220
		221	actual_temp = (actual_temp * 1000) / 2;
		222	}
		223
		224	return actual_temp;
		225	}
		226
		227	int sumo_get_temp(struct radeon_device *rdev)
		228	{
		229	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
		230	int actual_temp = temp - 49;
		231
		232	return actual_temp * 1000;
		233	}
		234
2997	Serge	235	/**
		236	* sumo_pm_init_profile - Initialize power profiles callback.
		237	*
		238	* @rdev: radeon_device pointer
		239	*
		240	* Initialize the power states used in profile mode
		241	* (sumo, trinity, SI).
		242	* Used for profile mode only.
		243	*/
		244	void sumo_pm_init_profile(struct radeon_device *rdev)
		245	{
		246	int idx;
		247
		248	/* default */
		249	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		250	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		251	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		252	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
		253
		254	/* low,mid sh/mh */
		255	if (rdev->flags & RADEON_IS_MOBILITY)
		256	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
		257	else
		258	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		259
		260	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
		261	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
		262	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		263	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		264
		265	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
		266	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
		267	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		268	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		269
		270	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
		271	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
		272	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		273	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
		274
		275	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
		276	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
		277	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		278	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
		279
		280	/* high sh/mh */
		281	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		282	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
		283	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
		284	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		285	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
		286	rdev->pm.power_state[idx].num_clock_modes - 1;
		287
		288	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
		289	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
		290	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		291	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
		292	rdev->pm.power_state[idx].num_clock_modes - 1;
		293	}
		294
		295	/**
		296	* evergreen_pm_misc - set additional pm hw parameters callback.
		297	*
		298	* @rdev: radeon_device pointer
		299	*
		300	* Set non-clock parameters associated with a power state
		301	* (voltage, etc.) (evergreen+).
		302	*/
1990	serge	303	void evergreen_pm_misc(struct radeon_device *rdev)
		304	{
		305	int req_ps_idx = rdev->pm.requested_power_state_index;
		306	int req_cm_idx = rdev->pm.requested_clock_mode_index;
		307	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
		308	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
		309
		310	if (voltage->type == VOLTAGE_SW) {
		311	/* 0xff01 is a flag rather then an actual voltage */
		312	if (voltage->voltage == 0xff01)
		313	return;
		314	if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
		315	radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
		316	rdev->pm.current_vddc = voltage->voltage;
		317	DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
		318	}
		319	/* 0xff01 is a flag rather then an actual voltage */
		320	if (voltage->vddci == 0xff01)
		321	return;
		322	if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
		323	radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
		324	rdev->pm.current_vddci = voltage->vddci;
		325	DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
		326	}
		327	}
		328	}
		329
2997	Serge	330	/**
		331	* evergreen_pm_prepare - pre-power state change callback.
		332	*
		333	* @rdev: radeon_device pointer
		334	*
		335	* Prepare for a power state change (evergreen+).
		336	*/
1990	serge	337	void evergreen_pm_prepare(struct radeon_device *rdev)
		338	{
		339	struct drm_device *ddev = rdev->ddev;
		340	struct drm_crtc *crtc;
		341	struct radeon_crtc *radeon_crtc;
		342	u32 tmp;
		343
		344	/* disable any active CRTCs */
		345	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		346	radeon_crtc = to_radeon_crtc(crtc);
		347	if (radeon_crtc->enabled) {
		348	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		349	tmp \|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		350	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		351	}
		352	}
		353	}
		354
2997	Serge	355	/**
		356	* evergreen_pm_finish - post-power state change callback.
		357	*
		358	* @rdev: radeon_device pointer
		359	*
		360	* Clean up after a power state change (evergreen+).
		361	*/
1990	serge	362	void evergreen_pm_finish(struct radeon_device *rdev)
		363	{
		364	struct drm_device *ddev = rdev->ddev;
		365	struct drm_crtc *crtc;
		366	struct radeon_crtc *radeon_crtc;
		367	u32 tmp;
		368
		369	/* enable any active CRTCs */
		370	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		371	radeon_crtc = to_radeon_crtc(crtc);
		372	if (radeon_crtc->enabled) {
		373	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		374	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		375	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		376	}
		377	}
		378	}
		379
2997	Serge	380	/**
		381	* evergreen_hpd_sense - hpd sense callback.
		382	*
		383	* @rdev: radeon_device pointer
		384	* @hpd: hpd (hotplug detect) pin
		385	*
		386	* Checks if a digital monitor is connected (evergreen+).
		387	* Returns true if connected, false if not connected.
		388	*/
1430	serge	389	bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		390	{
		391	bool connected = false;
1963	serge	392
		393	switch (hpd) {
		394	case RADEON_HPD_1:
		395	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		396	connected = true;
		397	break;
		398	case RADEON_HPD_2:
		399	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		400	connected = true;
		401	break;
		402	case RADEON_HPD_3:
		403	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		404	connected = true;
		405	break;
		406	case RADEON_HPD_4:
		407	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		408	connected = true;
		409	break;
		410	case RADEON_HPD_5:
		411	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		412	connected = true;
		413	break;
		414	case RADEON_HPD_6:
		415	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		416	connected = true;
		417	break;
		418	default:
		419	break;
		420	}
		421
1430	serge	422	return connected;
		423	}
		424
2997	Serge	425	/**
		426	* evergreen_hpd_set_polarity - hpd set polarity callback.
		427	*
		428	* @rdev: radeon_device pointer
		429	* @hpd: hpd (hotplug detect) pin
		430	*
		431	* Set the polarity of the hpd pin (evergreen+).
		432	*/
1430	serge	433	void evergreen_hpd_set_polarity(struct radeon_device *rdev,
		434	enum radeon_hpd_id hpd)
		435	{
1963	serge	436	u32 tmp;
		437	bool connected = evergreen_hpd_sense(rdev, hpd);
		438
		439	switch (hpd) {
		440	case RADEON_HPD_1:
		441	tmp = RREG32(DC_HPD1_INT_CONTROL);
		442	if (connected)
		443	tmp &= ~DC_HPDx_INT_POLARITY;
		444	else
		445	tmp \|= DC_HPDx_INT_POLARITY;
		446	WREG32(DC_HPD1_INT_CONTROL, tmp);
		447	break;
		448	case RADEON_HPD_2:
		449	tmp = RREG32(DC_HPD2_INT_CONTROL);
		450	if (connected)
		451	tmp &= ~DC_HPDx_INT_POLARITY;
		452	else
		453	tmp \|= DC_HPDx_INT_POLARITY;
		454	WREG32(DC_HPD2_INT_CONTROL, tmp);
		455	break;
		456	case RADEON_HPD_3:
		457	tmp = RREG32(DC_HPD3_INT_CONTROL);
		458	if (connected)
		459	tmp &= ~DC_HPDx_INT_POLARITY;
		460	else
		461	tmp \|= DC_HPDx_INT_POLARITY;
		462	WREG32(DC_HPD3_INT_CONTROL, tmp);
		463	break;
		464	case RADEON_HPD_4:
		465	tmp = RREG32(DC_HPD4_INT_CONTROL);
		466	if (connected)
		467	tmp &= ~DC_HPDx_INT_POLARITY;
		468	else
		469	tmp \|= DC_HPDx_INT_POLARITY;
		470	WREG32(DC_HPD4_INT_CONTROL, tmp);
		471	break;
		472	case RADEON_HPD_5:
		473	tmp = RREG32(DC_HPD5_INT_CONTROL);
		474	if (connected)
		475	tmp &= ~DC_HPDx_INT_POLARITY;
		476	else
		477	tmp \|= DC_HPDx_INT_POLARITY;
		478	WREG32(DC_HPD5_INT_CONTROL, tmp);
		479	break;
		480	case RADEON_HPD_6:
		481	tmp = RREG32(DC_HPD6_INT_CONTROL);
		482	if (connected)
		483	tmp &= ~DC_HPDx_INT_POLARITY;
		484	else
		485	tmp \|= DC_HPDx_INT_POLARITY;
		486	WREG32(DC_HPD6_INT_CONTROL, tmp);
		487	break;
		488	default:
		489	break;
		490	}
1430	serge	491	}
		492
2997	Serge	493	/**
		494	* evergreen_hpd_init - hpd setup callback.
		495	*
		496	* @rdev: radeon_device pointer
		497	*
		498	* Setup the hpd pins used by the card (evergreen+).
		499	* Enable the pin, set the polarity, and enable the hpd interrupts.
		500	*/
1430	serge	501	void evergreen_hpd_init(struct radeon_device *rdev)
		502	{
1963	serge	503	struct drm_device *dev = rdev->ddev;
		504	struct drm_connector *connector;
2997	Serge	505	unsigned enabled = 0;
1963	serge	506	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \|
		507	DC_HPDx_RX_INT_TIMER(0xfa) \| DC_HPDx_EN;
		508
		509	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		510	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		511	switch (radeon_connector->hpd.hpd) {
		512	case RADEON_HPD_1:
		513	WREG32(DC_HPD1_CONTROL, tmp);
		514	break;
		515	case RADEON_HPD_2:
		516	WREG32(DC_HPD2_CONTROL, tmp);
		517	break;
		518	case RADEON_HPD_3:
		519	WREG32(DC_HPD3_CONTROL, tmp);
		520	break;
		521	case RADEON_HPD_4:
		522	WREG32(DC_HPD4_CONTROL, tmp);
		523	break;
		524	case RADEON_HPD_5:
		525	WREG32(DC_HPD5_CONTROL, tmp);
		526	break;
		527	case RADEON_HPD_6:
		528	WREG32(DC_HPD6_CONTROL, tmp);
		529	break;
		530	default:
		531	break;
		532	}
2997	Serge	533	radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
		534	enabled \|= 1 << radeon_connector->hpd.hpd;
1963	serge	535	}
2997	Serge	536	// radeon_irq_kms_enable_hpd(rdev, enabled);
1430	serge	537	}
		538
2997	Serge	539	/**
		540	* evergreen_hpd_fini - hpd tear down callback.
		541	*
		542	* @rdev: radeon_device pointer
		543	*
		544	* Tear down the hpd pins used by the card (evergreen+).
		545	* Disable the hpd interrupts.
		546	*/
1963	serge	547	void evergreen_hpd_fini(struct radeon_device *rdev)
		548	{
		549	struct drm_device *dev = rdev->ddev;
		550	struct drm_connector *connector;
2997	Serge	551	unsigned disabled = 0;
1430	serge	552
1963	serge	553	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		554	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		555	switch (radeon_connector->hpd.hpd) {
		556	case RADEON_HPD_1:
		557	WREG32(DC_HPD1_CONTROL, 0);
		558	break;
		559	case RADEON_HPD_2:
		560	WREG32(DC_HPD2_CONTROL, 0);
		561	break;
		562	case RADEON_HPD_3:
		563	WREG32(DC_HPD3_CONTROL, 0);
		564	break;
		565	case RADEON_HPD_4:
		566	WREG32(DC_HPD4_CONTROL, 0);
		567	break;
		568	case RADEON_HPD_5:
		569	WREG32(DC_HPD5_CONTROL, 0);
		570	break;
		571	case RADEON_HPD_6:
		572	WREG32(DC_HPD6_CONTROL, 0);
		573	break;
		574	default:
		575	break;
		576	}
2997	Serge	577	disabled \|= 1 << radeon_connector->hpd.hpd;
1963	serge	578	}
2997	Serge	579	// radeon_irq_kms_disable_hpd(rdev, disabled);
1430	serge	580	}
		581
1986	serge	582	/* watermark setup */
1963	serge	583
1986	serge	584	static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
		585	struct radeon_crtc *radeon_crtc,
		586	struct drm_display_mode *mode,
		587	struct drm_display_mode *other_mode)
		588	{
		589	u32 tmp;
		590	/*
		591	* Line Buffer Setup
		592	* There are 3 line buffers, each one shared by 2 display controllers.
		593	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
		594	* the display controllers. The paritioning is done via one of four
		595	* preset allocations specified in bits 2:0:
		596	* first display controller
		597	* 0 - first half of lb (3840 * 2)
		598	* 1 - first 3/4 of lb (5760 * 2)
		599	* 2 - whole lb (7680 * 2), other crtc must be disabled
		600	* 3 - first 1/4 of lb (1920 * 2)
		601	* second display controller
		602	* 4 - second half of lb (3840 * 2)
		603	* 5 - second 3/4 of lb (5760 * 2)
		604	* 6 - whole lb (7680 * 2), other crtc must be disabled
		605	* 7 - last 1/4 of lb (1920 * 2)
		606	*/
		607	/* this can get tricky if we have two large displays on a paired group
		608	* of crtcs. Ideally for multiple large displays we'd assign them to
		609	* non-linked crtcs for maximum line buffer allocation.
		610	*/
		611	if (radeon_crtc->base.enabled && mode) {
		612	if (other_mode)
		613	tmp = 0; /* 1/2 */
		614	else
		615	tmp = 2; /* whole */
		616	} else
		617	tmp = 0;
1963	serge	618
1986	serge	619	/* second controller of the pair uses second half of the lb */
		620	if (radeon_crtc->crtc_id % 2)
		621	tmp += 4;
		622	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
		623
		624	if (radeon_crtc->base.enabled && mode) {
		625	switch (tmp) {
		626	case 0:
		627	case 4:
		628	default:
		629	if (ASIC_IS_DCE5(rdev))
		630	return 4096 * 2;
		631	else
		632	return 3840 * 2;
		633	case 1:
		634	case 5:
		635	if (ASIC_IS_DCE5(rdev))
		636	return 6144 * 2;
		637	else
		638	return 5760 * 2;
		639	case 2:
		640	case 6:
		641	if (ASIC_IS_DCE5(rdev))
		642	return 8192 * 2;
		643	else
		644	return 7680 * 2;
		645	case 3:
		646	case 7:
		647	if (ASIC_IS_DCE5(rdev))
		648	return 2048 * 2;
		649	else
		650	return 1920 * 2;
		651	}
		652	}
		653
		654	/* controller not enabled, so no lb used */
		655	return 0;
		656	}
		657
2997	Serge	658	u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1986	serge	659	{
		660	u32 tmp = RREG32(MC_SHARED_CHMAP);
		661
		662	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		663	case 0:
		664	default:
		665	return 1;
		666	case 1:
		667	return 2;
		668	case 2:
		669	return 4;
		670	case 3:
		671	return 8;
		672	}
		673	}
		674
		675	struct evergreen_wm_params {
		676	u32 dram_channels; /* number of dram channels */
		677	u32 yclk; /* bandwidth per dram data pin in kHz */
		678	u32 sclk; /* engine clock in kHz */
		679	u32 disp_clk; /* display clock in kHz */
		680	u32 src_width; /* viewport width */
		681	u32 active_time; /* active display time in ns */
		682	u32 blank_time; /* blank time in ns */
		683	bool interlaced; /* mode is interlaced */
		684	fixed20_12 vsc; /* vertical scale ratio */
		685	u32 num_heads; /* number of active crtcs */
		686	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
		687	u32 lb_size; /* line buffer allocated to pipe */
		688	u32 vtaps; /* vertical scaler taps */
		689	};
		690
		691	static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
		692	{
		693	/* Calculate DRAM Bandwidth and the part allocated to display. */
		694	fixed20_12 dram_efficiency; /* 0.7 */
		695	fixed20_12 yclk, dram_channels, bandwidth;
		696	fixed20_12 a;
		697
		698	a.full = dfixed_const(1000);
		699	yclk.full = dfixed_const(wm->yclk);
		700	yclk.full = dfixed_div(yclk, a);
		701	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		702	a.full = dfixed_const(10);
		703	dram_efficiency.full = dfixed_const(7);
		704	dram_efficiency.full = dfixed_div(dram_efficiency, a);
		705	bandwidth.full = dfixed_mul(dram_channels, yclk);
		706	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
		707
		708	return dfixed_trunc(bandwidth);
		709	}
		710
		711	static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		712	{
		713	/* Calculate DRAM Bandwidth and the part allocated to display. */
		714	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
		715	fixed20_12 yclk, dram_channels, bandwidth;
		716	fixed20_12 a;
		717
		718	a.full = dfixed_const(1000);
		719	yclk.full = dfixed_const(wm->yclk);
		720	yclk.full = dfixed_div(yclk, a);
		721	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		722	a.full = dfixed_const(10);
		723	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
		724	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
		725	bandwidth.full = dfixed_mul(dram_channels, yclk);
		726	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
		727
		728	return dfixed_trunc(bandwidth);
		729	}
		730
		731	static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
		732	{
		733	/* Calculate the display Data return Bandwidth */
		734	fixed20_12 return_efficiency; /* 0.8 */
		735	fixed20_12 sclk, bandwidth;
		736	fixed20_12 a;
		737
		738	a.full = dfixed_const(1000);
		739	sclk.full = dfixed_const(wm->sclk);
		740	sclk.full = dfixed_div(sclk, a);
		741	a.full = dfixed_const(10);
		742	return_efficiency.full = dfixed_const(8);
		743	return_efficiency.full = dfixed_div(return_efficiency, a);
		744	a.full = dfixed_const(32);
		745	bandwidth.full = dfixed_mul(a, sclk);
		746	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
		747
		748	return dfixed_trunc(bandwidth);
		749	}
		750
		751	static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
		752	{
		753	/* Calculate the DMIF Request Bandwidth */
		754	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
		755	fixed20_12 disp_clk, bandwidth;
		756	fixed20_12 a;
		757
		758	a.full = dfixed_const(1000);
		759	disp_clk.full = dfixed_const(wm->disp_clk);
		760	disp_clk.full = dfixed_div(disp_clk, a);
		761	a.full = dfixed_const(10);
		762	disp_clk_request_efficiency.full = dfixed_const(8);
		763	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
		764	a.full = dfixed_const(32);
		765	bandwidth.full = dfixed_mul(a, disp_clk);
		766	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
		767
		768	return dfixed_trunc(bandwidth);
		769	}
		770
		771	static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
		772	{
		773	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
		774	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
		775	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
		776	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
		777
		778	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
		779	}
		780
		781	static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
		782	{
		783	/* Calculate the display mode Average Bandwidth
		784	* DisplayMode should contain the source and destination dimensions,
		785	* timing, etc.
		786	*/
		787	fixed20_12 bpp;
		788	fixed20_12 line_time;
		789	fixed20_12 src_width;
		790	fixed20_12 bandwidth;
		791	fixed20_12 a;
		792
		793	a.full = dfixed_const(1000);
		794	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
		795	line_time.full = dfixed_div(line_time, a);
		796	bpp.full = dfixed_const(wm->bytes_per_pixel);
		797	src_width.full = dfixed_const(wm->src_width);
		798	bandwidth.full = dfixed_mul(src_width, bpp);
		799	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
		800	bandwidth.full = dfixed_div(bandwidth, line_time);
		801
		802	return dfixed_trunc(bandwidth);
		803	}
		804
		805	static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
		806	{
		807	/* First calcualte the latency in ns */
		808	u32 mc_latency = 2000; /* 2000 ns. */
		809	u32 available_bandwidth = evergreen_available_bandwidth(wm);
		810	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
		811	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
		812	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
		813	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
		814	(wm->num_heads * cursor_line_pair_return_time);
		815	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
		816	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
		817	fixed20_12 a, b, c;
		818
		819	if (wm->num_heads == 0)
		820	return 0;
		821
		822	a.full = dfixed_const(2);
		823	b.full = dfixed_const(1);
		824	if ((wm->vsc.full > a.full) \|\|
		825	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) \|\|
		826	(wm->vtaps >= 5) \|\|
		827	((wm->vsc.full >= a.full) && wm->interlaced))
		828	max_src_lines_per_dst_line = 4;
		829	else
		830	max_src_lines_per_dst_line = 2;
		831
		832	a.full = dfixed_const(available_bandwidth);
		833	b.full = dfixed_const(wm->num_heads);
		834	a.full = dfixed_div(a, b);
		835
		836	b.full = dfixed_const(1000);
		837	c.full = dfixed_const(wm->disp_clk);
		838	b.full = dfixed_div(c, b);
		839	c.full = dfixed_const(wm->bytes_per_pixel);
		840	b.full = dfixed_mul(b, c);
		841
		842	lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
		843
		844	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
		845	b.full = dfixed_const(1000);
		846	c.full = dfixed_const(lb_fill_bw);
		847	b.full = dfixed_div(c, b);
		848	a.full = dfixed_div(a, b);
		849	line_fill_time = dfixed_trunc(a);
		850
		851	if (line_fill_time < wm->active_time)
		852	return latency;
		853	else
		854	return latency + (line_fill_time - wm->active_time);
		855
		856	}
		857
		858	static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		859	{
		860	if (evergreen_average_bandwidth(wm) <=
		861	(evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
		862	return true;
		863	else
		864	return false;
		865	};
		866
		867	static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
		868	{
		869	if (evergreen_average_bandwidth(wm) <=
		870	(evergreen_available_bandwidth(wm) / wm->num_heads))
		871	return true;
		872	else
		873	return false;
		874	};
		875
		876	static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
		877	{
		878	u32 lb_partitions = wm->lb_size / wm->src_width;
		879	u32 line_time = wm->active_time + wm->blank_time;
		880	u32 latency_tolerant_lines;
		881	u32 latency_hiding;
		882	fixed20_12 a;
		883
		884	a.full = dfixed_const(1);
		885	if (wm->vsc.full > a.full)
		886	latency_tolerant_lines = 1;
		887	else {
		888	if (lb_partitions <= (wm->vtaps + 1))
		889	latency_tolerant_lines = 1;
		890	else
		891	latency_tolerant_lines = 2;
		892	}
		893
		894	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
		895
		896	if (evergreen_latency_watermark(wm) <= latency_hiding)
		897	return true;
		898	else
		899	return false;
		900	}
		901
		902	static void evergreen_program_watermarks(struct radeon_device *rdev,
		903	struct radeon_crtc *radeon_crtc,
		904	u32 lb_size, u32 num_heads)
		905	{
		906	struct drm_display_mode *mode = &radeon_crtc->base.mode;
		907	struct evergreen_wm_params wm;
		908	u32 pixel_period;
		909	u32 line_time = 0;
		910	u32 latency_watermark_a = 0, latency_watermark_b = 0;
		911	u32 priority_a_mark = 0, priority_b_mark = 0;
		912	u32 priority_a_cnt = PRIORITY_OFF;
		913	u32 priority_b_cnt = PRIORITY_OFF;
		914	u32 pipe_offset = radeon_crtc->crtc_id * 16;
		915	u32 tmp, arb_control3;
		916	fixed20_12 a, b, c;
		917
		918	if (radeon_crtc->base.enabled && num_heads && mode) {
		919	pixel_period = 1000000 / (u32)mode->clock;
		920	line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
		921	priority_a_cnt = 0;
		922	priority_b_cnt = 0;
		923
		924	wm.yclk = rdev->pm.current_mclk * 10;
		925	wm.sclk = rdev->pm.current_sclk * 10;
		926	wm.disp_clk = mode->clock;
		927	wm.src_width = mode->crtc_hdisplay;
		928	wm.active_time = mode->crtc_hdisplay * pixel_period;
		929	wm.blank_time = line_time - wm.active_time;
		930	wm.interlaced = false;
		931	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		932	wm.interlaced = true;
		933	wm.vsc = radeon_crtc->vsc;
		934	wm.vtaps = 1;
		935	if (radeon_crtc->rmx_type != RMX_OFF)
		936	wm.vtaps = 2;
		937	wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
		938	wm.lb_size = lb_size;
		939	wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
		940	wm.num_heads = num_heads;
		941
		942	/* set for high clocks */
		943	latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
		944	/* set for low clocks */
		945	/* wm.yclk = low clk; wm.sclk = low clk */
		946	latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
		947
		948	/* possibly force display priority to high */
		949	/* should really do this at mode validation time... */
		950	if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) \|\|
		951	!evergreen_average_bandwidth_vs_available_bandwidth(&wm) \|\|
		952	!evergreen_check_latency_hiding(&wm) \|\|
		953	(rdev->disp_priority == 2)) {
2160	serge	954	DRM_DEBUG_KMS("force priority to high\n");
1986	serge	955	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		956	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		957	}
		958
		959	a.full = dfixed_const(1000);
		960	b.full = dfixed_const(mode->clock);
		961	b.full = dfixed_div(b, a);
		962	c.full = dfixed_const(latency_watermark_a);
		963	c.full = dfixed_mul(c, b);
		964	c.full = dfixed_mul(c, radeon_crtc->hsc);
		965	c.full = dfixed_div(c, a);
		966	a.full = dfixed_const(16);
		967	c.full = dfixed_div(c, a);
		968	priority_a_mark = dfixed_trunc(c);
		969	priority_a_cnt \|= priority_a_mark & PRIORITY_MARK_MASK;
		970
		971	a.full = dfixed_const(1000);
		972	b.full = dfixed_const(mode->clock);
		973	b.full = dfixed_div(b, a);
		974	c.full = dfixed_const(latency_watermark_b);
		975	c.full = dfixed_mul(c, b);
		976	c.full = dfixed_mul(c, radeon_crtc->hsc);
		977	c.full = dfixed_div(c, a);
		978	a.full = dfixed_const(16);
		979	c.full = dfixed_div(c, a);
		980	priority_b_mark = dfixed_trunc(c);
		981	priority_b_cnt \|= priority_b_mark & PRIORITY_MARK_MASK;
		982	}
		983
		984	/* select wm A */
		985	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		986	tmp = arb_control3;
		987	tmp &= ~LATENCY_WATERMARK_MASK(3);
		988	tmp \|= LATENCY_WATERMARK_MASK(1);
		989	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		990	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		991	(LATENCY_LOW_WATERMARK(latency_watermark_a) \|
		992	LATENCY_HIGH_WATERMARK(line_time)));
		993	/* select wm B */
		994	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		995	tmp &= ~LATENCY_WATERMARK_MASK(3);
		996	tmp \|= LATENCY_WATERMARK_MASK(2);
		997	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		998	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		999	(LATENCY_LOW_WATERMARK(latency_watermark_b) \|
		1000	LATENCY_HIGH_WATERMARK(line_time)));
		1001	/* restore original selection */
		1002	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
		1003
		1004	/* write the priority marks */
		1005	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
		1006	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
		1007
		1008	}
		1009
2997	Serge	1010	/**
		1011	* evergreen_bandwidth_update - update display watermarks callback.
		1012	*
		1013	* @rdev: radeon_device pointer
		1014	*
		1015	* Update the display watermarks based on the requested mode(s)
		1016	* (evergreen+).
		1017	*/
1963	serge	1018	void evergreen_bandwidth_update(struct radeon_device *rdev)
1430	serge	1019	{
1986	serge	1020	struct drm_display_mode *mode0 = NULL;
		1021	struct drm_display_mode *mode1 = NULL;
		1022	u32 num_heads = 0, lb_size;
		1023	int i;
		1024
		1025	radeon_update_display_priority(rdev);
		1026
		1027	for (i = 0; i < rdev->num_crtc; i++) {
		1028	if (rdev->mode_info.crtcs[i]->base.enabled)
		1029	num_heads++;
		1030	}
		1031	for (i = 0; i < rdev->num_crtc; i += 2) {
		1032	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
		1033	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
		1034	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
		1035	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
		1036	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
		1037	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
		1038	}
1430	serge	1039	}
		1040
2997	Serge	1041	/**
		1042	* evergreen_mc_wait_for_idle - wait for MC idle callback.
		1043	*
		1044	* @rdev: radeon_device pointer
		1045	*
		1046	* Wait for the MC (memory controller) to be idle.
		1047	* (evergreen+).
		1048	* Returns 0 if the MC is idle, -1 if not.
		1049	*/
1963	serge	1050	int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1430	serge	1051	{
		1052	unsigned i;
		1053	u32 tmp;
		1054
		1055	for (i = 0; i < rdev->usec_timeout; i++) {
		1056	/* read MC_STATUS */
		1057	tmp = RREG32(SRBM_STATUS) & 0x1F00;
		1058	if (!tmp)
		1059	return 0;
		1060	udelay(1);
		1061	}
		1062	return -1;
		1063	}
		1064
		1065	/*
		1066	* GART
		1067	*/
1963	serge	1068	void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
		1069	{
		1070	unsigned i;
		1071	u32 tmp;
		1072
		1073	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		1074
		1075	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		1076	for (i = 0; i < rdev->usec_timeout; i++) {
		1077	/* read MC_STATUS */
		1078	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		1079	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		1080	if (tmp == 2) {
		1081	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		1082	return;
		1083	}
		1084	if (tmp) {
		1085	return;
		1086	}
		1087	udelay(1);
		1088	}
		1089	}
		1090
2997	Serge	1091	static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
1430	serge	1092	{
		1093	u32 tmp;
1963	serge	1094	int r;
1430	serge	1095
2997	Serge	1096	if (rdev->gart.robj == NULL) {
1430	serge	1097	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		1098	return -EINVAL;
		1099	}
		1100	r = radeon_gart_table_vram_pin(rdev);
		1101	if (r)
		1102	return r;
		1103	radeon_gart_restore(rdev);
		1104	/* Setup L2 cache */
		1105	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		1106	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1107	EFFECTIVE_L2_QUEUE_SIZE(7));
		1108	WREG32(VM_L2_CNTL2, 0);
		1109	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		1110	/* Setup TLB control */
		1111	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		1112	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		1113	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		1114	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
1963	serge	1115	if (rdev->flags & RADEON_IS_IGP) {
		1116	WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
		1117	WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
		1118	WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
		1119	} else {
3031	serge	1120	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		1121	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		1122	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2997	Serge	1123	if ((rdev->family == CHIP_JUNIPER) \|\|
		1124	(rdev->family == CHIP_CYPRESS) \|\|
		1125	(rdev->family == CHIP_HEMLOCK) \|\|
		1126	(rdev->family == CHIP_BARTS))
		1127	WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
1963	serge	1128	}
1430	serge	1129	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		1130	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		1131	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		1132	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
		1133	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		1134	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		1135	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		1136	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		1137	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		1138	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		1139	(u32)(rdev->dummy_page.addr >> 12));
1963	serge	1140	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	1141
1963	serge	1142	evergreen_pcie_gart_tlb_flush(rdev);
2997	Serge	1143	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		1144	(unsigned)(rdev->mc.gtt_size >> 20),
		1145	(unsigned long long)rdev->gart.table_addr);
1430	serge	1146	rdev->gart.ready = true;
		1147	return 0;
		1148	}
		1149
2997	Serge	1150	static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
1430	serge	1151	{
		1152	u32 tmp;
		1153
		1154	/* Disable all tables */
1963	serge	1155	WREG32(VM_CONTEXT0_CNTL, 0);
		1156	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	1157
		1158	/* Setup L2 cache */
		1159	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		1160	EFFECTIVE_L2_QUEUE_SIZE(7));
		1161	WREG32(VM_L2_CNTL2, 0);
		1162	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		1163	/* Setup TLB control */
		1164	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		1165	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		1166	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		1167	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		1168	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		1169	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		1170	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		1171	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2997	Serge	1172	radeon_gart_table_vram_unpin(rdev);
1430	serge	1173	}
		1174
2997	Serge	1175	static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
1430	serge	1176	{
		1177	evergreen_pcie_gart_disable(rdev);
		1178	radeon_gart_table_vram_free(rdev);
		1179	radeon_gart_fini(rdev);
		1180	}
		1181
		1182
2997	Serge	1183	static void evergreen_agp_enable(struct radeon_device *rdev)
1430	serge	1184	{
		1185	u32 tmp;
		1186
		1187	/* Setup L2 cache */
		1188	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		1189	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1190	EFFECTIVE_L2_QUEUE_SIZE(7));
		1191	WREG32(VM_L2_CNTL2, 0);
		1192	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		1193	/* Setup TLB control */
		1194	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		1195	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		1196	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		1197	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		1198	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		1199	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		1200	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		1201	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		1202	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		1203	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		1204	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1963	serge	1205	WREG32(VM_CONTEXT0_CNTL, 0);
		1206	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	1207	}
		1208
1963	serge	1209	void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	1210	{
2997	Serge	1211	u32 crtc_enabled, tmp, frame_count, blackout;
		1212	int i, j;
		1213
1430	serge	1214	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
		1215	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
		1216
2997	Serge	1217	/* disable VGA render */
1430	serge	1218	WREG32(VGA_RENDER_CONTROL, 0);
2997	Serge	1219	/* blank the display controllers */
		1220	for (i = 0; i < rdev->num_crtc; i++) {
		1221	crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
		1222	if (crtc_enabled) {
		1223	save->crtc_enabled[i] = true;
		1224	if (ASIC_IS_DCE6(rdev)) {
		1225	tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
		1226	if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
		1227	radeon_wait_for_vblank(rdev, i);
		1228	tmp \|= EVERGREEN_CRTC_BLANK_DATA_EN;
		1229	WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
3031	serge	1230	}
2997	Serge	1231	} else {
		1232	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
		1233	if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
		1234	radeon_wait_for_vblank(rdev, i);
		1235	tmp \|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		1236	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
3031	serge	1237	}
		1238	}
2997	Serge	1239	/* wait for the next frame */
		1240	frame_count = radeon_get_vblank_counter(rdev, i);
		1241	for (j = 0; j < rdev->usec_timeout; j++) {
		1242	if (radeon_get_vblank_counter(rdev, i) != frame_count)
		1243	break;
		1244	udelay(1);
3031	serge	1245	}
3120	serge	1246	} else {
		1247	save->crtc_enabled[i] = false;
3031	serge	1248	}
1963	serge	1249	}
1430	serge	1250
2997	Serge	1251	radeon_mc_wait_for_idle(rdev);
		1252
		1253	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
		1254	if ((blackout & BLACKOUT_MODE_MASK) != 1) {
		1255	/* Block CPU access */
		1256	WREG32(BIF_FB_EN, 0);
		1257	/* blackout the MC */
		1258	blackout &= ~BLACKOUT_MODE_MASK;
		1259	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout \| 1);
2005	serge	1260	}
1430	serge	1261	}
		1262
1963	serge	1263	void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	1264	{
2997	Serge	1265	u32 tmp, frame_count;
		1266	int i, j;
1430	serge	1267
2997	Serge	1268	/* update crtc base addresses */
		1269	for (i = 0; i < rdev->num_crtc; i++) {
		1270	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
3031	serge	1271	upper_32_bits(rdev->mc.vram_start));
2997	Serge	1272	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
3031	serge	1273	upper_32_bits(rdev->mc.vram_start));
2997	Serge	1274	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
3031	serge	1275	(u32)rdev->mc.vram_start);
2997	Serge	1276	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
3031	serge	1277	(u32)rdev->mc.vram_start);
2997	Serge	1278	}
		1279	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
		1280	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
1430	serge	1281
2997	Serge	1282	/* unblackout the MC */
		1283	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
		1284	tmp &= ~BLACKOUT_MODE_MASK;
		1285	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
		1286	/* allow CPU access */
		1287	WREG32(BIF_FB_EN, FB_READ_EN \| FB_WRITE_EN);
1430	serge	1288
2997	Serge	1289	for (i = 0; i < rdev->num_crtc; i++) {
3031	serge	1290	if (save->crtc_enabled[i]) {
2997	Serge	1291	if (ASIC_IS_DCE6(rdev)) {
		1292	tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
		1293	tmp \|= EVERGREEN_CRTC_BLANK_DATA_EN;
		1294	WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
		1295	} else {
		1296	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
		1297	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		1298	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
		1299	}
		1300	/* wait for the next frame */
		1301	frame_count = radeon_get_vblank_counter(rdev, i);
		1302	for (j = 0; j < rdev->usec_timeout; j++) {
		1303	if (radeon_get_vblank_counter(rdev, i) != frame_count)
		1304	break;
		1305	udelay(1);
		1306	}
3031	serge	1307	}
2005	serge	1308	}
2997	Serge	1309	/* Unlock vga access */
1430	serge	1310	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
		1311	mdelay(1);
		1312	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
		1313	}
		1314
1963	serge	1315	void evergreen_mc_program(struct radeon_device *rdev)
1430	serge	1316	{
		1317	struct evergreen_mc_save save;
		1318	u32 tmp;
		1319	int i, j;
		1320
		1321	/* Initialize HDP */
		1322	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1323	WREG32((0x2c14 + j), 0x00000000);
		1324	WREG32((0x2c18 + j), 0x00000000);
		1325	WREG32((0x2c1c + j), 0x00000000);
		1326	WREG32((0x2c20 + j), 0x00000000);
		1327	WREG32((0x2c24 + j), 0x00000000);
		1328	}
		1329	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		1330
		1331	evergreen_mc_stop(rdev, &save);
		1332	if (evergreen_mc_wait_for_idle(rdev)) {
		1333	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1334	}
		1335	/* Lockout access through VGA aperture*/
		1336	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		1337	/* Update configuration */
		1338	if (rdev->flags & RADEON_IS_AGP) {
		1339	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		1340	/* VRAM before AGP */
		1341	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1342	rdev->mc.vram_start >> 12);
		1343	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1344	rdev->mc.gtt_end >> 12);
		1345	} else {
		1346	/* VRAM after AGP */
		1347	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1348	rdev->mc.gtt_start >> 12);
		1349	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1350	rdev->mc.vram_end >> 12);
		1351	}
		1352	} else {
		1353	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1354	rdev->mc.vram_start >> 12);
		1355	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1356	rdev->mc.vram_end >> 12);
		1357	}
2997	Serge	1358	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
		1359	/* llano/ontario only */
		1360	if ((rdev->family == CHIP_PALM) \|\|
		1361	(rdev->family == CHIP_SUMO) \|\|
		1362	(rdev->family == CHIP_SUMO2)) {
1963	serge	1363	tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
		1364	tmp \|= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
		1365	tmp \|= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
		1366	WREG32(MC_FUS_VM_FB_OFFSET, tmp);
		1367	}
1430	serge	1368	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		1369	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		1370	WREG32(MC_VM_FB_LOCATION, tmp);
		1371	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1963	serge	1372	WREG32(HDP_NONSURFACE_INFO, (2 << 7) \| (1 << 30));
		1373	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1430	serge	1374	if (rdev->flags & RADEON_IS_AGP) {
		1375	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
		1376	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
		1377	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		1378	} else {
		1379	WREG32(MC_VM_AGP_BASE, 0);
		1380	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		1381	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		1382	}
		1383	if (evergreen_mc_wait_for_idle(rdev)) {
		1384	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1385	}
		1386	evergreen_mc_resume(rdev, &save);
		1387	/* we need to own VRAM, so turn off the VGA renderer here
		1388	* to stop it overwriting our objects */
		1389	rv515_vga_render_disable(rdev);
		1390	}
		1391
		1392	/*
		1393	* CP.
		1394	*/
1986	serge	1395	void evergreen_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		1396	{
2997	Serge	1397	struct radeon_ring *ring = &rdev->ring[ib->ring];
		1398	u32 next_rptr;
		1399
1986	serge	1400	/* set to DX10/11 mode */
2997	Serge	1401	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
		1402	radeon_ring_write(ring, 1);
		1403
		1404	if (ring->rptr_save_reg) {
		1405	next_rptr = ring->wptr + 3 + 4;
		1406	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3031	serge	1407	radeon_ring_write(ring, ((ring->rptr_save_reg -
2997	Serge	1408	PACKET3_SET_CONFIG_REG_START) >> 2));
		1409	radeon_ring_write(ring, next_rptr);
		1410	} else if (rdev->wb.enabled) {
		1411	next_rptr = ring->wptr + 5 + 4;
		1412	radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
		1413	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		1414	radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) \| (1 << 18));
		1415	radeon_ring_write(ring, next_rptr);
		1416	radeon_ring_write(ring, 0);
		1417	}
		1418
		1419	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		1420	radeon_ring_write(ring,
1986	serge	1421	#ifdef __BIG_ENDIAN
		1422	(2 << 0) \|
		1423	#endif
		1424	(ib->gpu_addr & 0xFFFFFFFC));
2997	Serge	1425	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
		1426	radeon_ring_write(ring, ib->length_dw);
1986	serge	1427	}
1963	serge	1428
1986	serge	1429
1963	serge	1430	static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1430	serge	1431	{
1963	serge	1432	const __be32 *fw_data;
		1433	int i;
		1434
		1435	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1436	return -EINVAL;
		1437
		1438	r700_cp_stop(rdev);
		1439	WREG32(CP_RB_CNTL,
		1440	#ifdef __BIG_ENDIAN
		1441	BUF_SWAP_32BIT \|
		1442	#endif
		1443	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
		1444
		1445	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1446	WREG32(CP_PFP_UCODE_ADDR, 0);
		1447	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
		1448	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		1449	WREG32(CP_PFP_UCODE_ADDR, 0);
		1450
		1451	fw_data = (const __be32 *)rdev->me_fw->data;
		1452	WREG32(CP_ME_RAM_WADDR, 0);
		1453	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
		1454	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		1455
		1456	WREG32(CP_PFP_UCODE_ADDR, 0);
		1457	WREG32(CP_ME_RAM_WADDR, 0);
		1458	WREG32(CP_ME_RAM_RADDR, 0);
		1459	return 0;
1430	serge	1460	}
		1461
1963	serge	1462	static int evergreen_cp_start(struct radeon_device *rdev)
		1463	{
2997	Serge	1464	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	1465	int r, i;
		1466	uint32_t cp_me;
1430	serge	1467
2997	Serge	1468	r = radeon_ring_lock(rdev, ring, 7);
1963	serge	1469	if (r) {
		1470	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1471	return r;
		1472	}
2997	Serge	1473	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1474	radeon_ring_write(ring, 0x1);
		1475	radeon_ring_write(ring, 0x0);
		1476	radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
		1477	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1478	radeon_ring_write(ring, 0);
		1479	radeon_ring_write(ring, 0);
		1480	radeon_ring_unlock_commit(rdev, ring);
1963	serge	1481
		1482	cp_me = 0xff;
		1483	WREG32(CP_ME_CNTL, cp_me);
		1484
2997	Serge	1485	r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
1963	serge	1486	if (r) {
		1487	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1488	return r;
		1489	}
		1490
		1491	/* setup clear context state */
2997	Serge	1492	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1493	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1963	serge	1494
		1495	for (i = 0; i < evergreen_default_size; i++)
2997	Serge	1496	radeon_ring_write(ring, evergreen_default_state[i]);
1963	serge	1497
2997	Serge	1498	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1499	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1963	serge	1500
		1501	/* set clear context state */
2997	Serge	1502	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
		1503	radeon_ring_write(ring, 0);
1963	serge	1504
		1505	/* SQ_VTX_BASE_VTX_LOC */
2997	Serge	1506	radeon_ring_write(ring, 0xc0026f00);
		1507	radeon_ring_write(ring, 0x00000000);
		1508	radeon_ring_write(ring, 0x00000000);
		1509	radeon_ring_write(ring, 0x00000000);
1963	serge	1510
		1511	/* Clear consts */
2997	Serge	1512	radeon_ring_write(ring, 0xc0036f00);
		1513	radeon_ring_write(ring, 0x00000bc4);
		1514	radeon_ring_write(ring, 0xffffffff);
		1515	radeon_ring_write(ring, 0xffffffff);
		1516	radeon_ring_write(ring, 0xffffffff);
1963	serge	1517
2997	Serge	1518	radeon_ring_write(ring, 0xc0026900);
		1519	radeon_ring_write(ring, 0x00000316);
		1520	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		1521	radeon_ring_write(ring, 0x00000010); /* */
1963	serge	1522
2997	Serge	1523	radeon_ring_unlock_commit(rdev, ring);
1963	serge	1524
		1525	return 0;
		1526	}
		1527
2997	Serge	1528	static int evergreen_cp_resume(struct radeon_device *rdev)
1430	serge	1529	{
2997	Serge	1530	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	1531	u32 tmp;
		1532	u32 rb_bufsz;
		1533	int r;
1430	serge	1534
1963	serge	1535	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
		1536	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP \|
		1537	SOFT_RESET_PA \|
		1538	SOFT_RESET_SH \|
		1539	SOFT_RESET_VGT \|
2160	serge	1540	SOFT_RESET_SPI \|
1963	serge	1541	SOFT_RESET_SX));
		1542	RREG32(GRBM_SOFT_RESET);
		1543	mdelay(15);
		1544	WREG32(GRBM_SOFT_RESET, 0);
		1545	RREG32(GRBM_SOFT_RESET);
		1546
		1547	/* Set ring buffer size */
2997	Serge	1548	rb_bufsz = drm_order(ring->ring_size / 8);
1963	serge	1549	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
		1550	#ifdef __BIG_ENDIAN
		1551	tmp \|= BUF_SWAP_32BIT;
		1552	#endif
		1553	WREG32(CP_RB_CNTL, tmp);
2997	Serge	1554	WREG32(CP_SEM_WAIT_TIMER, 0x0);
		1555	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1963	serge	1556
		1557	/* Set the write pointer delay */
		1558	WREG32(CP_RB_WPTR_DELAY, 0);
		1559
		1560	/* Initialize the ring buffer's read and write pointers */
		1561	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1562	WREG32(CP_RB_RPTR_WR, 0);
2997	Serge	1563	ring->wptr = 0;
		1564	WREG32(CP_RB_WPTR, ring->wptr);
1963	serge	1565
3120	serge	1566	/* set the wb address whether it's enabled or not */
1963	serge	1567	WREG32(CP_RB_RPTR_ADDR,
		1568	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1569	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1570	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1571
		1572	if (rdev->wb.enabled)
		1573	WREG32(SCRATCH_UMSK, 0xff);
		1574	else {
		1575	tmp \|= RB_NO_UPDATE;
		1576	WREG32(SCRATCH_UMSK, 0);
		1577	}
		1578
		1579	mdelay(1);
		1580	WREG32(CP_RB_CNTL, tmp);
		1581
2997	Serge	1582	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1963	serge	1583	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1584
2997	Serge	1585	ring->rptr = RREG32(CP_RB_RPTR);
1963	serge	1586
		1587	evergreen_cp_start(rdev);
2997	Serge	1588	ring->ready = true;
		1589	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
1963	serge	1590	if (r) {
2997	Serge	1591	ring->ready = false;
1963	serge	1592	return r;
		1593	}
1430	serge	1594	return 0;
		1595	}
		1596
		1597	/*
		1598	* Core functions
		1599	*/
		1600	static void evergreen_gpu_init(struct radeon_device *rdev)
		1601	{
2997	Serge	1602	u32 gb_addr_config;
1963	serge	1603	u32 mc_shared_chmap, mc_arb_ramcfg;
		1604	u32 sx_debug_1;
		1605	u32 smx_dc_ctl0;
		1606	u32 sq_config;
		1607	u32 sq_lds_resource_mgmt;
		1608	u32 sq_gpr_resource_mgmt_1;
		1609	u32 sq_gpr_resource_mgmt_2;
		1610	u32 sq_gpr_resource_mgmt_3;
		1611	u32 sq_thread_resource_mgmt;
		1612	u32 sq_thread_resource_mgmt_2;
		1613	u32 sq_stack_resource_mgmt_1;
		1614	u32 sq_stack_resource_mgmt_2;
		1615	u32 sq_stack_resource_mgmt_3;
		1616	u32 vgt_cache_invalidation;
		1617	u32 hdp_host_path_cntl, tmp;
2997	Serge	1618	u32 disabled_rb_mask;
1963	serge	1619	int i, j, num_shader_engines, ps_thread_count;
		1620
		1621	switch (rdev->family) {
		1622	case CHIP_CYPRESS:
		1623	case CHIP_HEMLOCK:
		1624	rdev->config.evergreen.num_ses = 2;
		1625	rdev->config.evergreen.max_pipes = 4;
		1626	rdev->config.evergreen.max_tile_pipes = 8;
		1627	rdev->config.evergreen.max_simds = 10;
		1628	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1629	rdev->config.evergreen.max_gprs = 256;
		1630	rdev->config.evergreen.max_threads = 248;
		1631	rdev->config.evergreen.max_gs_threads = 32;
		1632	rdev->config.evergreen.max_stack_entries = 512;
		1633	rdev->config.evergreen.sx_num_of_sets = 4;
		1634	rdev->config.evergreen.sx_max_export_size = 256;
		1635	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1636	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1637	rdev->config.evergreen.max_hw_contexts = 8;
		1638	rdev->config.evergreen.sq_num_cf_insts = 2;
		1639
		1640	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1641	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1642	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1643	gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1644	break;
		1645	case CHIP_JUNIPER:
		1646	rdev->config.evergreen.num_ses = 1;
		1647	rdev->config.evergreen.max_pipes = 4;
		1648	rdev->config.evergreen.max_tile_pipes = 4;
		1649	rdev->config.evergreen.max_simds = 10;
		1650	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1651	rdev->config.evergreen.max_gprs = 256;
		1652	rdev->config.evergreen.max_threads = 248;
		1653	rdev->config.evergreen.max_gs_threads = 32;
		1654	rdev->config.evergreen.max_stack_entries = 512;
		1655	rdev->config.evergreen.sx_num_of_sets = 4;
		1656	rdev->config.evergreen.sx_max_export_size = 256;
		1657	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1658	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1659	rdev->config.evergreen.max_hw_contexts = 8;
		1660	rdev->config.evergreen.sq_num_cf_insts = 2;
		1661
		1662	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1663	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1664	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1665	gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1666	break;
		1667	case CHIP_REDWOOD:
		1668	rdev->config.evergreen.num_ses = 1;
		1669	rdev->config.evergreen.max_pipes = 4;
		1670	rdev->config.evergreen.max_tile_pipes = 4;
		1671	rdev->config.evergreen.max_simds = 5;
		1672	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1673	rdev->config.evergreen.max_gprs = 256;
		1674	rdev->config.evergreen.max_threads = 248;
		1675	rdev->config.evergreen.max_gs_threads = 32;
		1676	rdev->config.evergreen.max_stack_entries = 256;
		1677	rdev->config.evergreen.sx_num_of_sets = 4;
		1678	rdev->config.evergreen.sx_max_export_size = 256;
		1679	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1680	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1681	rdev->config.evergreen.max_hw_contexts = 8;
		1682	rdev->config.evergreen.sq_num_cf_insts = 2;
		1683
		1684	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1685	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1686	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1687	gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1688	break;
		1689	case CHIP_CEDAR:
		1690	default:
		1691	rdev->config.evergreen.num_ses = 1;
		1692	rdev->config.evergreen.max_pipes = 2;
		1693	rdev->config.evergreen.max_tile_pipes = 2;
		1694	rdev->config.evergreen.max_simds = 2;
		1695	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1696	rdev->config.evergreen.max_gprs = 256;
		1697	rdev->config.evergreen.max_threads = 192;
		1698	rdev->config.evergreen.max_gs_threads = 16;
		1699	rdev->config.evergreen.max_stack_entries = 256;
		1700	rdev->config.evergreen.sx_num_of_sets = 4;
		1701	rdev->config.evergreen.sx_max_export_size = 128;
		1702	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1703	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1704	rdev->config.evergreen.max_hw_contexts = 4;
		1705	rdev->config.evergreen.sq_num_cf_insts = 1;
		1706
		1707	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1708	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1709	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1710	gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1711	break;
		1712	case CHIP_PALM:
		1713	rdev->config.evergreen.num_ses = 1;
		1714	rdev->config.evergreen.max_pipes = 2;
		1715	rdev->config.evergreen.max_tile_pipes = 2;
		1716	rdev->config.evergreen.max_simds = 2;
		1717	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1718	rdev->config.evergreen.max_gprs = 256;
		1719	rdev->config.evergreen.max_threads = 192;
		1720	rdev->config.evergreen.max_gs_threads = 16;
		1721	rdev->config.evergreen.max_stack_entries = 256;
		1722	rdev->config.evergreen.sx_num_of_sets = 4;
		1723	rdev->config.evergreen.sx_max_export_size = 128;
		1724	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1725	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1726	rdev->config.evergreen.max_hw_contexts = 4;
		1727	rdev->config.evergreen.sq_num_cf_insts = 1;
		1728
		1729	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1730	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1731	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1732	gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1733	break;
		1734	case CHIP_SUMO:
		1735	rdev->config.evergreen.num_ses = 1;
		1736	rdev->config.evergreen.max_pipes = 4;
		1737	rdev->config.evergreen.max_tile_pipes = 2;
		1738	if (rdev->pdev->device == 0x9648)
		1739	rdev->config.evergreen.max_simds = 3;
		1740	else if ((rdev->pdev->device == 0x9647) \|\|
		1741	(rdev->pdev->device == 0x964a))
		1742	rdev->config.evergreen.max_simds = 4;
		1743	else
		1744	rdev->config.evergreen.max_simds = 5;
		1745	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1746	rdev->config.evergreen.max_gprs = 256;
		1747	rdev->config.evergreen.max_threads = 248;
		1748	rdev->config.evergreen.max_gs_threads = 32;
		1749	rdev->config.evergreen.max_stack_entries = 256;
		1750	rdev->config.evergreen.sx_num_of_sets = 4;
		1751	rdev->config.evergreen.sx_max_export_size = 256;
		1752	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1753	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1754	rdev->config.evergreen.max_hw_contexts = 8;
		1755	rdev->config.evergreen.sq_num_cf_insts = 2;
		1756
		1757	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1758	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1759	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1760	gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1761	break;
		1762	case CHIP_SUMO2:
		1763	rdev->config.evergreen.num_ses = 1;
		1764	rdev->config.evergreen.max_pipes = 4;
		1765	rdev->config.evergreen.max_tile_pipes = 4;
		1766	rdev->config.evergreen.max_simds = 2;
		1767	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1768	rdev->config.evergreen.max_gprs = 256;
		1769	rdev->config.evergreen.max_threads = 248;
		1770	rdev->config.evergreen.max_gs_threads = 32;
		1771	rdev->config.evergreen.max_stack_entries = 512;
		1772	rdev->config.evergreen.sx_num_of_sets = 4;
		1773	rdev->config.evergreen.sx_max_export_size = 256;
		1774	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1775	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1776	rdev->config.evergreen.max_hw_contexts = 8;
		1777	rdev->config.evergreen.sq_num_cf_insts = 2;
		1778
		1779	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1780	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1781	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1782	gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1783	break;
		1784	case CHIP_BARTS:
		1785	rdev->config.evergreen.num_ses = 2;
		1786	rdev->config.evergreen.max_pipes = 4;
		1787	rdev->config.evergreen.max_tile_pipes = 8;
		1788	rdev->config.evergreen.max_simds = 7;
		1789	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1790	rdev->config.evergreen.max_gprs = 256;
		1791	rdev->config.evergreen.max_threads = 248;
		1792	rdev->config.evergreen.max_gs_threads = 32;
		1793	rdev->config.evergreen.max_stack_entries = 512;
		1794	rdev->config.evergreen.sx_num_of_sets = 4;
		1795	rdev->config.evergreen.sx_max_export_size = 256;
		1796	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1797	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1798	rdev->config.evergreen.max_hw_contexts = 8;
		1799	rdev->config.evergreen.sq_num_cf_insts = 2;
		1800
		1801	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1802	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1803	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1804	gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1805	break;
		1806	case CHIP_TURKS:
		1807	rdev->config.evergreen.num_ses = 1;
		1808	rdev->config.evergreen.max_pipes = 4;
		1809	rdev->config.evergreen.max_tile_pipes = 4;
		1810	rdev->config.evergreen.max_simds = 6;
		1811	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1812	rdev->config.evergreen.max_gprs = 256;
		1813	rdev->config.evergreen.max_threads = 248;
		1814	rdev->config.evergreen.max_gs_threads = 32;
		1815	rdev->config.evergreen.max_stack_entries = 256;
		1816	rdev->config.evergreen.sx_num_of_sets = 4;
		1817	rdev->config.evergreen.sx_max_export_size = 256;
		1818	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1819	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1820	rdev->config.evergreen.max_hw_contexts = 8;
		1821	rdev->config.evergreen.sq_num_cf_insts = 2;
		1822
		1823	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1824	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1825	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1826	gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1827	break;
		1828	case CHIP_CAICOS:
		1829	rdev->config.evergreen.num_ses = 1;
		1830	rdev->config.evergreen.max_pipes = 4;
		1831	rdev->config.evergreen.max_tile_pipes = 2;
		1832	rdev->config.evergreen.max_simds = 2;
		1833	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1834	rdev->config.evergreen.max_gprs = 256;
		1835	rdev->config.evergreen.max_threads = 192;
		1836	rdev->config.evergreen.max_gs_threads = 16;
		1837	rdev->config.evergreen.max_stack_entries = 256;
		1838	rdev->config.evergreen.sx_num_of_sets = 4;
		1839	rdev->config.evergreen.sx_max_export_size = 128;
		1840	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1841	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1842	rdev->config.evergreen.max_hw_contexts = 4;
		1843	rdev->config.evergreen.sq_num_cf_insts = 1;
		1844
		1845	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1846	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1847	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1848	gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1849	break;
		1850	}
		1851
		1852	/* Initialize HDP */
		1853	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1854	WREG32((0x2c14 + j), 0x00000000);
		1855	WREG32((0x2c18 + j), 0x00000000);
		1856	WREG32((0x2c1c + j), 0x00000000);
		1857	WREG32((0x2c20 + j), 0x00000000);
		1858	WREG32((0x2c24 + j), 0x00000000);
		1859	}
		1860
		1861	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1862
2997	Serge	1863	evergreen_fix_pci_max_read_req_size(rdev);
1963	serge	1864
		1865	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2997	Serge	1866	if ((rdev->family == CHIP_PALM) \|\|
		1867	(rdev->family == CHIP_SUMO) \|\|
		1868	(rdev->family == CHIP_SUMO2))
1963	serge	1869	mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
		1870	else
3031	serge	1871	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1963	serge	1872
		1873	/* setup tiling info dword. gb_addr_config is not adequate since it does
		1874	* not have bank info, so create a custom tiling dword.
		1875	* bits 3:0 num_pipes
		1876	* bits 7:4 num_banks
		1877	* bits 11:8 group_size
		1878	* bits 15:12 row_size
		1879	*/
		1880	rdev->config.evergreen.tile_config = 0;
		1881	switch (rdev->config.evergreen.max_tile_pipes) {
		1882	case 1:
		1883	default:
		1884	rdev->config.evergreen.tile_config \|= (0 << 0);
		1885	break;
		1886	case 2:
		1887	rdev->config.evergreen.tile_config \|= (1 << 0);
		1888	break;
		1889	case 4:
		1890	rdev->config.evergreen.tile_config \|= (2 << 0);
		1891	break;
		1892	case 8:
		1893	rdev->config.evergreen.tile_config \|= (3 << 0);
		1894	break;
		1895	}
1986	serge	1896	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1963	serge	1897	if (rdev->flags & RADEON_IS_IGP)
1986	serge	1898	rdev->config.evergreen.tile_config \|= 1 << 4;
2997	Serge	1899	else {
		1900	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
		1901	case 0: /* four banks */
		1902	rdev->config.evergreen.tile_config \|= 0 << 4;
		1903	break;
		1904	case 1: /* eight banks */
		1905	rdev->config.evergreen.tile_config \|= 1 << 4;
		1906	break;
		1907	case 2: /* sixteen banks */
		1908	default:
		1909	rdev->config.evergreen.tile_config \|= 2 << 4;
		1910	break;
		1911	}
		1912	}
		1913	rdev->config.evergreen.tile_config \|= 0 << 8;
1963	serge	1914	rdev->config.evergreen.tile_config \|=
		1915	((gb_addr_config & 0x30000000) >> 28) << 12;
		1916
2997	Serge	1917	num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
1963	serge	1918
2997	Serge	1919	if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
		1920	u32 efuse_straps_4;
		1921	u32 efuse_straps_3;
1963	serge	1922
2997	Serge	1923	WREG32(RCU_IND_INDEX, 0x204);
		1924	efuse_straps_4 = RREG32(RCU_IND_DATA);
		1925	WREG32(RCU_IND_INDEX, 0x203);
		1926	efuse_straps_3 = RREG32(RCU_IND_DATA);
		1927	tmp = (((efuse_straps_4 & 0xf) << 4) \|
		1928	((efuse_straps_3 & 0xf0000000) >> 28));
		1929	} else {
		1930	tmp = 0;
		1931	for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
		1932	u32 rb_disable_bitmap;
1963	serge	1933
2997	Serge	1934	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		1935	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		1936	rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
		1937	tmp <<= 4;
		1938	tmp \|= rb_disable_bitmap;
1963	serge	1939	}
2997	Serge	1940	}
		1941	/* enabled rb are just the one not disabled :) */
		1942	disabled_rb_mask = tmp;
1963	serge	1943
2997	Serge	1944	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
		1945	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
1963	serge	1946
2997	Serge	1947	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		1948	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
		1949	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1963	serge	1950
2997	Serge	1951	tmp = gb_addr_config & NUM_PIPES_MASK;
		1952	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
		1953	EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
		1954	WREG32(GB_BACKEND_MAP, tmp);
1963	serge	1955
		1956	WREG32(CGTS_SYS_TCC_DISABLE, 0);
		1957	WREG32(CGTS_TCC_DISABLE, 0);
		1958	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
		1959	WREG32(CGTS_USER_TCC_DISABLE, 0);
		1960
		1961	/* set HW defaults for 3D engine */
		1962	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \|
		1963	ROQ_IB2_START(0x2b)));
		1964
		1965	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
		1966
		1967	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \|
		1968	SYNC_GRADIENT \|
		1969	SYNC_WALKER \|
		1970	SYNC_ALIGNER));
		1971
		1972	sx_debug_1 = RREG32(SX_DEBUG_1);
		1973	sx_debug_1 \|= ENABLE_NEW_SMX_ADDRESS;
		1974	WREG32(SX_DEBUG_1, sx_debug_1);
		1975
		1976
		1977	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
		1978	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
		1979	smx_dc_ctl0 \|= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
		1980	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
		1981
2997	Serge	1982	if (rdev->family <= CHIP_SUMO2)
		1983	WREG32(SMX_SAR_CTL0, 0x00010000);
		1984
1963	serge	1985	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) \|
		1986	POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) \|
		1987	SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
		1988
		1989	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) \|
		1990	SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) \|
		1991	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
		1992
		1993	WREG32(VGT_NUM_INSTANCES, 1);
		1994	WREG32(SPI_CONFIG_CNTL, 0);
		1995	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
		1996	WREG32(CP_PERFMON_CNTL, 0);
		1997
		1998	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) \|
		1999	FETCH_FIFO_HIWATER(0x4) \|
		2000	DONE_FIFO_HIWATER(0xe0) \|
		2001	ALU_UPDATE_FIFO_HIWATER(0x8)));
		2002
		2003	sq_config = RREG32(SQ_CONFIG);
		2004	sq_config &= ~(PS_PRIO(3) \|
		2005	VS_PRIO(3) \|
		2006	GS_PRIO(3) \|
		2007	ES_PRIO(3));
		2008	sq_config \|= (VC_ENABLE \|
		2009	EXPORT_SRC_C \|
		2010	PS_PRIO(0) \|
		2011	VS_PRIO(1) \|
		2012	GS_PRIO(2) \|
		2013	ES_PRIO(3));
		2014
		2015	switch (rdev->family) {
		2016	case CHIP_CEDAR:
		2017	case CHIP_PALM:
		2018	case CHIP_SUMO:
		2019	case CHIP_SUMO2:
		2020	case CHIP_CAICOS:
		2021	/* no vertex cache */
		2022	sq_config &= ~VC_ENABLE;
		2023	break;
		2024	default:
		2025	break;
		2026	}
		2027
		2028	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
		2029
		2030	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
		2031	sq_gpr_resource_mgmt_1 \|= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
		2032	sq_gpr_resource_mgmt_1 \|= NUM_CLAUSE_TEMP_GPRS(4);
		2033	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		2034	sq_gpr_resource_mgmt_2 \|= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		2035	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		2036	sq_gpr_resource_mgmt_3 \|= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		2037
		2038	switch (rdev->family) {
		2039	case CHIP_CEDAR:
		2040	case CHIP_PALM:
		2041	case CHIP_SUMO:
		2042	case CHIP_SUMO2:
		2043	ps_thread_count = 96;
		2044	break;
		2045	default:
		2046	ps_thread_count = 128;
		2047	break;
		2048	}
		2049
		2050	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
		2051	sq_thread_resource_mgmt \|= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2052	sq_thread_resource_mgmt \|= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2053	sq_thread_resource_mgmt \|= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2054	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2055	sq_thread_resource_mgmt_2 \|= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2056
		2057	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2058	sq_stack_resource_mgmt_1 \|= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2059	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2060	sq_stack_resource_mgmt_2 \|= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2061	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2062	sq_stack_resource_mgmt_3 \|= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2063
		2064	WREG32(SQ_CONFIG, sq_config);
		2065	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		2066	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		2067	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
		2068	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		2069	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
		2070	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		2071	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		2072	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
		2073	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
		2074	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
		2075
		2076	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		2077	FORCE_EOV_MAX_REZ_CNT(255)));
		2078
		2079	switch (rdev->family) {
		2080	case CHIP_CEDAR:
		2081	case CHIP_PALM:
		2082	case CHIP_SUMO:
		2083	case CHIP_SUMO2:
		2084	case CHIP_CAICOS:
		2085	vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
		2086	break;
		2087	default:
		2088	vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
		2089	break;
		2090	}
		2091	vgt_cache_invalidation \|= AUTO_INVLD_EN(ES_AND_GS_AUTO);
		2092	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
		2093
		2094	WREG32(VGT_GS_VERTEX_REUSE, 16);
		2095	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
		2096	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		2097
		2098	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
		2099	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
		2100
		2101	WREG32(CB_PERF_CTR0_SEL_0, 0);
		2102	WREG32(CB_PERF_CTR0_SEL_1, 0);
		2103	WREG32(CB_PERF_CTR1_SEL_0, 0);
		2104	WREG32(CB_PERF_CTR1_SEL_1, 0);
		2105	WREG32(CB_PERF_CTR2_SEL_0, 0);
		2106	WREG32(CB_PERF_CTR2_SEL_1, 0);
		2107	WREG32(CB_PERF_CTR3_SEL_0, 0);
		2108	WREG32(CB_PERF_CTR3_SEL_1, 0);
		2109
		2110	/* clear render buffer base addresses */
		2111	WREG32(CB_COLOR0_BASE, 0);
		2112	WREG32(CB_COLOR1_BASE, 0);
		2113	WREG32(CB_COLOR2_BASE, 0);
		2114	WREG32(CB_COLOR3_BASE, 0);
		2115	WREG32(CB_COLOR4_BASE, 0);
		2116	WREG32(CB_COLOR5_BASE, 0);
		2117	WREG32(CB_COLOR6_BASE, 0);
		2118	WREG32(CB_COLOR7_BASE, 0);
		2119	WREG32(CB_COLOR8_BASE, 0);
		2120	WREG32(CB_COLOR9_BASE, 0);
		2121	WREG32(CB_COLOR10_BASE, 0);
		2122	WREG32(CB_COLOR11_BASE, 0);
		2123
		2124	/* set the shader const cache sizes to 0 */
		2125	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
		2126	WREG32(i, 0);
		2127	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
		2128	WREG32(i, 0);
		2129
		2130	tmp = RREG32(HDP_MISC_CNTL);
		2131	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		2132	WREG32(HDP_MISC_CNTL, tmp);
		2133
		2134	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		2135	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		2136
		2137	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		2138
		2139	udelay(50);
		2140
1430	serge	2141	}
		2142
		2143	int evergreen_mc_init(struct radeon_device *rdev)
		2144	{
		2145	u32 tmp;
		2146	int chansize, numchan;
		2147
		2148	/* Get VRAM informations */
		2149	rdev->mc.vram_is_ddr = true;
2997	Serge	2150	if ((rdev->family == CHIP_PALM) \|\|
		2151	(rdev->family == CHIP_SUMO) \|\|
		2152	(rdev->family == CHIP_SUMO2))
2004	serge	2153	tmp = RREG32(FUS_MC_ARB_RAMCFG);
		2154	else
3031	serge	2155	tmp = RREG32(MC_ARB_RAMCFG);
1430	serge	2156	if (tmp & CHANSIZE_OVERRIDE) {
		2157	chansize = 16;
		2158	} else if (tmp & CHANSIZE_MASK) {
		2159	chansize = 64;
		2160	} else {
		2161	chansize = 32;
		2162	}
		2163	tmp = RREG32(MC_SHARED_CHMAP);
		2164	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		2165	case 0:
		2166	default:
		2167	numchan = 1;
		2168	break;
		2169	case 1:
		2170	numchan = 2;
		2171	break;
		2172	case 2:
		2173	numchan = 4;
		2174	break;
		2175	case 3:
		2176	numchan = 8;
		2177	break;
		2178	}
		2179	rdev->mc.vram_width = numchan * chansize;
		2180	/* Could aper size report 0 ? */
1963	serge	2181	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		2182	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1430	serge	2183	/* Setup GPU memory space */
2997	Serge	2184	if ((rdev->family == CHIP_PALM) \|\|
		2185	(rdev->family == CHIP_SUMO) \|\|
		2186	(rdev->family == CHIP_SUMO2)) {
1963	serge	2187	/* size in bytes on fusion */
		2188	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		2189	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
		2190	} else {
2997	Serge	2191	/* size in MB on evergreen/cayman/tn */
3031	serge	2192	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
		2193	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
1963	serge	2194	}
1430	serge	2195	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1963	serge	2196	r700_vram_gtt_location(rdev, &rdev->mc);
		2197	radeon_update_bandwidth_info(rdev);
		2198
1430	serge	2199	return 0;
		2200	}
		2201
2997	Serge	2202	bool evergreen_gpu_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
1430	serge	2203	{
1986	serge	2204	u32 srbm_status;
		2205	u32 grbm_status;
		2206	u32 grbm_status_se0, grbm_status_se1;
		2207
		2208	srbm_status = RREG32(SRBM_STATUS);
		2209	grbm_status = RREG32(GRBM_STATUS);
		2210	grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
		2211	grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
		2212	if (!(grbm_status & GUI_ACTIVE)) {
2997	Serge	2213	radeon_ring_lockup_update(ring);
3031	serge	2214	return false;
1986	serge	2215	}
		2216	/* force CP activities */
2997	Serge	2217	radeon_ring_force_activity(rdev, ring);
		2218	return radeon_ring_test_lockup(rdev, ring);
1963	serge	2219	}
		2220
		2221	static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
		2222	{
		2223	struct evergreen_mc_save save;
		2224	u32 grbm_reset = 0;
		2225
		2226	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
		2227	return 0;
		2228
		2229	dev_info(rdev->dev, "GPU softreset \n");
		2230	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2231	RREG32(GRBM_STATUS));
		2232	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2233	RREG32(GRBM_STATUS_SE0));
		2234	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2235	RREG32(GRBM_STATUS_SE1));
		2236	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2237	RREG32(SRBM_STATUS));
2997	Serge	2238	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		2239	RREG32(CP_STALLED_STAT1));
		2240	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		2241	RREG32(CP_STALLED_STAT2));
		2242	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		2243	RREG32(CP_BUSY_STAT));
		2244	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		2245	RREG32(CP_STAT));
1963	serge	2246	evergreen_mc_stop(rdev, &save);
		2247	if (evergreen_mc_wait_for_idle(rdev)) {
		2248	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		2249	}
		2250	/* Disable CP parsing/prefetching */
		2251	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT);
		2252
		2253	/* reset all the gfx blocks */
		2254	grbm_reset = (SOFT_RESET_CP \|
		2255	SOFT_RESET_CB \|
		2256	SOFT_RESET_DB \|
		2257	SOFT_RESET_PA \|
		2258	SOFT_RESET_SC \|
		2259	SOFT_RESET_SPI \|
		2260	SOFT_RESET_SH \|
		2261	SOFT_RESET_SX \|
		2262	SOFT_RESET_TC \|
		2263	SOFT_RESET_TA \|
		2264	SOFT_RESET_VC \|
		2265	SOFT_RESET_VGT);
		2266
		2267	dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
		2268	WREG32(GRBM_SOFT_RESET, grbm_reset);
		2269	(void)RREG32(GRBM_SOFT_RESET);
		2270	udelay(50);
		2271	WREG32(GRBM_SOFT_RESET, 0);
		2272	(void)RREG32(GRBM_SOFT_RESET);
		2273	/* Wait a little for things to settle down */
		2274	udelay(50);
		2275	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2276	RREG32(GRBM_STATUS));
		2277	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2278	RREG32(GRBM_STATUS_SE0));
		2279	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2280	RREG32(GRBM_STATUS_SE1));
		2281	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2282	RREG32(SRBM_STATUS));
2997	Serge	2283	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		2284	RREG32(CP_STALLED_STAT1));
		2285	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		2286	RREG32(CP_STALLED_STAT2));
		2287	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		2288	RREG32(CP_BUSY_STAT));
		2289	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		2290	RREG32(CP_STAT));
1963	serge	2291	evergreen_mc_resume(rdev, &save);
1430	serge	2292	return 0;
		2293	}
		2294
1963	serge	2295	int evergreen_asic_reset(struct radeon_device *rdev)
		2296	{
		2297	return evergreen_gpu_soft_reset(rdev);
		2298	}
		2299
		2300	/* Interrupts */
		2301
		2302	u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
		2303	{
2997	Serge	2304	if (crtc >= rdev->num_crtc)
3031	serge	2305	return 0;
2997	Serge	2306	else
		2307	return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
1963	serge	2308	}
		2309
		2310	void evergreen_disable_interrupt_state(struct radeon_device *rdev)
		2311	{
		2312	u32 tmp;
		2313
2997	Serge	2314	if (rdev->family >= CHIP_CAYMAN) {
		2315	cayman_cp_int_cntl_setup(rdev, 0,
		2316	CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		2317	cayman_cp_int_cntl_setup(rdev, 1, 0);
		2318	cayman_cp_int_cntl_setup(rdev, 2, 0);
		2319	} else
3031	serge	2320	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
1963	serge	2321	WREG32(GRBM_INT_CNTL, 0);
		2322	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2323	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005	serge	2324	if (rdev->num_crtc >= 4) {
3031	serge	2325	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2326	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005	serge	2327	}
		2328	if (rdev->num_crtc >= 6) {
3031	serge	2329	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2330	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	2331	}
		2332
		2333	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2334	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005	serge	2335	if (rdev->num_crtc >= 4) {
3031	serge	2336	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2337	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005	serge	2338	}
		2339	if (rdev->num_crtc >= 6) {
3031	serge	2340	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2341	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	2342	}
		2343
2997	Serge	2344	/* only one DAC on DCE6 */
		2345	if (!ASIC_IS_DCE6(rdev))
3031	serge	2346	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
1963	serge	2347	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2348
		2349	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2350	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2351	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2352	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2353	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2354	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2355	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2356	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2357	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2358	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2359	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2360	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2361
		2362	}
2005	serge	2363
		2364	int evergreen_irq_set(struct radeon_device *rdev)
		2365	{
		2366	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
2997	Serge	2367	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
2005	serge	2368	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
		2369	u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
		2370	u32 grbm_int_cntl = 0;
		2371	u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2997	Serge	2372	u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
2005	serge	2373
		2374	if (!rdev->irq.installed) {
		2375	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		2376	return -EINVAL;
		2377	}
		2378	/* don't enable anything if the ih is disabled */
		2379	if (!rdev->ih.enabled) {
		2380	r600_disable_interrupts(rdev);
		2381	/* force the active interrupt state to all disabled */
		2382	evergreen_disable_interrupt_state(rdev);
		2383	return 0;
		2384	}
		2385
		2386	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2387	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2388	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2389	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2390	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2391	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2392
2997	Serge	2393	afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2394	afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2395	afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2396	afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2397	afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2398	afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2399
		2400	if (rdev->family >= CHIP_CAYMAN) {
		2401	/* enable CP interrupts on all rings */
		2402	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		2403	DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
		2404	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		2405	}
		2406	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
		2407	DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
		2408	cp_int_cntl1 \|= TIME_STAMP_INT_ENABLE;
		2409	}
		2410	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
		2411	DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
		2412	cp_int_cntl2 \|= TIME_STAMP_INT_ENABLE;
		2413	}
		2414	} else {
		2415	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		2416	DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
3031	serge	2417	cp_int_cntl \|= RB_INT_ENABLE;
		2418	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		2419	}
2005	serge	2420	}
2997	Serge	2421
2005	serge	2422	if (rdev->irq.crtc_vblank_int[0] \|\|
2997	Serge	2423	atomic_read(&rdev->irq.pflip[0])) {
2005	serge	2424	DRM_DEBUG("evergreen_irq_set: vblank 0\n");
		2425	crtc1 \|= VBLANK_INT_MASK;
		2426	}
		2427	if (rdev->irq.crtc_vblank_int[1] \|\|
2997	Serge	2428	atomic_read(&rdev->irq.pflip[1])) {
2005	serge	2429	DRM_DEBUG("evergreen_irq_set: vblank 1\n");
		2430	crtc2 \|= VBLANK_INT_MASK;
		2431	}
		2432	if (rdev->irq.crtc_vblank_int[2] \|\|
2997	Serge	2433	atomic_read(&rdev->irq.pflip[2])) {
2005	serge	2434	DRM_DEBUG("evergreen_irq_set: vblank 2\n");
		2435	crtc3 \|= VBLANK_INT_MASK;
		2436	}
		2437	if (rdev->irq.crtc_vblank_int[3] \|\|
2997	Serge	2438	atomic_read(&rdev->irq.pflip[3])) {
2005	serge	2439	DRM_DEBUG("evergreen_irq_set: vblank 3\n");
		2440	crtc4 \|= VBLANK_INT_MASK;
		2441	}
		2442	if (rdev->irq.crtc_vblank_int[4] \|\|
2997	Serge	2443	atomic_read(&rdev->irq.pflip[4])) {
2005	serge	2444	DRM_DEBUG("evergreen_irq_set: vblank 4\n");
		2445	crtc5 \|= VBLANK_INT_MASK;
		2446	}
		2447	if (rdev->irq.crtc_vblank_int[5] \|\|
2997	Serge	2448	atomic_read(&rdev->irq.pflip[5])) {
2005	serge	2449	DRM_DEBUG("evergreen_irq_set: vblank 5\n");
		2450	crtc6 \|= VBLANK_INT_MASK;
		2451	}
		2452	if (rdev->irq.hpd[0]) {
		2453	DRM_DEBUG("evergreen_irq_set: hpd 1\n");
		2454	hpd1 \|= DC_HPDx_INT_EN;
		2455	}
		2456	if (rdev->irq.hpd[1]) {
		2457	DRM_DEBUG("evergreen_irq_set: hpd 2\n");
		2458	hpd2 \|= DC_HPDx_INT_EN;
		2459	}
		2460	if (rdev->irq.hpd[2]) {
		2461	DRM_DEBUG("evergreen_irq_set: hpd 3\n");
		2462	hpd3 \|= DC_HPDx_INT_EN;
		2463	}
		2464	if (rdev->irq.hpd[3]) {
		2465	DRM_DEBUG("evergreen_irq_set: hpd 4\n");
		2466	hpd4 \|= DC_HPDx_INT_EN;
		2467	}
		2468	if (rdev->irq.hpd[4]) {
		2469	DRM_DEBUG("evergreen_irq_set: hpd 5\n");
		2470	hpd5 \|= DC_HPDx_INT_EN;
		2471	}
		2472	if (rdev->irq.hpd[5]) {
		2473	DRM_DEBUG("evergreen_irq_set: hpd 6\n");
		2474	hpd6 \|= DC_HPDx_INT_EN;
		2475	}
2997	Serge	2476	if (rdev->irq.afmt[0]) {
		2477	DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
		2478	afmt1 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
2005	serge	2479	}
2997	Serge	2480	if (rdev->irq.afmt[1]) {
		2481	DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
		2482	afmt2 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2483	}
		2484	if (rdev->irq.afmt[2]) {
		2485	DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
		2486	afmt3 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2487	}
		2488	if (rdev->irq.afmt[3]) {
		2489	DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
		2490	afmt4 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2491	}
		2492	if (rdev->irq.afmt[4]) {
		2493	DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
		2494	afmt5 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2495	}
		2496	if (rdev->irq.afmt[5]) {
		2497	DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
		2498	afmt6 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2499	}
2005	serge	2500
2997	Serge	2501	if (rdev->family >= CHIP_CAYMAN) {
		2502	cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
		2503	cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
		2504	cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
		2505	} else
3031	serge	2506	WREG32(CP_INT_CNTL, cp_int_cntl);
2005	serge	2507	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		2508
		2509	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
		2510	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
		2511	if (rdev->num_crtc >= 4) {
		2512	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
		2513	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
		2514	}
		2515	if (rdev->num_crtc >= 6) {
		2516	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
		2517	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
		2518	}
		2519
		2520	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
		2521	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
		2522	if (rdev->num_crtc >= 4) {
		2523	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
		2524	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
		2525	}
		2526	if (rdev->num_crtc >= 6) {
		2527	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
		2528	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
		2529	}
		2530
		2531	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		2532	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		2533	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		2534	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		2535	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		2536	WREG32(DC_HPD6_INT_CONTROL, hpd6);
		2537
2997	Serge	2538	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
		2539	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
		2540	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
		2541	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
		2542	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
		2543	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
		2544
2005	serge	2545	return 0;
		2546	}
		2547
2997	Serge	2548	static void evergreen_irq_ack(struct radeon_device *rdev)
2005	serge	2549	{
		2550	u32 tmp;
		2551
		2552	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		2553	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		2554	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
		2555	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
		2556	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
		2557	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
		2558	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2559	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2560	if (rdev->num_crtc >= 4) {
		2561	rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2562	rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2563	}
		2564	if (rdev->num_crtc >= 6) {
		2565	rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2566	rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2567	}
		2568
2997	Serge	2569	rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2570	rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2571	rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2572	rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2573	rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2574	rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2575
2005	serge	2576	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
		2577	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2578	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
		2579	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2580	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
		2581	WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
		2582	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
		2583	WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
		2584	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
		2585	WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
		2586	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
		2587	WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
		2588
		2589	if (rdev->num_crtc >= 4) {
		2590	if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
		2591	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2592	if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
		2593	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2594	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
		2595	WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
		2596	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
		2597	WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
		2598	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
		2599	WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
		2600	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
		2601	WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
		2602	}
		2603
		2604	if (rdev->num_crtc >= 6) {
		2605	if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
		2606	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2607	if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
		2608	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2609	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
		2610	WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
		2611	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
		2612	WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
		2613	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
		2614	WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
		2615	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
		2616	WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
		2617	}
		2618
		2619	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		2620	tmp = RREG32(DC_HPD1_INT_CONTROL);
		2621	tmp \|= DC_HPDx_INT_ACK;
		2622	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2623	}
		2624	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		2625	tmp = RREG32(DC_HPD2_INT_CONTROL);
		2626	tmp \|= DC_HPDx_INT_ACK;
		2627	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2628	}
		2629	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		2630	tmp = RREG32(DC_HPD3_INT_CONTROL);
		2631	tmp \|= DC_HPDx_INT_ACK;
		2632	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2633	}
		2634	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		2635	tmp = RREG32(DC_HPD4_INT_CONTROL);
		2636	tmp \|= DC_HPDx_INT_ACK;
		2637	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2638	}
		2639	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		2640	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2641	tmp \|= DC_HPDx_INT_ACK;
		2642	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2643	}
		2644	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		2645	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2646	tmp \|= DC_HPDx_INT_ACK;
		2647	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2648	}
2997	Serge	2649	if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
		2650	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2651	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2652	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
		2653	}
		2654	if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
		2655	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2656	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2657	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
		2658	}
		2659	if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
		2660	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2661	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2662	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
		2663	}
		2664	if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
		2665	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2666	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2667	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
		2668	}
		2669	if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
		2670	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2671	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2672	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
		2673	}
		2674	if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
		2675	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2676	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2677	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
		2678	}
2005	serge	2679	}
2997	Serge	2680
		2681	static void evergreen_irq_disable(struct radeon_device *rdev)
2005	serge	2682	{
2997	Serge	2683	r600_disable_interrupts(rdev);
		2684	/* Wait and acknowledge irq */
		2685	mdelay(1);
		2686	evergreen_irq_ack(rdev);
		2687	evergreen_disable_interrupt_state(rdev);
		2688	}
		2689
		2690	void evergreen_irq_suspend(struct radeon_device *rdev)
		2691	{
		2692	evergreen_irq_disable(rdev);
		2693	r600_rlc_stop(rdev);
		2694	}
		2695
		2696	static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
		2697	{
2005	serge	2698	u32 wptr, tmp;
		2699
		2700	if (rdev->wb.enabled)
		2701	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		2702	else
		2703	wptr = RREG32(IH_RB_WPTR);
		2704
		2705	if (wptr & RB_OVERFLOW) {
		2706	/* When a ring buffer overflow happen start parsing interrupt
		2707	* from the last not overwritten vector (wptr + 16). Hopefully
		2708	* this should allow us to catchup.
		2709	*/
		2710	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
		2711	wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
		2712	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		2713	tmp = RREG32(IH_RB_CNTL);
		2714	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		2715	WREG32(IH_RB_CNTL, tmp);
		2716	}
		2717	return (wptr & rdev->ih.ptr_mask);
		2718	}
		2719
		2720	int evergreen_irq_process(struct radeon_device *rdev)
		2721	{
		2722	u32 wptr;
		2723	u32 rptr;
		2724	u32 src_id, src_data;
		2725	u32 ring_index;
		2726	bool queue_hotplug = false;
2997	Serge	2727	bool queue_hdmi = false;
2005	serge	2728
		2729	if (!rdev->ih.enabled \|\| rdev->shutdown)
		2730	return IRQ_NONE;
		2731
		2732	wptr = evergreen_get_ih_wptr(rdev);
2997	Serge	2733
		2734	restart_ih:
		2735	/* is somebody else already processing irqs? */
		2736	if (atomic_xchg(&rdev->ih.lock, 1))
		2737	return IRQ_NONE;
		2738
2005	serge	2739	rptr = rdev->ih.rptr;
		2740	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		2741
2175	serge	2742	/* Order reading of wptr vs. reading of IH ring data */
		2743	rmb();
		2744
2005	serge	2745	/* display interrupts */
		2746	evergreen_irq_ack(rdev);
		2747
		2748	while (rptr != wptr) {
		2749	/* wptr/rptr are in bytes! */
		2750	ring_index = rptr / 4;
		2751	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		2752	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		2753
		2754	switch (src_id) {
		2755	case 1: /* D1 vblank/vline */
		2756	switch (src_data) {
		2757	case 0: /* D1 vblank */
		2758	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
		2759	if (rdev->irq.crtc_vblank_int[0]) {
		2760	// drm_handle_vblank(rdev->ddev, 0);
		2761	rdev->pm.vblank_sync = true;
		2762	// wake_up(&rdev->irq.vblank_queue);
		2763	}
		2764	// if (rdev->irq.pflip[0])
		2765	// radeon_crtc_handle_flip(rdev, 0);
		2766	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		2767	DRM_DEBUG("IH: D1 vblank\n");
		2768	}
		2769	break;
		2770	case 1: /* D1 vline */
		2771	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
		2772	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		2773	DRM_DEBUG("IH: D1 vline\n");
		2774	}
		2775	break;
		2776	default:
		2777	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2778	break;
		2779	}
		2780	break;
		2781	case 2: /* D2 vblank/vline */
		2782	switch (src_data) {
		2783	case 0: /* D2 vblank */
		2784	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
		2785	if (rdev->irq.crtc_vblank_int[1]) {
		2786	// drm_handle_vblank(rdev->ddev, 1);
		2787	rdev->pm.vblank_sync = true;
		2788	// wake_up(&rdev->irq.vblank_queue);
		2789	}
		2790	// if (rdev->irq.pflip[1])
		2791	// radeon_crtc_handle_flip(rdev, 1);
		2792	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
		2793	DRM_DEBUG("IH: D2 vblank\n");
		2794	}
		2795	break;
		2796	case 1: /* D2 vline */
		2797	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
		2798	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
		2799	DRM_DEBUG("IH: D2 vline\n");
		2800	}
		2801	break;
		2802	default:
		2803	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2804	break;
		2805	}
		2806	break;
		2807	case 3: /* D3 vblank/vline */
		2808	switch (src_data) {
		2809	case 0: /* D3 vblank */
		2810	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
		2811	if (rdev->irq.crtc_vblank_int[2]) {
		2812	// drm_handle_vblank(rdev->ddev, 2);
		2813	rdev->pm.vblank_sync = true;
		2814	// wake_up(&rdev->irq.vblank_queue);
		2815	}
		2816	// if (rdev->irq.pflip[2])
		2817	// radeon_crtc_handle_flip(rdev, 2);
		2818	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
		2819	DRM_DEBUG("IH: D3 vblank\n");
		2820	}
		2821	break;
		2822	case 1: /* D3 vline */
		2823	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
		2824	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
		2825	DRM_DEBUG("IH: D3 vline\n");
		2826	}
		2827	break;
		2828	default:
		2829	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2830	break;
		2831	}
		2832	break;
		2833	case 4: /* D4 vblank/vline */
		2834	switch (src_data) {
		2835	case 0: /* D4 vblank */
		2836	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
		2837	if (rdev->irq.crtc_vblank_int[3]) {
		2838	// drm_handle_vblank(rdev->ddev, 3);
		2839	rdev->pm.vblank_sync = true;
		2840	// wake_up(&rdev->irq.vblank_queue);
		2841	}
		2842	// if (rdev->irq.pflip[3])
		2843	// radeon_crtc_handle_flip(rdev, 3);
		2844	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
		2845	DRM_DEBUG("IH: D4 vblank\n");
		2846	}
		2847	break;
		2848	case 1: /* D4 vline */
		2849	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
		2850	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
		2851	DRM_DEBUG("IH: D4 vline\n");
		2852	}
		2853	break;
		2854	default:
		2855	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2856	break;
		2857	}
		2858	break;
		2859	case 5: /* D5 vblank/vline */
		2860	switch (src_data) {
		2861	case 0: /* D5 vblank */
		2862	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
		2863	if (rdev->irq.crtc_vblank_int[4]) {
		2864	// drm_handle_vblank(rdev->ddev, 4);
		2865	rdev->pm.vblank_sync = true;
		2866	// wake_up(&rdev->irq.vblank_queue);
		2867	}
		2868	// if (rdev->irq.pflip[4])
		2869	// radeon_crtc_handle_flip(rdev, 4);
		2870	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
		2871	DRM_DEBUG("IH: D5 vblank\n");
		2872	}
		2873	break;
		2874	case 1: /* D5 vline */
		2875	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
		2876	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
		2877	DRM_DEBUG("IH: D5 vline\n");
		2878	}
		2879	break;
		2880	default:
		2881	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2882	break;
		2883	}
		2884	break;
		2885	case 6: /* D6 vblank/vline */
		2886	switch (src_data) {
		2887	case 0: /* D6 vblank */
		2888	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
		2889	if (rdev->irq.crtc_vblank_int[5]) {
		2890	// drm_handle_vblank(rdev->ddev, 5);
		2891	rdev->pm.vblank_sync = true;
		2892	// wake_up(&rdev->irq.vblank_queue);
		2893	}
		2894	// if (rdev->irq.pflip[5])
		2895	// radeon_crtc_handle_flip(rdev, 5);
		2896	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
		2897	DRM_DEBUG("IH: D6 vblank\n");
		2898	}
		2899	break;
		2900	case 1: /* D6 vline */
		2901	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
		2902	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
		2903	DRM_DEBUG("IH: D6 vline\n");
		2904	}
		2905	break;
		2906	default:
		2907	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2908	break;
		2909	}
		2910	break;
		2911	case 42: /* HPD hotplug */
		2912	switch (src_data) {
		2913	case 0:
		2914	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		2915	rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
		2916	queue_hotplug = true;
		2917	DRM_DEBUG("IH: HPD1\n");
		2918	}
		2919	break;
		2920	case 1:
		2921	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		2922	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
		2923	queue_hotplug = true;
		2924	DRM_DEBUG("IH: HPD2\n");
		2925	}
		2926	break;
		2927	case 2:
		2928	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		2929	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
		2930	queue_hotplug = true;
		2931	DRM_DEBUG("IH: HPD3\n");
		2932	}
		2933	break;
		2934	case 3:
		2935	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		2936	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
		2937	queue_hotplug = true;
		2938	DRM_DEBUG("IH: HPD4\n");
		2939	}
		2940	break;
		2941	case 4:
		2942	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		2943	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
		2944	queue_hotplug = true;
		2945	DRM_DEBUG("IH: HPD5\n");
		2946	}
		2947	break;
		2948	case 5:
		2949	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		2950	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
		2951	queue_hotplug = true;
		2952	DRM_DEBUG("IH: HPD6\n");
		2953	}
		2954	break;
		2955	default:
		2956	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2957	break;
		2958	}
		2959	break;
2997	Serge	2960	case 44: /* hdmi */
		2961	switch (src_data) {
		2962	case 0:
		2963	if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
		2964	rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
		2965	queue_hdmi = true;
		2966	DRM_DEBUG("IH: HDMI0\n");
		2967	}
		2968	break;
		2969	case 1:
		2970	if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
		2971	rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
		2972	queue_hdmi = true;
		2973	DRM_DEBUG("IH: HDMI1\n");
		2974	}
		2975	break;
		2976	case 2:
		2977	if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
		2978	rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
		2979	queue_hdmi = true;
		2980	DRM_DEBUG("IH: HDMI2\n");
		2981	}
		2982	break;
		2983	case 3:
		2984	if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
		2985	rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
		2986	queue_hdmi = true;
		2987	DRM_DEBUG("IH: HDMI3\n");
		2988	}
		2989	break;
		2990	case 4:
		2991	if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
		2992	rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
		2993	queue_hdmi = true;
		2994	DRM_DEBUG("IH: HDMI4\n");
		2995	}
		2996	break;
		2997	case 5:
		2998	if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
		2999	rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
		3000	queue_hdmi = true;
		3001	DRM_DEBUG("IH: HDMI5\n");
		3002	}
		3003	break;
		3004	default:
		3005	DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
		3006	break;
		3007	}
		3008	break;
2005	serge	3009	case 176: /* CP_INT in ring buffer */
		3010	case 177: /* CP_INT in IB1 */
		3011	case 178: /* CP_INT in IB2 */
		3012	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2997	Serge	3013	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2005	serge	3014	break;
		3015	case 181: /* CP EOP event */
		3016	DRM_DEBUG("IH: CP EOP\n");
2997	Serge	3017	if (rdev->family >= CHIP_CAYMAN) {
		3018	switch (src_data) {
		3019	case 0:
		3020	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
		3021	break;
		3022	case 1:
		3023	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		3024	break;
		3025	case 2:
		3026	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		3027	break;
		3028	}
		3029	} else
		3030	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2005	serge	3031	break;
		3032	case 233: /* GUI IDLE */
		3033	DRM_DEBUG("IH: GUI idle\n");
		3034	break;
		3035	default:
		3036	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3037	break;
		3038	}
		3039
		3040	/* wptr/rptr are in bytes! */
		3041	rptr += 16;
		3042	rptr &= rdev->ih.ptr_mask;
		3043	}
2997	Serge	3044	rdev->ih.rptr = rptr;
		3045	WREG32(IH_RB_RPTR, rdev->ih.rptr);
		3046	atomic_set(&rdev->ih.lock, 0);
		3047
2005	serge	3048	/* make sure wptr hasn't changed while processing */
		3049	wptr = evergreen_get_ih_wptr(rdev);
2997	Serge	3050	if (wptr != rptr)
2005	serge	3051	goto restart_ih;
2997	Serge	3052
2005	serge	3053	return IRQ_HANDLED;
		3054	}
		3055
1430	serge	3056	static int evergreen_startup(struct radeon_device *rdev)
		3057	{
2997	Serge	3058	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1430	serge	3059	int r;
		3060
1990	serge	3061	/* enable pcie gen2 link */
3031	serge	3062	evergreen_pcie_gen2_enable(rdev);
1990	serge	3063
		3064	if (ASIC_IS_DCE5(rdev)) {
		3065	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw \|\| !rdev->mc_fw) {
		3066	r = ni_init_microcode(rdev);
		3067	if (r) {
		3068	DRM_ERROR("Failed to load firmware!\n");
		3069	return r;
		3070	}
		3071	}
		3072	r = ni_mc_load_microcode(rdev);
		3073	if (r) {
		3074	DRM_ERROR("Failed to load MC firmware!\n");
		3075	return r;
		3076	}
		3077	} else {
3031	serge	3078	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		3079	r = r600_init_microcode(rdev);
		3080	if (r) {
		3081	DRM_ERROR("Failed to load firmware!\n");
		3082	return r;
		3083	}
1430	serge	3084	}
		3085	}
1963	serge	3086
2997	Serge	3087	r = r600_vram_scratch_init(rdev);
		3088	if (r)
		3089	return r;
		3090
1430	serge	3091	evergreen_mc_program(rdev);
		3092	if (rdev->flags & RADEON_IS_AGP) {
1963	serge	3093	evergreen_agp_enable(rdev);
1430	serge	3094	} else {
		3095	r = evergreen_pcie_gart_enable(rdev);
		3096	if (r)
		3097	return r;
		3098	}
		3099	evergreen_gpu_init(rdev);
2005	serge	3100
1963	serge	3101	r = evergreen_blit_init(rdev);
		3102	if (r) {
2997	Serge	3103	// r600_blit_fini(rdev);
		3104	rdev->asic->copy.copy = NULL;
1963	serge	3105	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		3106	}
1430	serge	3107
1963	serge	3108	/* allocate wb buffer */
		3109	r = radeon_wb_init(rdev);
		3110	if (r)
		3111	return r;
		3112
		3113	/* Enable IRQ */
2005	serge	3114	r = r600_irq_init(rdev);
		3115	if (r) {
		3116	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		3117	// radeon_irq_kms_fini(rdev);
		3118	return r;
		3119	}
		3120	evergreen_irq_set(rdev);
1963	serge	3121
2997	Serge	3122	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
		3123	R600_CP_RB_RPTR, R600_CP_RB_WPTR,
		3124	0, 0xfffff, RADEON_CP_PACKET2);
1430	serge	3125	if (r)
		3126	return r;
		3127	r = evergreen_cp_load_microcode(rdev);
		3128	if (r)
		3129	return r;
1963	serge	3130	r = evergreen_cp_resume(rdev);
1430	serge	3131	if (r)
		3132	return r;
1963	serge	3133
1430	serge	3134	return 0;
		3135	}
		3136
		3137
		3138
2997	Serge	3139	#if 0
1430	serge	3140
2005	serge	3141	int evergreen_copy_blit(struct radeon_device *rdev,
		3142	uint64_t src_offset, uint64_t dst_offset,
		3143	unsigned num_pages, struct radeon_fence *fence)
		3144	{
		3145	int r;
1430	serge	3146
2005	serge	3147	mutex_lock(&rdev->r600_blit.mutex);
		3148	rdev->r600_blit.vb_ib = NULL;
		3149	r = evergreen_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
		3150	if (r) {
		3151	if (rdev->r600_blit.vb_ib)
		3152	radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
		3153	mutex_unlock(&rdev->r600_blit.mutex);
		3154	return r;
		3155	}
		3156	evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
		3157	evergreen_blit_done_copy(rdev, fence);
		3158	mutex_unlock(&rdev->r600_blit.mutex);
		3159	return 0;
		3160	}
2997	Serge	3161	#endif
1430	serge	3162
		3163	/* Plan is to move initialization in that function and use
		3164	* helper function so that radeon_device_init pretty much
		3165	* do nothing more than calling asic specific function. This
		3166	* should also allow to remove a bunch of callback function
		3167	* like vram_info.
		3168	*/
		3169	int evergreen_init(struct radeon_device *rdev)
		3170	{
		3171	int r;
		3172
		3173	/* Read BIOS */
		3174	if (!radeon_get_bios(rdev)) {
		3175	if (ASIC_IS_AVIVO(rdev))
		3176	return -EINVAL;
		3177	}
		3178	/* Must be an ATOMBIOS */
		3179	if (!rdev->is_atom_bios) {
1986	serge	3180	dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
1430	serge	3181	return -EINVAL;
		3182	}
		3183	r = radeon_atombios_init(rdev);
		3184	if (r)
		3185	return r;
1986	serge	3186	/* reset the asic, the gfx blocks are often in a bad state
		3187	* after the driver is unloaded or after a resume
		3188	*/
		3189	if (radeon_asic_reset(rdev))
		3190	dev_warn(rdev->dev, "GPU reset failed !\n");
1430	serge	3191	/* Post card if necessary */
1986	serge	3192	if (!radeon_card_posted(rdev)) {
1430	serge	3193	if (!rdev->bios) {
		3194	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		3195	return -EINVAL;
		3196	}
		3197	DRM_INFO("GPU not posted. posting now...\n");
		3198	atom_asic_init(rdev->mode_info.atom_context);
		3199	}
		3200	/* Initialize scratch registers */
		3201	r600_scratch_init(rdev);
		3202	/* Initialize surface registers */
		3203	radeon_surface_init(rdev);
		3204	/* Initialize clocks */
		3205	radeon_get_clock_info(rdev->ddev);
		3206	/* Fence driver */
2005	serge	3207	r = radeon_fence_driver_init(rdev);
		3208	if (r)
		3209	return r;
3031	serge	3210	/* initialize AGP */
1430	serge	3211	if (rdev->flags & RADEON_IS_AGP) {
		3212	r = radeon_agp_init(rdev);
		3213	if (r)
		3214	radeon_agp_disable(rdev);
		3215	}
		3216	/* initialize memory controller */
		3217	r = evergreen_mc_init(rdev);
		3218	if (r)
		3219	return r;
		3220	/* Memory manager */
		3221	r = radeon_bo_init(rdev);
		3222	if (r)
		3223	return r;
1963	serge	3224
2005	serge	3225	r = radeon_irq_kms_init(rdev);
		3226	if (r)
		3227	return r;
1430	serge	3228
2997	Serge	3229	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
		3230	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1430	serge	3231
2005	serge	3232	rdev->ih.ring_obj = NULL;
		3233	r600_ih_ring_init(rdev, 64 * 1024);
1430	serge	3234
		3235	r = r600_pcie_gart_init(rdev);
		3236	if (r)
		3237	return r;
1963	serge	3238
		3239	rdev->accel_working = true;
1430	serge	3240	r = evergreen_startup(rdev);
		3241	if (r) {
1963	serge	3242	dev_err(rdev->dev, "disabling GPU acceleration\n");
1430	serge	3243	rdev->accel_working = false;
		3244	}
2997	Serge	3245
		3246	/* Don't start up if the MC ucode is missing on BTC parts.
		3247	* The default clocks and voltages before the MC ucode
		3248	* is loaded are not suffient for advanced operations.
		3249	*/
		3250	if (ASIC_IS_DCE5(rdev)) {
		3251	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
		3252	DRM_ERROR("radeon: MC ucode required for NI+.\n");
		3253	return -EINVAL;
2005	serge	3254	}
1430	serge	3255	}
2997	Serge	3256
1430	serge	3257	return 0;
		3258	}
		3259
1986	serge	3260
2997	Serge	3261	void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
1430	serge	3262	{
2997	Serge	3263	u32 link_width_cntl, speed_cntl, mask;
		3264	int ret;
1986	serge	3265
		3266	if (radeon_pcie_gen2 == 0)
		3267	return;
		3268
		3269	if (rdev->flags & RADEON_IS_IGP)
		3270	return;
		3271
		3272	if (!(rdev->flags & RADEON_IS_PCIE))
		3273	return;
		3274
		3275	/* x2 cards have a special sequence */
		3276	if (ASIC_IS_X2(rdev))
		3277	return;
		3278
2997	Serge	3279	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
		3280	if (ret != 0)
		3281	return;
		3282
		3283	if (!(mask & DRM_PCIE_SPEED_50))
		3284	return;
		3285
1986	serge	3286	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
2997	Serge	3287	if (speed_cntl & LC_CURRENT_DATA_RATE) {
		3288	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
		3289	return;
		3290	}
		3291
		3292	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
		3293
1986	serge	3294	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) \|\|
		3295	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		3296
		3297	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3298	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3299	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3300
		3301	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3302	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		3303	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3304
		3305	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3306	speed_cntl \|= LC_CLR_FAILED_SPD_CHANGE_CNT;
		3307	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3308
		3309	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3310	speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
		3311	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3312
		3313	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3314	speed_cntl \|= LC_GEN2_EN_STRAP;
		3315	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3316
		3317	} else {
		3318	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3319	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		3320	if (1)
		3321	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3322	else
		3323	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3324	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3325	}
1430	serge	3326	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/evergreen.c @ 3031 – Rev 3120