WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/evergreen.c

Rev	Author	Line No.	Line
1430	serge	1	/*
		2	* Copyright 2010 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	//#include
1963	serge	26	#include
2997	Serge	27	#include
1430	serge	28	#include "radeon.h"
1963	serge	29	#include "radeon_asic.h"
2997	Serge	30	#include
1963	serge	31	#include "evergreend.h"
1430	serge	32	#include "atom.h"
		33	#include "avivod.h"
		34	#include "evergreen_reg.h"
1986	serge	35	#include "evergreen_blit_shaders.h"
1430	serge	36
1963	serge	37	#define EVERGREEN_PFP_UCODE_SIZE 1120
		38	#define EVERGREEN_PM4_UCODE_SIZE 1376
		39
2997	Serge	40	static const u32 crtc_offsets[6] =
		41	{
		42	EVERGREEN_CRTC0_REGISTER_OFFSET,
		43	EVERGREEN_CRTC1_REGISTER_OFFSET,
		44	EVERGREEN_CRTC2_REGISTER_OFFSET,
		45	EVERGREEN_CRTC3_REGISTER_OFFSET,
		46	EVERGREEN_CRTC4_REGISTER_OFFSET,
		47	EVERGREEN_CRTC5_REGISTER_OFFSET
		48	};
		49
1430	serge	50	static void evergreen_gpu_init(struct radeon_device *rdev);
		51	void evergreen_fini(struct radeon_device *rdev);
2997	Serge	52	void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
		53	extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
		54	int ring, u32 cp_int_cntl);
1430	serge	55
2997	Serge	56	void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
		57	unsigned bankh, unsigned mtaspect,
		58	unsigned *tile_split)
		59	{
		60	*bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
		61	*bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
		62	*mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
		63	*tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
		64	switch (*bankw) {
		65	default:
		66	case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
		67	case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
		68	case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
		69	case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
		70	}
		71	switch (*bankh) {
		72	default:
		73	case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
		74	case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
		75	case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
		76	case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
		77	}
		78	switch (*mtaspect) {
		79	default:
		80	case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
		81	case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
		82	case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
		83	case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
		84	}
		85	}
1990	serge	86
2997	Serge	87	void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
		88	{
		89	u16 ctl, v;
		90	int err;
1990	serge	91
2997	Serge	92	err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
		93	if (err)
		94	return;
		95
		96	v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
		97
		98	/* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
		99	* to avoid hangs or perfomance issues
		100	*/
		101	if ((v == 0) \|\| (v == 6) \|\| (v == 7)) {
		102	ctl &= ~PCI_EXP_DEVCTL_READRQ;
		103	ctl \|= (2 << 12);
		104	pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
		105	}
		106	}
		107
		108	/**
		109	* dce4_wait_for_vblank - vblank wait asic callback.
		110	*
		111	* @rdev: radeon_device pointer
		112	* @crtc: crtc to wait for vblank on
		113	*
		114	* Wait for vblank on the requested crtc (evergreen+).
		115	*/
		116	void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
		117	{
		118	int i;
		119
		120	if (crtc >= rdev->num_crtc)
		121	return;
		122
		123	if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN) {
		124	for (i = 0; i < rdev->usec_timeout; i++) {
		125	if (!(RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK))
		126	break;
		127	udelay(1);
		128	}
		129	for (i = 0; i < rdev->usec_timeout; i++) {
		130	if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
		131	break;
		132	udelay(1);
		133	}
		134	}
		135	}
		136
		137
		138	/**
		139	* evergreen_page_flip - pageflip callback.
		140	*
		141	* @rdev: radeon_device pointer
		142	* @crtc_id: crtc to cleanup pageflip on
		143	* @crtc_base: new address of the crtc (GPU MC address)
		144	*
		145	* Does the actual pageflip (evergreen+).
		146	* During vblank we take the crtc lock and wait for the update_pending
		147	* bit to go high, when it does, we release the lock, and allow the
		148	* double buffered update to take place.
		149	* Returns the current update pending status.
		150	*/
1990	serge	151	u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
		152	{
		153	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
		154	u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
2997	Serge	155	int i;
1990	serge	156
		157	/* Lock the graphics update lock */
		158	tmp \|= EVERGREEN_GRPH_UPDATE_LOCK;
		159	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		160
		161	/* update the scanout addresses */
		162	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		163	upper_32_bits(crtc_base));
		164	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		165	(u32)crtc_base);
		166
		167	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		168	upper_32_bits(crtc_base));
		169	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		170	(u32)crtc_base);
		171
		172	/* Wait for update_pending to go high. */
2997	Serge	173	for (i = 0; i < rdev->usec_timeout; i++) {
		174	if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
		175	break;
		176	udelay(1);
		177	}
1990	serge	178	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
		179
		180	/* Unlock the lock, so double-buffering can take place inside vblank */
		181	tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
		182	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		183
		184	/* Return current update_pending status: */
		185	return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
		186	}
		187
		188	/* get temperature in millidegrees */
		189	int evergreen_get_temp(struct radeon_device *rdev)
		190	{
		191	u32 temp, toffset;
		192	int actual_temp = 0;
		193
		194	if (rdev->family == CHIP_JUNIPER) {
		195	toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
		196	TOFFSET_SHIFT;
		197	temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
		198	TS0_ADC_DOUT_SHIFT;
		199
		200	if (toffset & 0x100)
		201	actual_temp = temp / 2 - (0x200 - toffset);
		202	else
		203	actual_temp = temp / 2 + toffset;
		204
		205	actual_temp = actual_temp * 1000;
		206
		207	} else {
		208	temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
		209	ASIC_T_SHIFT;
		210
		211	if (temp & 0x400)
		212	actual_temp = -256;
		213	else if (temp & 0x200)
		214	actual_temp = 255;
		215	else if (temp & 0x100) {
		216	actual_temp = temp & 0x1ff;
		217	actual_temp \|= ~0x1ff;
		218	} else
		219	actual_temp = temp & 0xff;
		220
		221	actual_temp = (actual_temp * 1000) / 2;
		222	}
		223
		224	return actual_temp;
		225	}
		226
		227	int sumo_get_temp(struct radeon_device *rdev)
		228	{
		229	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
		230	int actual_temp = temp - 49;
		231
		232	return actual_temp * 1000;
		233	}
		234
2997	Serge	235	/**
		236	* sumo_pm_init_profile - Initialize power profiles callback.
		237	*
		238	* @rdev: radeon_device pointer
		239	*
		240	* Initialize the power states used in profile mode
		241	* (sumo, trinity, SI).
		242	* Used for profile mode only.
		243	*/
		244	void sumo_pm_init_profile(struct radeon_device *rdev)
		245	{
		246	int idx;
		247
		248	/* default */
		249	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		250	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		251	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		252	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
		253
		254	/* low,mid sh/mh */
		255	if (rdev->flags & RADEON_IS_MOBILITY)
		256	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
		257	else
		258	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		259
		260	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
		261	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
		262	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		263	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		264
		265	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
		266	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
		267	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		268	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		269
		270	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
		271	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
		272	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		273	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
		274
		275	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
		276	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
		277	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		278	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
		279
		280	/* high sh/mh */
		281	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		282	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
		283	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
		284	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		285	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
		286	rdev->pm.power_state[idx].num_clock_modes - 1;
		287
		288	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
		289	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
		290	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		291	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
		292	rdev->pm.power_state[idx].num_clock_modes - 1;
		293	}
		294
		295	/**
		296	* evergreen_pm_misc - set additional pm hw parameters callback.
		297	*
		298	* @rdev: radeon_device pointer
		299	*
		300	* Set non-clock parameters associated with a power state
		301	* (voltage, etc.) (evergreen+).
		302	*/
1990	serge	303	void evergreen_pm_misc(struct radeon_device *rdev)
		304	{
		305	int req_ps_idx = rdev->pm.requested_power_state_index;
		306	int req_cm_idx = rdev->pm.requested_clock_mode_index;
		307	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
		308	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
		309
		310	if (voltage->type == VOLTAGE_SW) {
		311	/* 0xff01 is a flag rather then an actual voltage */
		312	if (voltage->voltage == 0xff01)
		313	return;
		314	if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
		315	radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
		316	rdev->pm.current_vddc = voltage->voltage;
		317	DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
		318	}
		319	/* 0xff01 is a flag rather then an actual voltage */
		320	if (voltage->vddci == 0xff01)
		321	return;
		322	if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
		323	radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
		324	rdev->pm.current_vddci = voltage->vddci;
		325	DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
		326	}
		327	}
		328	}
		329
2997	Serge	330	/**
		331	* evergreen_pm_prepare - pre-power state change callback.
		332	*
		333	* @rdev: radeon_device pointer
		334	*
		335	* Prepare for a power state change (evergreen+).
		336	*/
1990	serge	337	void evergreen_pm_prepare(struct radeon_device *rdev)
		338	{
		339	struct drm_device *ddev = rdev->ddev;
		340	struct drm_crtc *crtc;
		341	struct radeon_crtc *radeon_crtc;
		342	u32 tmp;
		343
		344	/* disable any active CRTCs */
		345	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		346	radeon_crtc = to_radeon_crtc(crtc);
		347	if (radeon_crtc->enabled) {
		348	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		349	tmp \|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		350	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		351	}
		352	}
		353	}
		354
2997	Serge	355	/**
		356	* evergreen_pm_finish - post-power state change callback.
		357	*
		358	* @rdev: radeon_device pointer
		359	*
		360	* Clean up after a power state change (evergreen+).
		361	*/
1990	serge	362	void evergreen_pm_finish(struct radeon_device *rdev)
		363	{
		364	struct drm_device *ddev = rdev->ddev;
		365	struct drm_crtc *crtc;
		366	struct radeon_crtc *radeon_crtc;
		367	u32 tmp;
		368
		369	/* enable any active CRTCs */
		370	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		371	radeon_crtc = to_radeon_crtc(crtc);
		372	if (radeon_crtc->enabled) {
		373	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		374	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		375	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		376	}
		377	}
		378	}
		379
2997	Serge	380	/**
		381	* evergreen_hpd_sense - hpd sense callback.
		382	*
		383	* @rdev: radeon_device pointer
		384	* @hpd: hpd (hotplug detect) pin
		385	*
		386	* Checks if a digital monitor is connected (evergreen+).
		387	* Returns true if connected, false if not connected.
		388	*/
1430	serge	389	bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		390	{
		391	bool connected = false;
1963	serge	392
		393	switch (hpd) {
		394	case RADEON_HPD_1:
		395	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		396	connected = true;
		397	break;
		398	case RADEON_HPD_2:
		399	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		400	connected = true;
		401	break;
		402	case RADEON_HPD_3:
		403	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		404	connected = true;
		405	break;
		406	case RADEON_HPD_4:
		407	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		408	connected = true;
		409	break;
		410	case RADEON_HPD_5:
		411	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		412	connected = true;
		413	break;
		414	case RADEON_HPD_6:
		415	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		416	connected = true;
		417	break;
		418	default:
		419	break;
		420	}
		421
1430	serge	422	return connected;
		423	}
		424
2997	Serge	425	/**
		426	* evergreen_hpd_set_polarity - hpd set polarity callback.
		427	*
		428	* @rdev: radeon_device pointer
		429	* @hpd: hpd (hotplug detect) pin
		430	*
		431	* Set the polarity of the hpd pin (evergreen+).
		432	*/
1430	serge	433	void evergreen_hpd_set_polarity(struct radeon_device *rdev,
		434	enum radeon_hpd_id hpd)
		435	{
1963	serge	436	u32 tmp;
		437	bool connected = evergreen_hpd_sense(rdev, hpd);
		438
		439	switch (hpd) {
		440	case RADEON_HPD_1:
		441	tmp = RREG32(DC_HPD1_INT_CONTROL);
		442	if (connected)
		443	tmp &= ~DC_HPDx_INT_POLARITY;
		444	else
		445	tmp \|= DC_HPDx_INT_POLARITY;
		446	WREG32(DC_HPD1_INT_CONTROL, tmp);
		447	break;
		448	case RADEON_HPD_2:
		449	tmp = RREG32(DC_HPD2_INT_CONTROL);
		450	if (connected)
		451	tmp &= ~DC_HPDx_INT_POLARITY;
		452	else
		453	tmp \|= DC_HPDx_INT_POLARITY;
		454	WREG32(DC_HPD2_INT_CONTROL, tmp);
		455	break;
		456	case RADEON_HPD_3:
		457	tmp = RREG32(DC_HPD3_INT_CONTROL);
		458	if (connected)
		459	tmp &= ~DC_HPDx_INT_POLARITY;
		460	else
		461	tmp \|= DC_HPDx_INT_POLARITY;
		462	WREG32(DC_HPD3_INT_CONTROL, tmp);
		463	break;
		464	case RADEON_HPD_4:
		465	tmp = RREG32(DC_HPD4_INT_CONTROL);
		466	if (connected)
		467	tmp &= ~DC_HPDx_INT_POLARITY;
		468	else
		469	tmp \|= DC_HPDx_INT_POLARITY;
		470	WREG32(DC_HPD4_INT_CONTROL, tmp);
		471	break;
		472	case RADEON_HPD_5:
		473	tmp = RREG32(DC_HPD5_INT_CONTROL);
		474	if (connected)
		475	tmp &= ~DC_HPDx_INT_POLARITY;
		476	else
		477	tmp \|= DC_HPDx_INT_POLARITY;
		478	WREG32(DC_HPD5_INT_CONTROL, tmp);
		479	break;
		480	case RADEON_HPD_6:
		481	tmp = RREG32(DC_HPD6_INT_CONTROL);
		482	if (connected)
		483	tmp &= ~DC_HPDx_INT_POLARITY;
		484	else
		485	tmp \|= DC_HPDx_INT_POLARITY;
		486	WREG32(DC_HPD6_INT_CONTROL, tmp);
		487	break;
		488	default:
		489	break;
		490	}
1430	serge	491	}
		492
2997	Serge	493	/**
		494	* evergreen_hpd_init - hpd setup callback.
		495	*
		496	* @rdev: radeon_device pointer
		497	*
		498	* Setup the hpd pins used by the card (evergreen+).
		499	* Enable the pin, set the polarity, and enable the hpd interrupts.
		500	*/
1430	serge	501	void evergreen_hpd_init(struct radeon_device *rdev)
		502	{
1963	serge	503	struct drm_device *dev = rdev->ddev;
		504	struct drm_connector *connector;
2997	Serge	505	unsigned enabled = 0;
1963	serge	506	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \|
		507	DC_HPDx_RX_INT_TIMER(0xfa) \| DC_HPDx_EN;
		508
		509	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		510	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		511	switch (radeon_connector->hpd.hpd) {
		512	case RADEON_HPD_1:
		513	WREG32(DC_HPD1_CONTROL, tmp);
		514	break;
		515	case RADEON_HPD_2:
		516	WREG32(DC_HPD2_CONTROL, tmp);
		517	break;
		518	case RADEON_HPD_3:
		519	WREG32(DC_HPD3_CONTROL, tmp);
		520	break;
		521	case RADEON_HPD_4:
		522	WREG32(DC_HPD4_CONTROL, tmp);
		523	break;
		524	case RADEON_HPD_5:
		525	WREG32(DC_HPD5_CONTROL, tmp);
		526	break;
		527	case RADEON_HPD_6:
		528	WREG32(DC_HPD6_CONTROL, tmp);
		529	break;
		530	default:
		531	break;
		532	}
2997	Serge	533	radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
		534	enabled \|= 1 << radeon_connector->hpd.hpd;
1963	serge	535	}
2997	Serge	536	// radeon_irq_kms_enable_hpd(rdev, enabled);
1430	serge	537	}
		538
2997	Serge	539	/**
		540	* evergreen_hpd_fini - hpd tear down callback.
		541	*
		542	* @rdev: radeon_device pointer
		543	*
		544	* Tear down the hpd pins used by the card (evergreen+).
		545	* Disable the hpd interrupts.
		546	*/
1963	serge	547	void evergreen_hpd_fini(struct radeon_device *rdev)
		548	{
		549	struct drm_device *dev = rdev->ddev;
		550	struct drm_connector *connector;
2997	Serge	551	unsigned disabled = 0;
1430	serge	552
1963	serge	553	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		554	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		555	switch (radeon_connector->hpd.hpd) {
		556	case RADEON_HPD_1:
		557	WREG32(DC_HPD1_CONTROL, 0);
		558	break;
		559	case RADEON_HPD_2:
		560	WREG32(DC_HPD2_CONTROL, 0);
		561	break;
		562	case RADEON_HPD_3:
		563	WREG32(DC_HPD3_CONTROL, 0);
		564	break;
		565	case RADEON_HPD_4:
		566	WREG32(DC_HPD4_CONTROL, 0);
		567	break;
		568	case RADEON_HPD_5:
		569	WREG32(DC_HPD5_CONTROL, 0);
		570	break;
		571	case RADEON_HPD_6:
		572	WREG32(DC_HPD6_CONTROL, 0);
		573	break;
		574	default:
		575	break;
		576	}
2997	Serge	577	disabled \|= 1 << radeon_connector->hpd.hpd;
1963	serge	578	}
2997	Serge	579	// radeon_irq_kms_disable_hpd(rdev, disabled);
1430	serge	580	}
		581
1986	serge	582	/* watermark setup */
1963	serge	583
1986	serge	584	static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
		585	struct radeon_crtc *radeon_crtc,
		586	struct drm_display_mode *mode,
		587	struct drm_display_mode *other_mode)
		588	{
		589	u32 tmp;
		590	/*
		591	* Line Buffer Setup
		592	* There are 3 line buffers, each one shared by 2 display controllers.
		593	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
		594	* the display controllers. The paritioning is done via one of four
		595	* preset allocations specified in bits 2:0:
		596	* first display controller
		597	* 0 - first half of lb (3840 * 2)
		598	* 1 - first 3/4 of lb (5760 * 2)
		599	* 2 - whole lb (7680 * 2), other crtc must be disabled
		600	* 3 - first 1/4 of lb (1920 * 2)
		601	* second display controller
		602	* 4 - second half of lb (3840 * 2)
		603	* 5 - second 3/4 of lb (5760 * 2)
		604	* 6 - whole lb (7680 * 2), other crtc must be disabled
		605	* 7 - last 1/4 of lb (1920 * 2)
		606	*/
		607	/* this can get tricky if we have two large displays on a paired group
		608	* of crtcs. Ideally for multiple large displays we'd assign them to
		609	* non-linked crtcs for maximum line buffer allocation.
		610	*/
		611	if (radeon_crtc->base.enabled && mode) {
		612	if (other_mode)
		613	tmp = 0; /* 1/2 */
		614	else
		615	tmp = 2; /* whole */
		616	} else
		617	tmp = 0;
1963	serge	618
1986	serge	619	/* second controller of the pair uses second half of the lb */
		620	if (radeon_crtc->crtc_id % 2)
		621	tmp += 4;
		622	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
		623
		624	if (radeon_crtc->base.enabled && mode) {
		625	switch (tmp) {
		626	case 0:
		627	case 4:
		628	default:
		629	if (ASIC_IS_DCE5(rdev))
		630	return 4096 * 2;
		631	else
		632	return 3840 * 2;
		633	case 1:
		634	case 5:
		635	if (ASIC_IS_DCE5(rdev))
		636	return 6144 * 2;
		637	else
		638	return 5760 * 2;
		639	case 2:
		640	case 6:
		641	if (ASIC_IS_DCE5(rdev))
		642	return 8192 * 2;
		643	else
		644	return 7680 * 2;
		645	case 3:
		646	case 7:
		647	if (ASIC_IS_DCE5(rdev))
		648	return 2048 * 2;
		649	else
		650	return 1920 * 2;
		651	}
		652	}
		653
		654	/* controller not enabled, so no lb used */
		655	return 0;
		656	}
		657
2997	Serge	658	u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1986	serge	659	{
		660	u32 tmp = RREG32(MC_SHARED_CHMAP);
		661
		662	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		663	case 0:
		664	default:
		665	return 1;
		666	case 1:
		667	return 2;
		668	case 2:
		669	return 4;
		670	case 3:
		671	return 8;
		672	}
		673	}
		674
		675	struct evergreen_wm_params {
		676	u32 dram_channels; /* number of dram channels */
		677	u32 yclk; /* bandwidth per dram data pin in kHz */
		678	u32 sclk; /* engine clock in kHz */
		679	u32 disp_clk; /* display clock in kHz */
		680	u32 src_width; /* viewport width */
		681	u32 active_time; /* active display time in ns */
		682	u32 blank_time; /* blank time in ns */
		683	bool interlaced; /* mode is interlaced */
		684	fixed20_12 vsc; /* vertical scale ratio */
		685	u32 num_heads; /* number of active crtcs */
		686	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
		687	u32 lb_size; /* line buffer allocated to pipe */
		688	u32 vtaps; /* vertical scaler taps */
		689	};
		690
		691	static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
		692	{
		693	/* Calculate DRAM Bandwidth and the part allocated to display. */
		694	fixed20_12 dram_efficiency; /* 0.7 */
		695	fixed20_12 yclk, dram_channels, bandwidth;
		696	fixed20_12 a;
		697
		698	a.full = dfixed_const(1000);
		699	yclk.full = dfixed_const(wm->yclk);
		700	yclk.full = dfixed_div(yclk, a);
		701	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		702	a.full = dfixed_const(10);
		703	dram_efficiency.full = dfixed_const(7);
		704	dram_efficiency.full = dfixed_div(dram_efficiency, a);
		705	bandwidth.full = dfixed_mul(dram_channels, yclk);
		706	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
		707
		708	return dfixed_trunc(bandwidth);
		709	}
		710
		711	static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		712	{
		713	/* Calculate DRAM Bandwidth and the part allocated to display. */
		714	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
		715	fixed20_12 yclk, dram_channels, bandwidth;
		716	fixed20_12 a;
		717
		718	a.full = dfixed_const(1000);
		719	yclk.full = dfixed_const(wm->yclk);
		720	yclk.full = dfixed_div(yclk, a);
		721	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		722	a.full = dfixed_const(10);
		723	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
		724	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
		725	bandwidth.full = dfixed_mul(dram_channels, yclk);
		726	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
		727
		728	return dfixed_trunc(bandwidth);
		729	}
		730
		731	static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
		732	{
		733	/* Calculate the display Data return Bandwidth */
		734	fixed20_12 return_efficiency; /* 0.8 */
		735	fixed20_12 sclk, bandwidth;
		736	fixed20_12 a;
		737
		738	a.full = dfixed_const(1000);
		739	sclk.full = dfixed_const(wm->sclk);
		740	sclk.full = dfixed_div(sclk, a);
		741	a.full = dfixed_const(10);
		742	return_efficiency.full = dfixed_const(8);
		743	return_efficiency.full = dfixed_div(return_efficiency, a);
		744	a.full = dfixed_const(32);
		745	bandwidth.full = dfixed_mul(a, sclk);
		746	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
		747
		748	return dfixed_trunc(bandwidth);
		749	}
		750
		751	static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
		752	{
		753	/* Calculate the DMIF Request Bandwidth */
		754	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
		755	fixed20_12 disp_clk, bandwidth;
		756	fixed20_12 a;
		757
		758	a.full = dfixed_const(1000);
		759	disp_clk.full = dfixed_const(wm->disp_clk);
		760	disp_clk.full = dfixed_div(disp_clk, a);
		761	a.full = dfixed_const(10);
		762	disp_clk_request_efficiency.full = dfixed_const(8);
		763	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
		764	a.full = dfixed_const(32);
		765	bandwidth.full = dfixed_mul(a, disp_clk);
		766	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
		767
		768	return dfixed_trunc(bandwidth);
		769	}
		770
		771	static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
		772	{
		773	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
		774	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
		775	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
		776	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
		777
		778	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
		779	}
		780
		781	static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
		782	{
		783	/* Calculate the display mode Average Bandwidth
		784	* DisplayMode should contain the source and destination dimensions,
		785	* timing, etc.
		786	*/
		787	fixed20_12 bpp;
		788	fixed20_12 line_time;
		789	fixed20_12 src_width;
		790	fixed20_12 bandwidth;
		791	fixed20_12 a;
		792
		793	a.full = dfixed_const(1000);
		794	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
		795	line_time.full = dfixed_div(line_time, a);
		796	bpp.full = dfixed_const(wm->bytes_per_pixel);
		797	src_width.full = dfixed_const(wm->src_width);
		798	bandwidth.full = dfixed_mul(src_width, bpp);
		799	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
		800	bandwidth.full = dfixed_div(bandwidth, line_time);
		801
		802	return dfixed_trunc(bandwidth);
		803	}
		804
		805	static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
		806	{
		807	/* First calcualte the latency in ns */
		808	u32 mc_latency = 2000; /* 2000 ns. */
		809	u32 available_bandwidth = evergreen_available_bandwidth(wm);
		810	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
		811	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
		812	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
		813	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
		814	(wm->num_heads * cursor_line_pair_return_time);
		815	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
		816	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
		817	fixed20_12 a, b, c;
		818
		819	if (wm->num_heads == 0)
		820	return 0;
		821
		822	a.full = dfixed_const(2);
		823	b.full = dfixed_const(1);
		824	if ((wm->vsc.full > a.full) \|\|
		825	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) \|\|
		826	(wm->vtaps >= 5) \|\|
		827	((wm->vsc.full >= a.full) && wm->interlaced))
		828	max_src_lines_per_dst_line = 4;
		829	else
		830	max_src_lines_per_dst_line = 2;
		831
		832	a.full = dfixed_const(available_bandwidth);
		833	b.full = dfixed_const(wm->num_heads);
		834	a.full = dfixed_div(a, b);
		835
		836	b.full = dfixed_const(1000);
		837	c.full = dfixed_const(wm->disp_clk);
		838	b.full = dfixed_div(c, b);
		839	c.full = dfixed_const(wm->bytes_per_pixel);
		840	b.full = dfixed_mul(b, c);
		841
		842	lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
		843
		844	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
		845	b.full = dfixed_const(1000);
		846	c.full = dfixed_const(lb_fill_bw);
		847	b.full = dfixed_div(c, b);
		848	a.full = dfixed_div(a, b);
		849	line_fill_time = dfixed_trunc(a);
		850
		851	if (line_fill_time < wm->active_time)
		852	return latency;
		853	else
		854	return latency + (line_fill_time - wm->active_time);
		855
		856	}
		857
		858	static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		859	{
		860	if (evergreen_average_bandwidth(wm) <=
		861	(evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
		862	return true;
		863	else
		864	return false;
		865	};
		866
		867	static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
		868	{
		869	if (evergreen_average_bandwidth(wm) <=
		870	(evergreen_available_bandwidth(wm) / wm->num_heads))
		871	return true;
		872	else
		873	return false;
		874	};
		875
		876	static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
		877	{
		878	u32 lb_partitions = wm->lb_size / wm->src_width;
		879	u32 line_time = wm->active_time + wm->blank_time;
		880	u32 latency_tolerant_lines;
		881	u32 latency_hiding;
		882	fixed20_12 a;
		883
		884	a.full = dfixed_const(1);
		885	if (wm->vsc.full > a.full)
		886	latency_tolerant_lines = 1;
		887	else {
		888	if (lb_partitions <= (wm->vtaps + 1))
		889	latency_tolerant_lines = 1;
		890	else
		891	latency_tolerant_lines = 2;
		892	}
		893
		894	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
		895
		896	if (evergreen_latency_watermark(wm) <= latency_hiding)
		897	return true;
		898	else
		899	return false;
		900	}
		901
		902	static void evergreen_program_watermarks(struct radeon_device *rdev,
		903	struct radeon_crtc *radeon_crtc,
		904	u32 lb_size, u32 num_heads)
		905	{
		906	struct drm_display_mode *mode = &radeon_crtc->base.mode;
		907	struct evergreen_wm_params wm;
		908	u32 pixel_period;
		909	u32 line_time = 0;
		910	u32 latency_watermark_a = 0, latency_watermark_b = 0;
		911	u32 priority_a_mark = 0, priority_b_mark = 0;
		912	u32 priority_a_cnt = PRIORITY_OFF;
		913	u32 priority_b_cnt = PRIORITY_OFF;
		914	u32 pipe_offset = radeon_crtc->crtc_id * 16;
		915	u32 tmp, arb_control3;
		916	fixed20_12 a, b, c;
		917
		918	if (radeon_crtc->base.enabled && num_heads && mode) {
		919	pixel_period = 1000000 / (u32)mode->clock;
		920	line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
		921	priority_a_cnt = 0;
		922	priority_b_cnt = 0;
		923
		924	wm.yclk = rdev->pm.current_mclk * 10;
		925	wm.sclk = rdev->pm.current_sclk * 10;
		926	wm.disp_clk = mode->clock;
		927	wm.src_width = mode->crtc_hdisplay;
		928	wm.active_time = mode->crtc_hdisplay * pixel_period;
		929	wm.blank_time = line_time - wm.active_time;
		930	wm.interlaced = false;
		931	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		932	wm.interlaced = true;
		933	wm.vsc = radeon_crtc->vsc;
		934	wm.vtaps = 1;
		935	if (radeon_crtc->rmx_type != RMX_OFF)
		936	wm.vtaps = 2;
		937	wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
		938	wm.lb_size = lb_size;
		939	wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
		940	wm.num_heads = num_heads;
		941
		942	/* set for high clocks */
		943	latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
		944	/* set for low clocks */
		945	/* wm.yclk = low clk; wm.sclk = low clk */
		946	latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
		947
		948	/* possibly force display priority to high */
		949	/* should really do this at mode validation time... */
		950	if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) \|\|
		951	!evergreen_average_bandwidth_vs_available_bandwidth(&wm) \|\|
		952	!evergreen_check_latency_hiding(&wm) \|\|
		953	(rdev->disp_priority == 2)) {
2160	serge	954	DRM_DEBUG_KMS("force priority to high\n");
1986	serge	955	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		956	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		957	}
		958
		959	a.full = dfixed_const(1000);
		960	b.full = dfixed_const(mode->clock);
		961	b.full = dfixed_div(b, a);
		962	c.full = dfixed_const(latency_watermark_a);
		963	c.full = dfixed_mul(c, b);
		964	c.full = dfixed_mul(c, radeon_crtc->hsc);
		965	c.full = dfixed_div(c, a);
		966	a.full = dfixed_const(16);
		967	c.full = dfixed_div(c, a);
		968	priority_a_mark = dfixed_trunc(c);
		969	priority_a_cnt \|= priority_a_mark & PRIORITY_MARK_MASK;
		970
		971	a.full = dfixed_const(1000);
		972	b.full = dfixed_const(mode->clock);
		973	b.full = dfixed_div(b, a);
		974	c.full = dfixed_const(latency_watermark_b);
		975	c.full = dfixed_mul(c, b);
		976	c.full = dfixed_mul(c, radeon_crtc->hsc);
		977	c.full = dfixed_div(c, a);
		978	a.full = dfixed_const(16);
		979	c.full = dfixed_div(c, a);
		980	priority_b_mark = dfixed_trunc(c);
		981	priority_b_cnt \|= priority_b_mark & PRIORITY_MARK_MASK;
		982	}
		983
		984	/* select wm A */
		985	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		986	tmp = arb_control3;
		987	tmp &= ~LATENCY_WATERMARK_MASK(3);
		988	tmp \|= LATENCY_WATERMARK_MASK(1);
		989	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		990	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		991	(LATENCY_LOW_WATERMARK(latency_watermark_a) \|
		992	LATENCY_HIGH_WATERMARK(line_time)));
		993	/* select wm B */
		994	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		995	tmp &= ~LATENCY_WATERMARK_MASK(3);
		996	tmp \|= LATENCY_WATERMARK_MASK(2);
		997	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		998	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		999	(LATENCY_LOW_WATERMARK(latency_watermark_b) \|
		1000	LATENCY_HIGH_WATERMARK(line_time)));
		1001	/* restore original selection */
		1002	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
		1003
		1004	/* write the priority marks */
		1005	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
		1006	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
		1007
		1008	}
		1009
2997	Serge	1010	/**
		1011	* evergreen_bandwidth_update - update display watermarks callback.
		1012	*
		1013	* @rdev: radeon_device pointer
		1014	*
		1015	* Update the display watermarks based on the requested mode(s)
		1016	* (evergreen+).
		1017	*/
1963	serge	1018	void evergreen_bandwidth_update(struct radeon_device *rdev)
1430	serge	1019	{
1986	serge	1020	struct drm_display_mode *mode0 = NULL;
		1021	struct drm_display_mode *mode1 = NULL;
		1022	u32 num_heads = 0, lb_size;
		1023	int i;
		1024
		1025	radeon_update_display_priority(rdev);
		1026
		1027	for (i = 0; i < rdev->num_crtc; i++) {
		1028	if (rdev->mode_info.crtcs[i]->base.enabled)
		1029	num_heads++;
		1030	}
		1031	for (i = 0; i < rdev->num_crtc; i += 2) {
		1032	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
		1033	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
		1034	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
		1035	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
		1036	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
		1037	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
		1038	}
1430	serge	1039	}
		1040
2997	Serge	1041	/**
		1042	* evergreen_mc_wait_for_idle - wait for MC idle callback.
		1043	*
		1044	* @rdev: radeon_device pointer
		1045	*
		1046	* Wait for the MC (memory controller) to be idle.
		1047	* (evergreen+).
		1048	* Returns 0 if the MC is idle, -1 if not.
		1049	*/
1963	serge	1050	int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1430	serge	1051	{
		1052	unsigned i;
		1053	u32 tmp;
		1054
		1055	for (i = 0; i < rdev->usec_timeout; i++) {
		1056	/* read MC_STATUS */
		1057	tmp = RREG32(SRBM_STATUS) & 0x1F00;
		1058	if (!tmp)
		1059	return 0;
		1060	udelay(1);
		1061	}
		1062	return -1;
		1063	}
		1064
		1065	/*
		1066	* GART
		1067	*/
1963	serge	1068	void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
		1069	{
		1070	unsigned i;
		1071	u32 tmp;
		1072
		1073	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		1074
		1075	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		1076	for (i = 0; i < rdev->usec_timeout; i++) {
		1077	/* read MC_STATUS */
		1078	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		1079	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		1080	if (tmp == 2) {
		1081	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		1082	return;
		1083	}
		1084	if (tmp) {
		1085	return;
		1086	}
		1087	udelay(1);
		1088	}
		1089	}
		1090
2997	Serge	1091	static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
1430	serge	1092	{
		1093	u32 tmp;
1963	serge	1094	int r;
1430	serge	1095
2997	Serge	1096	if (rdev->gart.robj == NULL) {
1430	serge	1097	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		1098	return -EINVAL;
		1099	}
		1100	r = radeon_gart_table_vram_pin(rdev);
		1101	if (r)
		1102	return r;
		1103	radeon_gart_restore(rdev);
		1104	/* Setup L2 cache */
		1105	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		1106	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1107	EFFECTIVE_L2_QUEUE_SIZE(7));
		1108	WREG32(VM_L2_CNTL2, 0);
		1109	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		1110	/* Setup TLB control */
		1111	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		1112	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		1113	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		1114	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
1963	serge	1115	if (rdev->flags & RADEON_IS_IGP) {
		1116	WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
		1117	WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
		1118	WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
		1119	} else {
3031	serge	1120	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		1121	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		1122	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2997	Serge	1123	if ((rdev->family == CHIP_JUNIPER) \|\|
		1124	(rdev->family == CHIP_CYPRESS) \|\|
		1125	(rdev->family == CHIP_HEMLOCK) \|\|
		1126	(rdev->family == CHIP_BARTS))
		1127	WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
1963	serge	1128	}
1430	serge	1129	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		1130	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		1131	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		1132	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
		1133	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		1134	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		1135	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		1136	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		1137	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		1138	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		1139	(u32)(rdev->dummy_page.addr >> 12));
1963	serge	1140	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	1141
1963	serge	1142	evergreen_pcie_gart_tlb_flush(rdev);
2997	Serge	1143	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		1144	(unsigned)(rdev->mc.gtt_size >> 20),
		1145	(unsigned long long)rdev->gart.table_addr);
1430	serge	1146	rdev->gart.ready = true;
		1147	return 0;
		1148	}
		1149
2997	Serge	1150	static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
1430	serge	1151	{
		1152	u32 tmp;
		1153
		1154	/* Disable all tables */
1963	serge	1155	WREG32(VM_CONTEXT0_CNTL, 0);
		1156	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	1157
		1158	/* Setup L2 cache */
		1159	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		1160	EFFECTIVE_L2_QUEUE_SIZE(7));
		1161	WREG32(VM_L2_CNTL2, 0);
		1162	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		1163	/* Setup TLB control */
		1164	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		1165	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		1166	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		1167	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		1168	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		1169	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		1170	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		1171	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2997	Serge	1172	radeon_gart_table_vram_unpin(rdev);
1430	serge	1173	}
		1174
2997	Serge	1175	static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
1430	serge	1176	{
		1177	evergreen_pcie_gart_disable(rdev);
		1178	radeon_gart_table_vram_free(rdev);
		1179	radeon_gart_fini(rdev);
		1180	}
		1181
		1182
2997	Serge	1183	static void evergreen_agp_enable(struct radeon_device *rdev)
1430	serge	1184	{
		1185	u32 tmp;
		1186
		1187	/* Setup L2 cache */
		1188	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		1189	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1190	EFFECTIVE_L2_QUEUE_SIZE(7));
		1191	WREG32(VM_L2_CNTL2, 0);
		1192	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		1193	/* Setup TLB control */
		1194	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		1195	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		1196	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		1197	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		1198	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		1199	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		1200	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		1201	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		1202	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		1203	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		1204	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1963	serge	1205	WREG32(VM_CONTEXT0_CNTL, 0);
		1206	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	1207	}
		1208
1963	serge	1209	void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	1210	{
2997	Serge	1211	u32 crtc_enabled, tmp, frame_count, blackout;
		1212	int i, j;
		1213
1430	serge	1214	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
		1215	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
		1216
2997	Serge	1217	/* disable VGA render */
1430	serge	1218	WREG32(VGA_RENDER_CONTROL, 0);
2997	Serge	1219	/* blank the display controllers */
		1220	for (i = 0; i < rdev->num_crtc; i++) {
		1221	crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
		1222	if (crtc_enabled) {
		1223	save->crtc_enabled[i] = true;
		1224	if (ASIC_IS_DCE6(rdev)) {
		1225	tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
		1226	if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
		1227	radeon_wait_for_vblank(rdev, i);
		1228	tmp \|= EVERGREEN_CRTC_BLANK_DATA_EN;
		1229	WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
3031	serge	1230	}
2997	Serge	1231	} else {
		1232	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
		1233	if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
		1234	radeon_wait_for_vblank(rdev, i);
		1235	tmp \|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		1236	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
3031	serge	1237	}
		1238	}
2997	Serge	1239	/* wait for the next frame */
		1240	frame_count = radeon_get_vblank_counter(rdev, i);
		1241	for (j = 0; j < rdev->usec_timeout; j++) {
		1242	if (radeon_get_vblank_counter(rdev, i) != frame_count)
		1243	break;
		1244	udelay(1);
3031	serge	1245	}
		1246	}
1963	serge	1247	}
1430	serge	1248
2997	Serge	1249	radeon_mc_wait_for_idle(rdev);
		1250
		1251	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
		1252	if ((blackout & BLACKOUT_MODE_MASK) != 1) {
		1253	/* Block CPU access */
		1254	WREG32(BIF_FB_EN, 0);
		1255	/* blackout the MC */
		1256	blackout &= ~BLACKOUT_MODE_MASK;
		1257	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout \| 1);
2005	serge	1258	}
1430	serge	1259	}
		1260
1963	serge	1261	void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	1262	{
2997	Serge	1263	u32 tmp, frame_count;
		1264	int i, j;
1430	serge	1265
2997	Serge	1266	/* update crtc base addresses */
		1267	for (i = 0; i < rdev->num_crtc; i++) {
		1268	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
3031	serge	1269	upper_32_bits(rdev->mc.vram_start));
2997	Serge	1270	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
3031	serge	1271	upper_32_bits(rdev->mc.vram_start));
2997	Serge	1272	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
3031	serge	1273	(u32)rdev->mc.vram_start);
2997	Serge	1274	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
3031	serge	1275	(u32)rdev->mc.vram_start);
2997	Serge	1276	}
		1277	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
		1278	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
1430	serge	1279
2997	Serge	1280	/* unblackout the MC */
		1281	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
		1282	tmp &= ~BLACKOUT_MODE_MASK;
		1283	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
		1284	/* allow CPU access */
		1285	WREG32(BIF_FB_EN, FB_READ_EN \| FB_WRITE_EN);
1430	serge	1286
2997	Serge	1287	for (i = 0; i < rdev->num_crtc; i++) {
3031	serge	1288	if (save->crtc_enabled[i]) {
2997	Serge	1289	if (ASIC_IS_DCE6(rdev)) {
		1290	tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
		1291	tmp \|= EVERGREEN_CRTC_BLANK_DATA_EN;
		1292	WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
		1293	} else {
		1294	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
		1295	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		1296	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
		1297	}
		1298	/* wait for the next frame */
		1299	frame_count = radeon_get_vblank_counter(rdev, i);
		1300	for (j = 0; j < rdev->usec_timeout; j++) {
		1301	if (radeon_get_vblank_counter(rdev, i) != frame_count)
		1302	break;
		1303	udelay(1);
		1304	}
3031	serge	1305	}
2005	serge	1306	}
2997	Serge	1307	/* Unlock vga access */
1430	serge	1308	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
		1309	mdelay(1);
		1310	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
		1311	}
		1312
1963	serge	1313	void evergreen_mc_program(struct radeon_device *rdev)
1430	serge	1314	{
		1315	struct evergreen_mc_save save;
		1316	u32 tmp;
		1317	int i, j;
		1318
		1319	/* Initialize HDP */
		1320	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1321	WREG32((0x2c14 + j), 0x00000000);
		1322	WREG32((0x2c18 + j), 0x00000000);
		1323	WREG32((0x2c1c + j), 0x00000000);
		1324	WREG32((0x2c20 + j), 0x00000000);
		1325	WREG32((0x2c24 + j), 0x00000000);
		1326	}
		1327	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		1328
		1329	evergreen_mc_stop(rdev, &save);
		1330	if (evergreen_mc_wait_for_idle(rdev)) {
		1331	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1332	}
		1333	/* Lockout access through VGA aperture*/
		1334	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		1335	/* Update configuration */
		1336	if (rdev->flags & RADEON_IS_AGP) {
		1337	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		1338	/* VRAM before AGP */
		1339	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1340	rdev->mc.vram_start >> 12);
		1341	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1342	rdev->mc.gtt_end >> 12);
		1343	} else {
		1344	/* VRAM after AGP */
		1345	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1346	rdev->mc.gtt_start >> 12);
		1347	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1348	rdev->mc.vram_end >> 12);
		1349	}
		1350	} else {
		1351	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1352	rdev->mc.vram_start >> 12);
		1353	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1354	rdev->mc.vram_end >> 12);
		1355	}
2997	Serge	1356	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
		1357	/* llano/ontario only */
		1358	if ((rdev->family == CHIP_PALM) \|\|
		1359	(rdev->family == CHIP_SUMO) \|\|
		1360	(rdev->family == CHIP_SUMO2)) {
1963	serge	1361	tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
		1362	tmp \|= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
		1363	tmp \|= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
		1364	WREG32(MC_FUS_VM_FB_OFFSET, tmp);
		1365	}
1430	serge	1366	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		1367	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		1368	WREG32(MC_VM_FB_LOCATION, tmp);
		1369	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1963	serge	1370	WREG32(HDP_NONSURFACE_INFO, (2 << 7) \| (1 << 30));
		1371	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1430	serge	1372	if (rdev->flags & RADEON_IS_AGP) {
		1373	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
		1374	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
		1375	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		1376	} else {
		1377	WREG32(MC_VM_AGP_BASE, 0);
		1378	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		1379	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		1380	}
		1381	if (evergreen_mc_wait_for_idle(rdev)) {
		1382	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1383	}
		1384	evergreen_mc_resume(rdev, &save);
		1385	/* we need to own VRAM, so turn off the VGA renderer here
		1386	* to stop it overwriting our objects */
		1387	rv515_vga_render_disable(rdev);
		1388	}
		1389
		1390	/*
		1391	* CP.
		1392	*/
1986	serge	1393	void evergreen_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		1394	{
2997	Serge	1395	struct radeon_ring *ring = &rdev->ring[ib->ring];
		1396	u32 next_rptr;
		1397
1986	serge	1398	/* set to DX10/11 mode */
2997	Serge	1399	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
		1400	radeon_ring_write(ring, 1);
		1401
		1402	if (ring->rptr_save_reg) {
		1403	next_rptr = ring->wptr + 3 + 4;
		1404	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3031	serge	1405	radeon_ring_write(ring, ((ring->rptr_save_reg -
2997	Serge	1406	PACKET3_SET_CONFIG_REG_START) >> 2));
		1407	radeon_ring_write(ring, next_rptr);
		1408	} else if (rdev->wb.enabled) {
		1409	next_rptr = ring->wptr + 5 + 4;
		1410	radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
		1411	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		1412	radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) \| (1 << 18));
		1413	radeon_ring_write(ring, next_rptr);
		1414	radeon_ring_write(ring, 0);
		1415	}
		1416
		1417	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		1418	radeon_ring_write(ring,
1986	serge	1419	#ifdef __BIG_ENDIAN
		1420	(2 << 0) \|
		1421	#endif
		1422	(ib->gpu_addr & 0xFFFFFFFC));
2997	Serge	1423	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
		1424	radeon_ring_write(ring, ib->length_dw);
1986	serge	1425	}
1963	serge	1426
1986	serge	1427
1963	serge	1428	static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1430	serge	1429	{
1963	serge	1430	const __be32 *fw_data;
		1431	int i;
		1432
		1433	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1434	return -EINVAL;
		1435
		1436	r700_cp_stop(rdev);
		1437	WREG32(CP_RB_CNTL,
		1438	#ifdef __BIG_ENDIAN
		1439	BUF_SWAP_32BIT \|
		1440	#endif
		1441	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
		1442
		1443	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1444	WREG32(CP_PFP_UCODE_ADDR, 0);
		1445	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
		1446	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		1447	WREG32(CP_PFP_UCODE_ADDR, 0);
		1448
		1449	fw_data = (const __be32 *)rdev->me_fw->data;
		1450	WREG32(CP_ME_RAM_WADDR, 0);
		1451	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
		1452	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		1453
		1454	WREG32(CP_PFP_UCODE_ADDR, 0);
		1455	WREG32(CP_ME_RAM_WADDR, 0);
		1456	WREG32(CP_ME_RAM_RADDR, 0);
		1457	return 0;
1430	serge	1458	}
		1459
1963	serge	1460	static int evergreen_cp_start(struct radeon_device *rdev)
		1461	{
2997	Serge	1462	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	1463	int r, i;
		1464	uint32_t cp_me;
1430	serge	1465
2997	Serge	1466	r = radeon_ring_lock(rdev, ring, 7);
1963	serge	1467	if (r) {
		1468	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1469	return r;
		1470	}
2997	Serge	1471	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1472	radeon_ring_write(ring, 0x1);
		1473	radeon_ring_write(ring, 0x0);
		1474	radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
		1475	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1476	radeon_ring_write(ring, 0);
		1477	radeon_ring_write(ring, 0);
		1478	radeon_ring_unlock_commit(rdev, ring);
1963	serge	1479
		1480	cp_me = 0xff;
		1481	WREG32(CP_ME_CNTL, cp_me);
		1482
2997	Serge	1483	r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
1963	serge	1484	if (r) {
		1485	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1486	return r;
		1487	}
		1488
		1489	/* setup clear context state */
2997	Serge	1490	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1491	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1963	serge	1492
		1493	for (i = 0; i < evergreen_default_size; i++)
2997	Serge	1494	radeon_ring_write(ring, evergreen_default_state[i]);
1963	serge	1495
2997	Serge	1496	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1497	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1963	serge	1498
		1499	/* set clear context state */
2997	Serge	1500	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
		1501	radeon_ring_write(ring, 0);
1963	serge	1502
		1503	/* SQ_VTX_BASE_VTX_LOC */
2997	Serge	1504	radeon_ring_write(ring, 0xc0026f00);
		1505	radeon_ring_write(ring, 0x00000000);
		1506	radeon_ring_write(ring, 0x00000000);
		1507	radeon_ring_write(ring, 0x00000000);
1963	serge	1508
		1509	/* Clear consts */
2997	Serge	1510	radeon_ring_write(ring, 0xc0036f00);
		1511	radeon_ring_write(ring, 0x00000bc4);
		1512	radeon_ring_write(ring, 0xffffffff);
		1513	radeon_ring_write(ring, 0xffffffff);
		1514	radeon_ring_write(ring, 0xffffffff);
1963	serge	1515
2997	Serge	1516	radeon_ring_write(ring, 0xc0026900);
		1517	radeon_ring_write(ring, 0x00000316);
		1518	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		1519	radeon_ring_write(ring, 0x00000010); /* */
1963	serge	1520
2997	Serge	1521	radeon_ring_unlock_commit(rdev, ring);
1963	serge	1522
		1523	return 0;
		1524	}
		1525
2997	Serge	1526	static int evergreen_cp_resume(struct radeon_device *rdev)
1430	serge	1527	{
2997	Serge	1528	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	1529	u32 tmp;
		1530	u32 rb_bufsz;
		1531	int r;
1430	serge	1532
1963	serge	1533	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
		1534	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP \|
		1535	SOFT_RESET_PA \|
		1536	SOFT_RESET_SH \|
		1537	SOFT_RESET_VGT \|
2160	serge	1538	SOFT_RESET_SPI \|
1963	serge	1539	SOFT_RESET_SX));
		1540	RREG32(GRBM_SOFT_RESET);
		1541	mdelay(15);
		1542	WREG32(GRBM_SOFT_RESET, 0);
		1543	RREG32(GRBM_SOFT_RESET);
		1544
		1545	/* Set ring buffer size */
2997	Serge	1546	rb_bufsz = drm_order(ring->ring_size / 8);
1963	serge	1547	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
		1548	#ifdef __BIG_ENDIAN
		1549	tmp \|= BUF_SWAP_32BIT;
		1550	#endif
		1551	WREG32(CP_RB_CNTL, tmp);
2997	Serge	1552	WREG32(CP_SEM_WAIT_TIMER, 0x0);
		1553	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1963	serge	1554
		1555	/* Set the write pointer delay */
		1556	WREG32(CP_RB_WPTR_DELAY, 0);
		1557
		1558	/* Initialize the ring buffer's read and write pointers */
		1559	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1560	WREG32(CP_RB_RPTR_WR, 0);
2997	Serge	1561	ring->wptr = 0;
		1562	WREG32(CP_RB_WPTR, ring->wptr);
1963	serge	1563
		1564	/* set the wb address wether it's enabled or not */
		1565	WREG32(CP_RB_RPTR_ADDR,
		1566	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1567	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1568	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1569
		1570	if (rdev->wb.enabled)
		1571	WREG32(SCRATCH_UMSK, 0xff);
		1572	else {
		1573	tmp \|= RB_NO_UPDATE;
		1574	WREG32(SCRATCH_UMSK, 0);
		1575	}
		1576
		1577	mdelay(1);
		1578	WREG32(CP_RB_CNTL, tmp);
		1579
2997	Serge	1580	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1963	serge	1581	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1582
2997	Serge	1583	ring->rptr = RREG32(CP_RB_RPTR);
1963	serge	1584
		1585	evergreen_cp_start(rdev);
2997	Serge	1586	ring->ready = true;
		1587	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
1963	serge	1588	if (r) {
2997	Serge	1589	ring->ready = false;
1963	serge	1590	return r;
		1591	}
1430	serge	1592	return 0;
		1593	}
		1594
		1595	/*
		1596	* Core functions
		1597	*/
		1598	static void evergreen_gpu_init(struct radeon_device *rdev)
		1599	{
2997	Serge	1600	u32 gb_addr_config;
1963	serge	1601	u32 mc_shared_chmap, mc_arb_ramcfg;
		1602	u32 sx_debug_1;
		1603	u32 smx_dc_ctl0;
		1604	u32 sq_config;
		1605	u32 sq_lds_resource_mgmt;
		1606	u32 sq_gpr_resource_mgmt_1;
		1607	u32 sq_gpr_resource_mgmt_2;
		1608	u32 sq_gpr_resource_mgmt_3;
		1609	u32 sq_thread_resource_mgmt;
		1610	u32 sq_thread_resource_mgmt_2;
		1611	u32 sq_stack_resource_mgmt_1;
		1612	u32 sq_stack_resource_mgmt_2;
		1613	u32 sq_stack_resource_mgmt_3;
		1614	u32 vgt_cache_invalidation;
		1615	u32 hdp_host_path_cntl, tmp;
2997	Serge	1616	u32 disabled_rb_mask;
1963	serge	1617	int i, j, num_shader_engines, ps_thread_count;
		1618
		1619	switch (rdev->family) {
		1620	case CHIP_CYPRESS:
		1621	case CHIP_HEMLOCK:
		1622	rdev->config.evergreen.num_ses = 2;
		1623	rdev->config.evergreen.max_pipes = 4;
		1624	rdev->config.evergreen.max_tile_pipes = 8;
		1625	rdev->config.evergreen.max_simds = 10;
		1626	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1627	rdev->config.evergreen.max_gprs = 256;
		1628	rdev->config.evergreen.max_threads = 248;
		1629	rdev->config.evergreen.max_gs_threads = 32;
		1630	rdev->config.evergreen.max_stack_entries = 512;
		1631	rdev->config.evergreen.sx_num_of_sets = 4;
		1632	rdev->config.evergreen.sx_max_export_size = 256;
		1633	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1634	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1635	rdev->config.evergreen.max_hw_contexts = 8;
		1636	rdev->config.evergreen.sq_num_cf_insts = 2;
		1637
		1638	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1639	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1640	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1641	gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1642	break;
		1643	case CHIP_JUNIPER:
		1644	rdev->config.evergreen.num_ses = 1;
		1645	rdev->config.evergreen.max_pipes = 4;
		1646	rdev->config.evergreen.max_tile_pipes = 4;
		1647	rdev->config.evergreen.max_simds = 10;
		1648	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1649	rdev->config.evergreen.max_gprs = 256;
		1650	rdev->config.evergreen.max_threads = 248;
		1651	rdev->config.evergreen.max_gs_threads = 32;
		1652	rdev->config.evergreen.max_stack_entries = 512;
		1653	rdev->config.evergreen.sx_num_of_sets = 4;
		1654	rdev->config.evergreen.sx_max_export_size = 256;
		1655	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1656	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1657	rdev->config.evergreen.max_hw_contexts = 8;
		1658	rdev->config.evergreen.sq_num_cf_insts = 2;
		1659
		1660	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1661	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1662	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1663	gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1664	break;
		1665	case CHIP_REDWOOD:
		1666	rdev->config.evergreen.num_ses = 1;
		1667	rdev->config.evergreen.max_pipes = 4;
		1668	rdev->config.evergreen.max_tile_pipes = 4;
		1669	rdev->config.evergreen.max_simds = 5;
		1670	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1671	rdev->config.evergreen.max_gprs = 256;
		1672	rdev->config.evergreen.max_threads = 248;
		1673	rdev->config.evergreen.max_gs_threads = 32;
		1674	rdev->config.evergreen.max_stack_entries = 256;
		1675	rdev->config.evergreen.sx_num_of_sets = 4;
		1676	rdev->config.evergreen.sx_max_export_size = 256;
		1677	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1678	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1679	rdev->config.evergreen.max_hw_contexts = 8;
		1680	rdev->config.evergreen.sq_num_cf_insts = 2;
		1681
		1682	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1683	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1684	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1685	gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1686	break;
		1687	case CHIP_CEDAR:
		1688	default:
		1689	rdev->config.evergreen.num_ses = 1;
		1690	rdev->config.evergreen.max_pipes = 2;
		1691	rdev->config.evergreen.max_tile_pipes = 2;
		1692	rdev->config.evergreen.max_simds = 2;
		1693	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1694	rdev->config.evergreen.max_gprs = 256;
		1695	rdev->config.evergreen.max_threads = 192;
		1696	rdev->config.evergreen.max_gs_threads = 16;
		1697	rdev->config.evergreen.max_stack_entries = 256;
		1698	rdev->config.evergreen.sx_num_of_sets = 4;
		1699	rdev->config.evergreen.sx_max_export_size = 128;
		1700	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1701	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1702	rdev->config.evergreen.max_hw_contexts = 4;
		1703	rdev->config.evergreen.sq_num_cf_insts = 1;
		1704
		1705	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1706	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1707	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1708	gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1709	break;
		1710	case CHIP_PALM:
		1711	rdev->config.evergreen.num_ses = 1;
		1712	rdev->config.evergreen.max_pipes = 2;
		1713	rdev->config.evergreen.max_tile_pipes = 2;
		1714	rdev->config.evergreen.max_simds = 2;
		1715	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1716	rdev->config.evergreen.max_gprs = 256;
		1717	rdev->config.evergreen.max_threads = 192;
		1718	rdev->config.evergreen.max_gs_threads = 16;
		1719	rdev->config.evergreen.max_stack_entries = 256;
		1720	rdev->config.evergreen.sx_num_of_sets = 4;
		1721	rdev->config.evergreen.sx_max_export_size = 128;
		1722	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1723	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1724	rdev->config.evergreen.max_hw_contexts = 4;
		1725	rdev->config.evergreen.sq_num_cf_insts = 1;
		1726
		1727	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1728	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1729	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1730	gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1731	break;
		1732	case CHIP_SUMO:
		1733	rdev->config.evergreen.num_ses = 1;
		1734	rdev->config.evergreen.max_pipes = 4;
		1735	rdev->config.evergreen.max_tile_pipes = 2;
		1736	if (rdev->pdev->device == 0x9648)
		1737	rdev->config.evergreen.max_simds = 3;
		1738	else if ((rdev->pdev->device == 0x9647) \|\|
		1739	(rdev->pdev->device == 0x964a))
		1740	rdev->config.evergreen.max_simds = 4;
		1741	else
		1742	rdev->config.evergreen.max_simds = 5;
		1743	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1744	rdev->config.evergreen.max_gprs = 256;
		1745	rdev->config.evergreen.max_threads = 248;
		1746	rdev->config.evergreen.max_gs_threads = 32;
		1747	rdev->config.evergreen.max_stack_entries = 256;
		1748	rdev->config.evergreen.sx_num_of_sets = 4;
		1749	rdev->config.evergreen.sx_max_export_size = 256;
		1750	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1751	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1752	rdev->config.evergreen.max_hw_contexts = 8;
		1753	rdev->config.evergreen.sq_num_cf_insts = 2;
		1754
		1755	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1756	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1757	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1758	gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1759	break;
		1760	case CHIP_SUMO2:
		1761	rdev->config.evergreen.num_ses = 1;
		1762	rdev->config.evergreen.max_pipes = 4;
		1763	rdev->config.evergreen.max_tile_pipes = 4;
		1764	rdev->config.evergreen.max_simds = 2;
		1765	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1766	rdev->config.evergreen.max_gprs = 256;
		1767	rdev->config.evergreen.max_threads = 248;
		1768	rdev->config.evergreen.max_gs_threads = 32;
		1769	rdev->config.evergreen.max_stack_entries = 512;
		1770	rdev->config.evergreen.sx_num_of_sets = 4;
		1771	rdev->config.evergreen.sx_max_export_size = 256;
		1772	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1773	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1774	rdev->config.evergreen.max_hw_contexts = 8;
		1775	rdev->config.evergreen.sq_num_cf_insts = 2;
		1776
		1777	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1778	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1779	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1780	gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1781	break;
		1782	case CHIP_BARTS:
		1783	rdev->config.evergreen.num_ses = 2;
		1784	rdev->config.evergreen.max_pipes = 4;
		1785	rdev->config.evergreen.max_tile_pipes = 8;
		1786	rdev->config.evergreen.max_simds = 7;
		1787	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1788	rdev->config.evergreen.max_gprs = 256;
		1789	rdev->config.evergreen.max_threads = 248;
		1790	rdev->config.evergreen.max_gs_threads = 32;
		1791	rdev->config.evergreen.max_stack_entries = 512;
		1792	rdev->config.evergreen.sx_num_of_sets = 4;
		1793	rdev->config.evergreen.sx_max_export_size = 256;
		1794	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1795	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1796	rdev->config.evergreen.max_hw_contexts = 8;
		1797	rdev->config.evergreen.sq_num_cf_insts = 2;
		1798
		1799	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1800	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1801	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1802	gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1803	break;
		1804	case CHIP_TURKS:
		1805	rdev->config.evergreen.num_ses = 1;
		1806	rdev->config.evergreen.max_pipes = 4;
		1807	rdev->config.evergreen.max_tile_pipes = 4;
		1808	rdev->config.evergreen.max_simds = 6;
		1809	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1810	rdev->config.evergreen.max_gprs = 256;
		1811	rdev->config.evergreen.max_threads = 248;
		1812	rdev->config.evergreen.max_gs_threads = 32;
		1813	rdev->config.evergreen.max_stack_entries = 256;
		1814	rdev->config.evergreen.sx_num_of_sets = 4;
		1815	rdev->config.evergreen.sx_max_export_size = 256;
		1816	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1817	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1818	rdev->config.evergreen.max_hw_contexts = 8;
		1819	rdev->config.evergreen.sq_num_cf_insts = 2;
		1820
		1821	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1822	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1823	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1824	gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1825	break;
		1826	case CHIP_CAICOS:
		1827	rdev->config.evergreen.num_ses = 1;
		1828	rdev->config.evergreen.max_pipes = 4;
		1829	rdev->config.evergreen.max_tile_pipes = 2;
		1830	rdev->config.evergreen.max_simds = 2;
		1831	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1832	rdev->config.evergreen.max_gprs = 256;
		1833	rdev->config.evergreen.max_threads = 192;
		1834	rdev->config.evergreen.max_gs_threads = 16;
		1835	rdev->config.evergreen.max_stack_entries = 256;
		1836	rdev->config.evergreen.sx_num_of_sets = 4;
		1837	rdev->config.evergreen.sx_max_export_size = 128;
		1838	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1839	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1840	rdev->config.evergreen.max_hw_contexts = 4;
		1841	rdev->config.evergreen.sq_num_cf_insts = 1;
		1842
		1843	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1844	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1845	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	1846	gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	1847	break;
		1848	}
		1849
		1850	/* Initialize HDP */
		1851	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1852	WREG32((0x2c14 + j), 0x00000000);
		1853	WREG32((0x2c18 + j), 0x00000000);
		1854	WREG32((0x2c1c + j), 0x00000000);
		1855	WREG32((0x2c20 + j), 0x00000000);
		1856	WREG32((0x2c24 + j), 0x00000000);
		1857	}
		1858
		1859	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1860
2997	Serge	1861	evergreen_fix_pci_max_read_req_size(rdev);
1963	serge	1862
		1863	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2997	Serge	1864	if ((rdev->family == CHIP_PALM) \|\|
		1865	(rdev->family == CHIP_SUMO) \|\|
		1866	(rdev->family == CHIP_SUMO2))
1963	serge	1867	mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
		1868	else
3031	serge	1869	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1963	serge	1870
		1871	/* setup tiling info dword. gb_addr_config is not adequate since it does
		1872	* not have bank info, so create a custom tiling dword.
		1873	* bits 3:0 num_pipes
		1874	* bits 7:4 num_banks
		1875	* bits 11:8 group_size
		1876	* bits 15:12 row_size
		1877	*/
		1878	rdev->config.evergreen.tile_config = 0;
		1879	switch (rdev->config.evergreen.max_tile_pipes) {
		1880	case 1:
		1881	default:
		1882	rdev->config.evergreen.tile_config \|= (0 << 0);
		1883	break;
		1884	case 2:
		1885	rdev->config.evergreen.tile_config \|= (1 << 0);
		1886	break;
		1887	case 4:
		1888	rdev->config.evergreen.tile_config \|= (2 << 0);
		1889	break;
		1890	case 8:
		1891	rdev->config.evergreen.tile_config \|= (3 << 0);
		1892	break;
		1893	}
1986	serge	1894	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1963	serge	1895	if (rdev->flags & RADEON_IS_IGP)
1986	serge	1896	rdev->config.evergreen.tile_config \|= 1 << 4;
2997	Serge	1897	else {
		1898	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
		1899	case 0: /* four banks */
		1900	rdev->config.evergreen.tile_config \|= 0 << 4;
		1901	break;
		1902	case 1: /* eight banks */
		1903	rdev->config.evergreen.tile_config \|= 1 << 4;
		1904	break;
		1905	case 2: /* sixteen banks */
		1906	default:
		1907	rdev->config.evergreen.tile_config \|= 2 << 4;
		1908	break;
		1909	}
		1910	}
		1911	rdev->config.evergreen.tile_config \|= 0 << 8;
1963	serge	1912	rdev->config.evergreen.tile_config \|=
		1913	((gb_addr_config & 0x30000000) >> 28) << 12;
		1914
2997	Serge	1915	num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
1963	serge	1916
2997	Serge	1917	if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
		1918	u32 efuse_straps_4;
		1919	u32 efuse_straps_3;
1963	serge	1920
2997	Serge	1921	WREG32(RCU_IND_INDEX, 0x204);
		1922	efuse_straps_4 = RREG32(RCU_IND_DATA);
		1923	WREG32(RCU_IND_INDEX, 0x203);
		1924	efuse_straps_3 = RREG32(RCU_IND_DATA);
		1925	tmp = (((efuse_straps_4 & 0xf) << 4) \|
		1926	((efuse_straps_3 & 0xf0000000) >> 28));
		1927	} else {
		1928	tmp = 0;
		1929	for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
		1930	u32 rb_disable_bitmap;
1963	serge	1931
2997	Serge	1932	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		1933	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		1934	rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
		1935	tmp <<= 4;
		1936	tmp \|= rb_disable_bitmap;
1963	serge	1937	}
2997	Serge	1938	}
		1939	/* enabled rb are just the one not disabled :) */
		1940	disabled_rb_mask = tmp;
1963	serge	1941
2997	Serge	1942	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
		1943	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
1963	serge	1944
2997	Serge	1945	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		1946	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
		1947	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1963	serge	1948
2997	Serge	1949	tmp = gb_addr_config & NUM_PIPES_MASK;
		1950	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
		1951	EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
		1952	WREG32(GB_BACKEND_MAP, tmp);
1963	serge	1953
		1954	WREG32(CGTS_SYS_TCC_DISABLE, 0);
		1955	WREG32(CGTS_TCC_DISABLE, 0);
		1956	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
		1957	WREG32(CGTS_USER_TCC_DISABLE, 0);
		1958
		1959	/* set HW defaults for 3D engine */
		1960	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \|
		1961	ROQ_IB2_START(0x2b)));
		1962
		1963	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
		1964
		1965	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \|
		1966	SYNC_GRADIENT \|
		1967	SYNC_WALKER \|
		1968	SYNC_ALIGNER));
		1969
		1970	sx_debug_1 = RREG32(SX_DEBUG_1);
		1971	sx_debug_1 \|= ENABLE_NEW_SMX_ADDRESS;
		1972	WREG32(SX_DEBUG_1, sx_debug_1);
		1973
		1974
		1975	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
		1976	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
		1977	smx_dc_ctl0 \|= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
		1978	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
		1979
2997	Serge	1980	if (rdev->family <= CHIP_SUMO2)
		1981	WREG32(SMX_SAR_CTL0, 0x00010000);
		1982
1963	serge	1983	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) \|
		1984	POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) \|
		1985	SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
		1986
		1987	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) \|
		1988	SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) \|
		1989	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
		1990
		1991	WREG32(VGT_NUM_INSTANCES, 1);
		1992	WREG32(SPI_CONFIG_CNTL, 0);
		1993	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
		1994	WREG32(CP_PERFMON_CNTL, 0);
		1995
		1996	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) \|
		1997	FETCH_FIFO_HIWATER(0x4) \|
		1998	DONE_FIFO_HIWATER(0xe0) \|
		1999	ALU_UPDATE_FIFO_HIWATER(0x8)));
		2000
		2001	sq_config = RREG32(SQ_CONFIG);
		2002	sq_config &= ~(PS_PRIO(3) \|
		2003	VS_PRIO(3) \|
		2004	GS_PRIO(3) \|
		2005	ES_PRIO(3));
		2006	sq_config \|= (VC_ENABLE \|
		2007	EXPORT_SRC_C \|
		2008	PS_PRIO(0) \|
		2009	VS_PRIO(1) \|
		2010	GS_PRIO(2) \|
		2011	ES_PRIO(3));
		2012
		2013	switch (rdev->family) {
		2014	case CHIP_CEDAR:
		2015	case CHIP_PALM:
		2016	case CHIP_SUMO:
		2017	case CHIP_SUMO2:
		2018	case CHIP_CAICOS:
		2019	/* no vertex cache */
		2020	sq_config &= ~VC_ENABLE;
		2021	break;
		2022	default:
		2023	break;
		2024	}
		2025
		2026	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
		2027
		2028	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
		2029	sq_gpr_resource_mgmt_1 \|= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
		2030	sq_gpr_resource_mgmt_1 \|= NUM_CLAUSE_TEMP_GPRS(4);
		2031	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		2032	sq_gpr_resource_mgmt_2 \|= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		2033	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		2034	sq_gpr_resource_mgmt_3 \|= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		2035
		2036	switch (rdev->family) {
		2037	case CHIP_CEDAR:
		2038	case CHIP_PALM:
		2039	case CHIP_SUMO:
		2040	case CHIP_SUMO2:
		2041	ps_thread_count = 96;
		2042	break;
		2043	default:
		2044	ps_thread_count = 128;
		2045	break;
		2046	}
		2047
		2048	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
		2049	sq_thread_resource_mgmt \|= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2050	sq_thread_resource_mgmt \|= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2051	sq_thread_resource_mgmt \|= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2052	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2053	sq_thread_resource_mgmt_2 \|= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2054
		2055	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2056	sq_stack_resource_mgmt_1 \|= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2057	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2058	sq_stack_resource_mgmt_2 \|= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2059	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2060	sq_stack_resource_mgmt_3 \|= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2061
		2062	WREG32(SQ_CONFIG, sq_config);
		2063	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		2064	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		2065	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
		2066	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		2067	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
		2068	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		2069	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		2070	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
		2071	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
		2072	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
		2073
		2074	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		2075	FORCE_EOV_MAX_REZ_CNT(255)));
		2076
		2077	switch (rdev->family) {
		2078	case CHIP_CEDAR:
		2079	case CHIP_PALM:
		2080	case CHIP_SUMO:
		2081	case CHIP_SUMO2:
		2082	case CHIP_CAICOS:
		2083	vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
		2084	break;
		2085	default:
		2086	vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
		2087	break;
		2088	}
		2089	vgt_cache_invalidation \|= AUTO_INVLD_EN(ES_AND_GS_AUTO);
		2090	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
		2091
		2092	WREG32(VGT_GS_VERTEX_REUSE, 16);
		2093	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
		2094	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		2095
		2096	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
		2097	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
		2098
		2099	WREG32(CB_PERF_CTR0_SEL_0, 0);
		2100	WREG32(CB_PERF_CTR0_SEL_1, 0);
		2101	WREG32(CB_PERF_CTR1_SEL_0, 0);
		2102	WREG32(CB_PERF_CTR1_SEL_1, 0);
		2103	WREG32(CB_PERF_CTR2_SEL_0, 0);
		2104	WREG32(CB_PERF_CTR2_SEL_1, 0);
		2105	WREG32(CB_PERF_CTR3_SEL_0, 0);
		2106	WREG32(CB_PERF_CTR3_SEL_1, 0);
		2107
		2108	/* clear render buffer base addresses */
		2109	WREG32(CB_COLOR0_BASE, 0);
		2110	WREG32(CB_COLOR1_BASE, 0);
		2111	WREG32(CB_COLOR2_BASE, 0);
		2112	WREG32(CB_COLOR3_BASE, 0);
		2113	WREG32(CB_COLOR4_BASE, 0);
		2114	WREG32(CB_COLOR5_BASE, 0);
		2115	WREG32(CB_COLOR6_BASE, 0);
		2116	WREG32(CB_COLOR7_BASE, 0);
		2117	WREG32(CB_COLOR8_BASE, 0);
		2118	WREG32(CB_COLOR9_BASE, 0);
		2119	WREG32(CB_COLOR10_BASE, 0);
		2120	WREG32(CB_COLOR11_BASE, 0);
		2121
		2122	/* set the shader const cache sizes to 0 */
		2123	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
		2124	WREG32(i, 0);
		2125	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
		2126	WREG32(i, 0);
		2127
		2128	tmp = RREG32(HDP_MISC_CNTL);
		2129	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		2130	WREG32(HDP_MISC_CNTL, tmp);
		2131
		2132	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		2133	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		2134
		2135	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		2136
		2137	udelay(50);
		2138
1430	serge	2139	}
		2140
		2141	int evergreen_mc_init(struct radeon_device *rdev)
		2142	{
		2143	u32 tmp;
		2144	int chansize, numchan;
		2145
		2146	/* Get VRAM informations */
		2147	rdev->mc.vram_is_ddr = true;
2997	Serge	2148	if ((rdev->family == CHIP_PALM) \|\|
		2149	(rdev->family == CHIP_SUMO) \|\|
		2150	(rdev->family == CHIP_SUMO2))
2004	serge	2151	tmp = RREG32(FUS_MC_ARB_RAMCFG);
		2152	else
3031	serge	2153	tmp = RREG32(MC_ARB_RAMCFG);
1430	serge	2154	if (tmp & CHANSIZE_OVERRIDE) {
		2155	chansize = 16;
		2156	} else if (tmp & CHANSIZE_MASK) {
		2157	chansize = 64;
		2158	} else {
		2159	chansize = 32;
		2160	}
		2161	tmp = RREG32(MC_SHARED_CHMAP);
		2162	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		2163	case 0:
		2164	default:
		2165	numchan = 1;
		2166	break;
		2167	case 1:
		2168	numchan = 2;
		2169	break;
		2170	case 2:
		2171	numchan = 4;
		2172	break;
		2173	case 3:
		2174	numchan = 8;
		2175	break;
		2176	}
		2177	rdev->mc.vram_width = numchan * chansize;
		2178	/* Could aper size report 0 ? */
1963	serge	2179	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		2180	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1430	serge	2181	/* Setup GPU memory space */
2997	Serge	2182	if ((rdev->family == CHIP_PALM) \|\|
		2183	(rdev->family == CHIP_SUMO) \|\|
		2184	(rdev->family == CHIP_SUMO2)) {
1963	serge	2185	/* size in bytes on fusion */
		2186	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		2187	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
		2188	} else {
2997	Serge	2189	/* size in MB on evergreen/cayman/tn */
3031	serge	2190	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
		2191	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
1963	serge	2192	}
1430	serge	2193	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1963	serge	2194	r700_vram_gtt_location(rdev, &rdev->mc);
		2195	radeon_update_bandwidth_info(rdev);
		2196
1430	serge	2197	return 0;
		2198	}
		2199
2997	Serge	2200	bool evergreen_gpu_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
1430	serge	2201	{
1986	serge	2202	u32 srbm_status;
		2203	u32 grbm_status;
		2204	u32 grbm_status_se0, grbm_status_se1;
		2205
		2206	srbm_status = RREG32(SRBM_STATUS);
		2207	grbm_status = RREG32(GRBM_STATUS);
		2208	grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
		2209	grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
		2210	if (!(grbm_status & GUI_ACTIVE)) {
2997	Serge	2211	radeon_ring_lockup_update(ring);
3031	serge	2212	return false;
1986	serge	2213	}
		2214	/* force CP activities */
2997	Serge	2215	radeon_ring_force_activity(rdev, ring);
		2216	return radeon_ring_test_lockup(rdev, ring);
1963	serge	2217	}
		2218
		2219	static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
		2220	{
		2221	struct evergreen_mc_save save;
		2222	u32 grbm_reset = 0;
		2223
		2224	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
		2225	return 0;
		2226
		2227	dev_info(rdev->dev, "GPU softreset \n");
		2228	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2229	RREG32(GRBM_STATUS));
		2230	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2231	RREG32(GRBM_STATUS_SE0));
		2232	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2233	RREG32(GRBM_STATUS_SE1));
		2234	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2235	RREG32(SRBM_STATUS));
2997	Serge	2236	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		2237	RREG32(CP_STALLED_STAT1));
		2238	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		2239	RREG32(CP_STALLED_STAT2));
		2240	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		2241	RREG32(CP_BUSY_STAT));
		2242	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		2243	RREG32(CP_STAT));
1963	serge	2244	evergreen_mc_stop(rdev, &save);
		2245	if (evergreen_mc_wait_for_idle(rdev)) {
		2246	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		2247	}
		2248	/* Disable CP parsing/prefetching */
		2249	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT);
		2250
		2251	/* reset all the gfx blocks */
		2252	grbm_reset = (SOFT_RESET_CP \|
		2253	SOFT_RESET_CB \|
		2254	SOFT_RESET_DB \|
		2255	SOFT_RESET_PA \|
		2256	SOFT_RESET_SC \|
		2257	SOFT_RESET_SPI \|
		2258	SOFT_RESET_SH \|
		2259	SOFT_RESET_SX \|
		2260	SOFT_RESET_TC \|
		2261	SOFT_RESET_TA \|
		2262	SOFT_RESET_VC \|
		2263	SOFT_RESET_VGT);
		2264
		2265	dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
		2266	WREG32(GRBM_SOFT_RESET, grbm_reset);
		2267	(void)RREG32(GRBM_SOFT_RESET);
		2268	udelay(50);
		2269	WREG32(GRBM_SOFT_RESET, 0);
		2270	(void)RREG32(GRBM_SOFT_RESET);
		2271	/* Wait a little for things to settle down */
		2272	udelay(50);
		2273	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2274	RREG32(GRBM_STATUS));
		2275	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2276	RREG32(GRBM_STATUS_SE0));
		2277	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2278	RREG32(GRBM_STATUS_SE1));
		2279	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2280	RREG32(SRBM_STATUS));
2997	Serge	2281	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		2282	RREG32(CP_STALLED_STAT1));
		2283	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		2284	RREG32(CP_STALLED_STAT2));
		2285	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		2286	RREG32(CP_BUSY_STAT));
		2287	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		2288	RREG32(CP_STAT));
1963	serge	2289	evergreen_mc_resume(rdev, &save);
1430	serge	2290	return 0;
		2291	}
		2292
1963	serge	2293	int evergreen_asic_reset(struct radeon_device *rdev)
		2294	{
		2295	return evergreen_gpu_soft_reset(rdev);
		2296	}
		2297
		2298	/* Interrupts */
		2299
		2300	u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
		2301	{
2997	Serge	2302	if (crtc >= rdev->num_crtc)
3031	serge	2303	return 0;
2997	Serge	2304	else
		2305	return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
1963	serge	2306	}
		2307
		2308	void evergreen_disable_interrupt_state(struct radeon_device *rdev)
		2309	{
		2310	u32 tmp;
		2311
2997	Serge	2312	if (rdev->family >= CHIP_CAYMAN) {
		2313	cayman_cp_int_cntl_setup(rdev, 0,
		2314	CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		2315	cayman_cp_int_cntl_setup(rdev, 1, 0);
		2316	cayman_cp_int_cntl_setup(rdev, 2, 0);
		2317	} else
3031	serge	2318	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
1963	serge	2319	WREG32(GRBM_INT_CNTL, 0);
		2320	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2321	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005	serge	2322	if (rdev->num_crtc >= 4) {
3031	serge	2323	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2324	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005	serge	2325	}
		2326	if (rdev->num_crtc >= 6) {
3031	serge	2327	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2328	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	2329	}
		2330
		2331	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2332	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005	serge	2333	if (rdev->num_crtc >= 4) {
3031	serge	2334	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2335	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005	serge	2336	}
		2337	if (rdev->num_crtc >= 6) {
3031	serge	2338	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2339	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	2340	}
		2341
2997	Serge	2342	/* only one DAC on DCE6 */
		2343	if (!ASIC_IS_DCE6(rdev))
3031	serge	2344	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
1963	serge	2345	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2346
		2347	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2348	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2349	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2350	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2351	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2352	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2353	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2354	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2355	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2356	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2357	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2358	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2359
		2360	}
2005	serge	2361
		2362	int evergreen_irq_set(struct radeon_device *rdev)
		2363	{
		2364	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
2997	Serge	2365	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
2005	serge	2366	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
		2367	u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
		2368	u32 grbm_int_cntl = 0;
		2369	u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2997	Serge	2370	u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
2005	serge	2371
		2372	if (!rdev->irq.installed) {
		2373	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		2374	return -EINVAL;
		2375	}
		2376	/* don't enable anything if the ih is disabled */
		2377	if (!rdev->ih.enabled) {
		2378	r600_disable_interrupts(rdev);
		2379	/* force the active interrupt state to all disabled */
		2380	evergreen_disable_interrupt_state(rdev);
		2381	return 0;
		2382	}
		2383
		2384	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2385	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2386	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2387	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2388	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2389	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2390
2997	Serge	2391	afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2392	afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2393	afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2394	afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2395	afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2396	afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2397
		2398	if (rdev->family >= CHIP_CAYMAN) {
		2399	/* enable CP interrupts on all rings */
		2400	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		2401	DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
		2402	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		2403	}
		2404	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
		2405	DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
		2406	cp_int_cntl1 \|= TIME_STAMP_INT_ENABLE;
		2407	}
		2408	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
		2409	DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
		2410	cp_int_cntl2 \|= TIME_STAMP_INT_ENABLE;
		2411	}
		2412	} else {
		2413	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		2414	DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
3031	serge	2415	cp_int_cntl \|= RB_INT_ENABLE;
		2416	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		2417	}
2005	serge	2418	}
2997	Serge	2419
2005	serge	2420	if (rdev->irq.crtc_vblank_int[0] \|\|
2997	Serge	2421	atomic_read(&rdev->irq.pflip[0])) {
2005	serge	2422	DRM_DEBUG("evergreen_irq_set: vblank 0\n");
		2423	crtc1 \|= VBLANK_INT_MASK;
		2424	}
		2425	if (rdev->irq.crtc_vblank_int[1] \|\|
2997	Serge	2426	atomic_read(&rdev->irq.pflip[1])) {
2005	serge	2427	DRM_DEBUG("evergreen_irq_set: vblank 1\n");
		2428	crtc2 \|= VBLANK_INT_MASK;
		2429	}
		2430	if (rdev->irq.crtc_vblank_int[2] \|\|
2997	Serge	2431	atomic_read(&rdev->irq.pflip[2])) {
2005	serge	2432	DRM_DEBUG("evergreen_irq_set: vblank 2\n");
		2433	crtc3 \|= VBLANK_INT_MASK;
		2434	}
		2435	if (rdev->irq.crtc_vblank_int[3] \|\|
2997	Serge	2436	atomic_read(&rdev->irq.pflip[3])) {
2005	serge	2437	DRM_DEBUG("evergreen_irq_set: vblank 3\n");
		2438	crtc4 \|= VBLANK_INT_MASK;
		2439	}
		2440	if (rdev->irq.crtc_vblank_int[4] \|\|
2997	Serge	2441	atomic_read(&rdev->irq.pflip[4])) {
2005	serge	2442	DRM_DEBUG("evergreen_irq_set: vblank 4\n");
		2443	crtc5 \|= VBLANK_INT_MASK;
		2444	}
		2445	if (rdev->irq.crtc_vblank_int[5] \|\|
2997	Serge	2446	atomic_read(&rdev->irq.pflip[5])) {
2005	serge	2447	DRM_DEBUG("evergreen_irq_set: vblank 5\n");
		2448	crtc6 \|= VBLANK_INT_MASK;
		2449	}
		2450	if (rdev->irq.hpd[0]) {
		2451	DRM_DEBUG("evergreen_irq_set: hpd 1\n");
		2452	hpd1 \|= DC_HPDx_INT_EN;
		2453	}
		2454	if (rdev->irq.hpd[1]) {
		2455	DRM_DEBUG("evergreen_irq_set: hpd 2\n");
		2456	hpd2 \|= DC_HPDx_INT_EN;
		2457	}
		2458	if (rdev->irq.hpd[2]) {
		2459	DRM_DEBUG("evergreen_irq_set: hpd 3\n");
		2460	hpd3 \|= DC_HPDx_INT_EN;
		2461	}
		2462	if (rdev->irq.hpd[3]) {
		2463	DRM_DEBUG("evergreen_irq_set: hpd 4\n");
		2464	hpd4 \|= DC_HPDx_INT_EN;
		2465	}
		2466	if (rdev->irq.hpd[4]) {
		2467	DRM_DEBUG("evergreen_irq_set: hpd 5\n");
		2468	hpd5 \|= DC_HPDx_INT_EN;
		2469	}
		2470	if (rdev->irq.hpd[5]) {
		2471	DRM_DEBUG("evergreen_irq_set: hpd 6\n");
		2472	hpd6 \|= DC_HPDx_INT_EN;
		2473	}
2997	Serge	2474	if (rdev->irq.afmt[0]) {
		2475	DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
		2476	afmt1 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
2005	serge	2477	}
2997	Serge	2478	if (rdev->irq.afmt[1]) {
		2479	DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
		2480	afmt2 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2481	}
		2482	if (rdev->irq.afmt[2]) {
		2483	DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
		2484	afmt3 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2485	}
		2486	if (rdev->irq.afmt[3]) {
		2487	DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
		2488	afmt4 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2489	}
		2490	if (rdev->irq.afmt[4]) {
		2491	DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
		2492	afmt5 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2493	}
		2494	if (rdev->irq.afmt[5]) {
		2495	DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
		2496	afmt6 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		2497	}
2005	serge	2498
2997	Serge	2499	if (rdev->family >= CHIP_CAYMAN) {
		2500	cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
		2501	cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
		2502	cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
		2503	} else
3031	serge	2504	WREG32(CP_INT_CNTL, cp_int_cntl);
2005	serge	2505	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		2506
		2507	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
		2508	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
		2509	if (rdev->num_crtc >= 4) {
		2510	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
		2511	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
		2512	}
		2513	if (rdev->num_crtc >= 6) {
		2514	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
		2515	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
		2516	}
		2517
		2518	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
		2519	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
		2520	if (rdev->num_crtc >= 4) {
		2521	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
		2522	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
		2523	}
		2524	if (rdev->num_crtc >= 6) {
		2525	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
		2526	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
		2527	}
		2528
		2529	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		2530	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		2531	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		2532	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		2533	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		2534	WREG32(DC_HPD6_INT_CONTROL, hpd6);
		2535
2997	Serge	2536	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
		2537	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
		2538	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
		2539	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
		2540	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
		2541	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
		2542
2005	serge	2543	return 0;
		2544	}
		2545
2997	Serge	2546	static void evergreen_irq_ack(struct radeon_device *rdev)
2005	serge	2547	{
		2548	u32 tmp;
		2549
		2550	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		2551	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		2552	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
		2553	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
		2554	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
		2555	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
		2556	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2557	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2558	if (rdev->num_crtc >= 4) {
		2559	rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2560	rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2561	}
		2562	if (rdev->num_crtc >= 6) {
		2563	rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2564	rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2565	}
		2566
2997	Serge	2567	rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2568	rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2569	rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2570	rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2571	rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2572	rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2573
2005	serge	2574	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
		2575	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2576	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
		2577	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2578	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
		2579	WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
		2580	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
		2581	WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
		2582	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
		2583	WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
		2584	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
		2585	WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
		2586
		2587	if (rdev->num_crtc >= 4) {
		2588	if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
		2589	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2590	if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
		2591	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2592	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
		2593	WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
		2594	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
		2595	WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
		2596	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
		2597	WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
		2598	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
		2599	WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
		2600	}
		2601
		2602	if (rdev->num_crtc >= 6) {
		2603	if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
		2604	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2605	if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
		2606	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		2607	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
		2608	WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
		2609	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
		2610	WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
		2611	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
		2612	WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
		2613	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
		2614	WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
		2615	}
		2616
		2617	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		2618	tmp = RREG32(DC_HPD1_INT_CONTROL);
		2619	tmp \|= DC_HPDx_INT_ACK;
		2620	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2621	}
		2622	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		2623	tmp = RREG32(DC_HPD2_INT_CONTROL);
		2624	tmp \|= DC_HPDx_INT_ACK;
		2625	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2626	}
		2627	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		2628	tmp = RREG32(DC_HPD3_INT_CONTROL);
		2629	tmp \|= DC_HPDx_INT_ACK;
		2630	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2631	}
		2632	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		2633	tmp = RREG32(DC_HPD4_INT_CONTROL);
		2634	tmp \|= DC_HPDx_INT_ACK;
		2635	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2636	}
		2637	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		2638	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2639	tmp \|= DC_HPDx_INT_ACK;
		2640	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2641	}
		2642	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		2643	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2644	tmp \|= DC_HPDx_INT_ACK;
		2645	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2646	}
2997	Serge	2647	if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
		2648	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2649	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2650	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
		2651	}
		2652	if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
		2653	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2654	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2655	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
		2656	}
		2657	if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
		2658	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2659	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2660	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
		2661	}
		2662	if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
		2663	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2664	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2665	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
		2666	}
		2667	if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
		2668	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2669	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2670	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
		2671	}
		2672	if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
		2673	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2674	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2675	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
		2676	}
2005	serge	2677	}
2997	Serge	2678
		2679	static void evergreen_irq_disable(struct radeon_device *rdev)
2005	serge	2680	{
2997	Serge	2681	r600_disable_interrupts(rdev);
		2682	/* Wait and acknowledge irq */
		2683	mdelay(1);
		2684	evergreen_irq_ack(rdev);
		2685	evergreen_disable_interrupt_state(rdev);
		2686	}
		2687
		2688	void evergreen_irq_suspend(struct radeon_device *rdev)
		2689	{
		2690	evergreen_irq_disable(rdev);
		2691	r600_rlc_stop(rdev);
		2692	}
		2693
		2694	static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
		2695	{
2005	serge	2696	u32 wptr, tmp;
		2697
		2698	if (rdev->wb.enabled)
		2699	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		2700	else
		2701	wptr = RREG32(IH_RB_WPTR);
		2702
		2703	if (wptr & RB_OVERFLOW) {
		2704	/* When a ring buffer overflow happen start parsing interrupt
		2705	* from the last not overwritten vector (wptr + 16). Hopefully
		2706	* this should allow us to catchup.
		2707	*/
		2708	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
		2709	wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
		2710	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		2711	tmp = RREG32(IH_RB_CNTL);
		2712	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		2713	WREG32(IH_RB_CNTL, tmp);
		2714	}
		2715	return (wptr & rdev->ih.ptr_mask);
		2716	}
		2717
		2718	int evergreen_irq_process(struct radeon_device *rdev)
		2719	{
		2720	u32 wptr;
		2721	u32 rptr;
		2722	u32 src_id, src_data;
		2723	u32 ring_index;
		2724	bool queue_hotplug = false;
2997	Serge	2725	bool queue_hdmi = false;
2005	serge	2726
		2727	if (!rdev->ih.enabled \|\| rdev->shutdown)
		2728	return IRQ_NONE;
		2729
		2730	wptr = evergreen_get_ih_wptr(rdev);
2997	Serge	2731
		2732	restart_ih:
		2733	/* is somebody else already processing irqs? */
		2734	if (atomic_xchg(&rdev->ih.lock, 1))
		2735	return IRQ_NONE;
		2736
2005	serge	2737	rptr = rdev->ih.rptr;
		2738	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		2739
2175	serge	2740	/* Order reading of wptr vs. reading of IH ring data */
		2741	rmb();
		2742
2005	serge	2743	/* display interrupts */
		2744	evergreen_irq_ack(rdev);
		2745
		2746	while (rptr != wptr) {
		2747	/* wptr/rptr are in bytes! */
		2748	ring_index = rptr / 4;
		2749	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		2750	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		2751
		2752	switch (src_id) {
		2753	case 1: /* D1 vblank/vline */
		2754	switch (src_data) {
		2755	case 0: /* D1 vblank */
		2756	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
		2757	if (rdev->irq.crtc_vblank_int[0]) {
		2758	// drm_handle_vblank(rdev->ddev, 0);
		2759	rdev->pm.vblank_sync = true;
		2760	// wake_up(&rdev->irq.vblank_queue);
		2761	}
		2762	// if (rdev->irq.pflip[0])
		2763	// radeon_crtc_handle_flip(rdev, 0);
		2764	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		2765	DRM_DEBUG("IH: D1 vblank\n");
		2766	}
		2767	break;
		2768	case 1: /* D1 vline */
		2769	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
		2770	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		2771	DRM_DEBUG("IH: D1 vline\n");
		2772	}
		2773	break;
		2774	default:
		2775	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2776	break;
		2777	}
		2778	break;
		2779	case 2: /* D2 vblank/vline */
		2780	switch (src_data) {
		2781	case 0: /* D2 vblank */
		2782	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
		2783	if (rdev->irq.crtc_vblank_int[1]) {
		2784	// drm_handle_vblank(rdev->ddev, 1);
		2785	rdev->pm.vblank_sync = true;
		2786	// wake_up(&rdev->irq.vblank_queue);
		2787	}
		2788	// if (rdev->irq.pflip[1])
		2789	// radeon_crtc_handle_flip(rdev, 1);
		2790	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
		2791	DRM_DEBUG("IH: D2 vblank\n");
		2792	}
		2793	break;
		2794	case 1: /* D2 vline */
		2795	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
		2796	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
		2797	DRM_DEBUG("IH: D2 vline\n");
		2798	}
		2799	break;
		2800	default:
		2801	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2802	break;
		2803	}
		2804	break;
		2805	case 3: /* D3 vblank/vline */
		2806	switch (src_data) {
		2807	case 0: /* D3 vblank */
		2808	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
		2809	if (rdev->irq.crtc_vblank_int[2]) {
		2810	// drm_handle_vblank(rdev->ddev, 2);
		2811	rdev->pm.vblank_sync = true;
		2812	// wake_up(&rdev->irq.vblank_queue);
		2813	}
		2814	// if (rdev->irq.pflip[2])
		2815	// radeon_crtc_handle_flip(rdev, 2);
		2816	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
		2817	DRM_DEBUG("IH: D3 vblank\n");
		2818	}
		2819	break;
		2820	case 1: /* D3 vline */
		2821	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
		2822	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
		2823	DRM_DEBUG("IH: D3 vline\n");
		2824	}
		2825	break;
		2826	default:
		2827	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2828	break;
		2829	}
		2830	break;
		2831	case 4: /* D4 vblank/vline */
		2832	switch (src_data) {
		2833	case 0: /* D4 vblank */
		2834	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
		2835	if (rdev->irq.crtc_vblank_int[3]) {
		2836	// drm_handle_vblank(rdev->ddev, 3);
		2837	rdev->pm.vblank_sync = true;
		2838	// wake_up(&rdev->irq.vblank_queue);
		2839	}
		2840	// if (rdev->irq.pflip[3])
		2841	// radeon_crtc_handle_flip(rdev, 3);
		2842	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
		2843	DRM_DEBUG("IH: D4 vblank\n");
		2844	}
		2845	break;
		2846	case 1: /* D4 vline */
		2847	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
		2848	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
		2849	DRM_DEBUG("IH: D4 vline\n");
		2850	}
		2851	break;
		2852	default:
		2853	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2854	break;
		2855	}
		2856	break;
		2857	case 5: /* D5 vblank/vline */
		2858	switch (src_data) {
		2859	case 0: /* D5 vblank */
		2860	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
		2861	if (rdev->irq.crtc_vblank_int[4]) {
		2862	// drm_handle_vblank(rdev->ddev, 4);
		2863	rdev->pm.vblank_sync = true;
		2864	// wake_up(&rdev->irq.vblank_queue);
		2865	}
		2866	// if (rdev->irq.pflip[4])
		2867	// radeon_crtc_handle_flip(rdev, 4);
		2868	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
		2869	DRM_DEBUG("IH: D5 vblank\n");
		2870	}
		2871	break;
		2872	case 1: /* D5 vline */
		2873	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
		2874	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
		2875	DRM_DEBUG("IH: D5 vline\n");
		2876	}
		2877	break;
		2878	default:
		2879	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2880	break;
		2881	}
		2882	break;
		2883	case 6: /* D6 vblank/vline */
		2884	switch (src_data) {
		2885	case 0: /* D6 vblank */
		2886	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
		2887	if (rdev->irq.crtc_vblank_int[5]) {
		2888	// drm_handle_vblank(rdev->ddev, 5);
		2889	rdev->pm.vblank_sync = true;
		2890	// wake_up(&rdev->irq.vblank_queue);
		2891	}
		2892	// if (rdev->irq.pflip[5])
		2893	// radeon_crtc_handle_flip(rdev, 5);
		2894	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
		2895	DRM_DEBUG("IH: D6 vblank\n");
		2896	}
		2897	break;
		2898	case 1: /* D6 vline */
		2899	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
		2900	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
		2901	DRM_DEBUG("IH: D6 vline\n");
		2902	}
		2903	break;
		2904	default:
		2905	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2906	break;
		2907	}
		2908	break;
		2909	case 42: /* HPD hotplug */
		2910	switch (src_data) {
		2911	case 0:
		2912	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		2913	rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
		2914	queue_hotplug = true;
		2915	DRM_DEBUG("IH: HPD1\n");
		2916	}
		2917	break;
		2918	case 1:
		2919	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		2920	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
		2921	queue_hotplug = true;
		2922	DRM_DEBUG("IH: HPD2\n");
		2923	}
		2924	break;
		2925	case 2:
		2926	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		2927	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
		2928	queue_hotplug = true;
		2929	DRM_DEBUG("IH: HPD3\n");
		2930	}
		2931	break;
		2932	case 3:
		2933	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		2934	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
		2935	queue_hotplug = true;
		2936	DRM_DEBUG("IH: HPD4\n");
		2937	}
		2938	break;
		2939	case 4:
		2940	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		2941	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
		2942	queue_hotplug = true;
		2943	DRM_DEBUG("IH: HPD5\n");
		2944	}
		2945	break;
		2946	case 5:
		2947	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		2948	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
		2949	queue_hotplug = true;
		2950	DRM_DEBUG("IH: HPD6\n");
		2951	}
		2952	break;
		2953	default:
		2954	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2955	break;
		2956	}
		2957	break;
2997	Serge	2958	case 44: /* hdmi */
		2959	switch (src_data) {
		2960	case 0:
		2961	if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
		2962	rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
		2963	queue_hdmi = true;
		2964	DRM_DEBUG("IH: HDMI0\n");
		2965	}
		2966	break;
		2967	case 1:
		2968	if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
		2969	rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
		2970	queue_hdmi = true;
		2971	DRM_DEBUG("IH: HDMI1\n");
		2972	}
		2973	break;
		2974	case 2:
		2975	if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
		2976	rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
		2977	queue_hdmi = true;
		2978	DRM_DEBUG("IH: HDMI2\n");
		2979	}
		2980	break;
		2981	case 3:
		2982	if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
		2983	rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
		2984	queue_hdmi = true;
		2985	DRM_DEBUG("IH: HDMI3\n");
		2986	}
		2987	break;
		2988	case 4:
		2989	if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
		2990	rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
		2991	queue_hdmi = true;
		2992	DRM_DEBUG("IH: HDMI4\n");
		2993	}
		2994	break;
		2995	case 5:
		2996	if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
		2997	rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
		2998	queue_hdmi = true;
		2999	DRM_DEBUG("IH: HDMI5\n");
		3000	}
		3001	break;
		3002	default:
		3003	DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
		3004	break;
		3005	}
		3006	break;
2005	serge	3007	case 176: /* CP_INT in ring buffer */
		3008	case 177: /* CP_INT in IB1 */
		3009	case 178: /* CP_INT in IB2 */
		3010	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2997	Serge	3011	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2005	serge	3012	break;
		3013	case 181: /* CP EOP event */
		3014	DRM_DEBUG("IH: CP EOP\n");
2997	Serge	3015	if (rdev->family >= CHIP_CAYMAN) {
		3016	switch (src_data) {
		3017	case 0:
		3018	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
		3019	break;
		3020	case 1:
		3021	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		3022	break;
		3023	case 2:
		3024	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		3025	break;
		3026	}
		3027	} else
		3028	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2005	serge	3029	break;
		3030	case 233: /* GUI IDLE */
		3031	DRM_DEBUG("IH: GUI idle\n");
		3032	break;
		3033	default:
		3034	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3035	break;
		3036	}
		3037
		3038	/* wptr/rptr are in bytes! */
		3039	rptr += 16;
		3040	rptr &= rdev->ih.ptr_mask;
		3041	}
2997	Serge	3042	rdev->ih.rptr = rptr;
		3043	WREG32(IH_RB_RPTR, rdev->ih.rptr);
		3044	atomic_set(&rdev->ih.lock, 0);
		3045
2005	serge	3046	/* make sure wptr hasn't changed while processing */
		3047	wptr = evergreen_get_ih_wptr(rdev);
2997	Serge	3048	if (wptr != rptr)
2005	serge	3049	goto restart_ih;
2997	Serge	3050
2005	serge	3051	return IRQ_HANDLED;
		3052	}
		3053
1430	serge	3054	static int evergreen_startup(struct radeon_device *rdev)
		3055	{
2997	Serge	3056	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1430	serge	3057	int r;
		3058
1990	serge	3059	/* enable pcie gen2 link */
3031	serge	3060	evergreen_pcie_gen2_enable(rdev);
1990	serge	3061
		3062	if (ASIC_IS_DCE5(rdev)) {
		3063	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw \|\| !rdev->mc_fw) {
		3064	r = ni_init_microcode(rdev);
		3065	if (r) {
		3066	DRM_ERROR("Failed to load firmware!\n");
		3067	return r;
		3068	}
		3069	}
		3070	r = ni_mc_load_microcode(rdev);
		3071	if (r) {
		3072	DRM_ERROR("Failed to load MC firmware!\n");
		3073	return r;
		3074	}
		3075	} else {
3031	serge	3076	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		3077	r = r600_init_microcode(rdev);
		3078	if (r) {
		3079	DRM_ERROR("Failed to load firmware!\n");
		3080	return r;
		3081	}
1430	serge	3082	}
		3083	}
1963	serge	3084
2997	Serge	3085	r = r600_vram_scratch_init(rdev);
		3086	if (r)
		3087	return r;
		3088
1430	serge	3089	evergreen_mc_program(rdev);
		3090	if (rdev->flags & RADEON_IS_AGP) {
1963	serge	3091	evergreen_agp_enable(rdev);
1430	serge	3092	} else {
		3093	r = evergreen_pcie_gart_enable(rdev);
		3094	if (r)
		3095	return r;
		3096	}
		3097	evergreen_gpu_init(rdev);
2005	serge	3098
1963	serge	3099	r = evergreen_blit_init(rdev);
		3100	if (r) {
2997	Serge	3101	// r600_blit_fini(rdev);
		3102	rdev->asic->copy.copy = NULL;
1963	serge	3103	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		3104	}
1430	serge	3105
1963	serge	3106	/* allocate wb buffer */
		3107	r = radeon_wb_init(rdev);
		3108	if (r)
		3109	return r;
		3110
		3111	/* Enable IRQ */
2005	serge	3112	r = r600_irq_init(rdev);
		3113	if (r) {
		3114	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		3115	// radeon_irq_kms_fini(rdev);
		3116	return r;
		3117	}
		3118	evergreen_irq_set(rdev);
1963	serge	3119
2997	Serge	3120	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
		3121	R600_CP_RB_RPTR, R600_CP_RB_WPTR,
		3122	0, 0xfffff, RADEON_CP_PACKET2);
1430	serge	3123	if (r)
		3124	return r;
		3125	r = evergreen_cp_load_microcode(rdev);
		3126	if (r)
		3127	return r;
1963	serge	3128	r = evergreen_cp_resume(rdev);
1430	serge	3129	if (r)
		3130	return r;
1963	serge	3131
1430	serge	3132	return 0;
		3133	}
		3134
		3135
		3136
2997	Serge	3137	#if 0
1430	serge	3138
2005	serge	3139	int evergreen_copy_blit(struct radeon_device *rdev,
		3140	uint64_t src_offset, uint64_t dst_offset,
		3141	unsigned num_pages, struct radeon_fence *fence)
		3142	{
		3143	int r;
1430	serge	3144
2005	serge	3145	mutex_lock(&rdev->r600_blit.mutex);
		3146	rdev->r600_blit.vb_ib = NULL;
		3147	r = evergreen_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
		3148	if (r) {
		3149	if (rdev->r600_blit.vb_ib)
		3150	radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
		3151	mutex_unlock(&rdev->r600_blit.mutex);
		3152	return r;
		3153	}
		3154	evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
		3155	evergreen_blit_done_copy(rdev, fence);
		3156	mutex_unlock(&rdev->r600_blit.mutex);
		3157	return 0;
		3158	}
2997	Serge	3159	#endif
1430	serge	3160
		3161	/* Plan is to move initialization in that function and use
		3162	* helper function so that radeon_device_init pretty much
		3163	* do nothing more than calling asic specific function. This
		3164	* should also allow to remove a bunch of callback function
		3165	* like vram_info.
		3166	*/
		3167	int evergreen_init(struct radeon_device *rdev)
		3168	{
		3169	int r;
		3170
		3171	/* Read BIOS */
		3172	if (!radeon_get_bios(rdev)) {
		3173	if (ASIC_IS_AVIVO(rdev))
		3174	return -EINVAL;
		3175	}
		3176	/* Must be an ATOMBIOS */
		3177	if (!rdev->is_atom_bios) {
1986	serge	3178	dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
1430	serge	3179	return -EINVAL;
		3180	}
		3181	r = radeon_atombios_init(rdev);
		3182	if (r)
		3183	return r;
1986	serge	3184	/* reset the asic, the gfx blocks are often in a bad state
		3185	* after the driver is unloaded or after a resume
		3186	*/
		3187	if (radeon_asic_reset(rdev))
		3188	dev_warn(rdev->dev, "GPU reset failed !\n");
1430	serge	3189	/* Post card if necessary */
1986	serge	3190	if (!radeon_card_posted(rdev)) {
1430	serge	3191	if (!rdev->bios) {
		3192	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		3193	return -EINVAL;
		3194	}
		3195	DRM_INFO("GPU not posted. posting now...\n");
		3196	atom_asic_init(rdev->mode_info.atom_context);
		3197	}
		3198	/* Initialize scratch registers */
		3199	r600_scratch_init(rdev);
		3200	/* Initialize surface registers */
		3201	radeon_surface_init(rdev);
		3202	/* Initialize clocks */
		3203	radeon_get_clock_info(rdev->ddev);
		3204	/* Fence driver */
2005	serge	3205	r = radeon_fence_driver_init(rdev);
		3206	if (r)
		3207	return r;
3031	serge	3208	/* initialize AGP */
1430	serge	3209	if (rdev->flags & RADEON_IS_AGP) {
		3210	r = radeon_agp_init(rdev);
		3211	if (r)
		3212	radeon_agp_disable(rdev);
		3213	}
		3214	/* initialize memory controller */
		3215	r = evergreen_mc_init(rdev);
		3216	if (r)
		3217	return r;
		3218	/* Memory manager */
		3219	r = radeon_bo_init(rdev);
		3220	if (r)
		3221	return r;
1963	serge	3222
2005	serge	3223	r = radeon_irq_kms_init(rdev);
		3224	if (r)
		3225	return r;
1430	serge	3226
2997	Serge	3227	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
		3228	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1430	serge	3229
2005	serge	3230	rdev->ih.ring_obj = NULL;
		3231	r600_ih_ring_init(rdev, 64 * 1024);
1430	serge	3232
		3233	r = r600_pcie_gart_init(rdev);
		3234	if (r)
		3235	return r;
1963	serge	3236
		3237	rdev->accel_working = true;
1430	serge	3238	r = evergreen_startup(rdev);
		3239	if (r) {
1963	serge	3240	dev_err(rdev->dev, "disabling GPU acceleration\n");
1430	serge	3241	rdev->accel_working = false;
		3242	}
2997	Serge	3243
		3244	/* Don't start up if the MC ucode is missing on BTC parts.
		3245	* The default clocks and voltages before the MC ucode
		3246	* is loaded are not suffient for advanced operations.
		3247	*/
		3248	if (ASIC_IS_DCE5(rdev)) {
		3249	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
		3250	DRM_ERROR("radeon: MC ucode required for NI+.\n");
		3251	return -EINVAL;
2005	serge	3252	}
1430	serge	3253	}
2997	Serge	3254
1430	serge	3255	return 0;
		3256	}
		3257
1986	serge	3258
2997	Serge	3259	void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
1430	serge	3260	{
2997	Serge	3261	u32 link_width_cntl, speed_cntl, mask;
		3262	int ret;
1986	serge	3263
		3264	if (radeon_pcie_gen2 == 0)
		3265	return;
		3266
		3267	if (rdev->flags & RADEON_IS_IGP)
		3268	return;
		3269
		3270	if (!(rdev->flags & RADEON_IS_PCIE))
		3271	return;
		3272
		3273	/* x2 cards have a special sequence */
		3274	if (ASIC_IS_X2(rdev))
		3275	return;
		3276
2997	Serge	3277	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
		3278	if (ret != 0)
		3279	return;
		3280
		3281	if (!(mask & DRM_PCIE_SPEED_50))
		3282	return;
		3283
1986	serge	3284	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
2997	Serge	3285	if (speed_cntl & LC_CURRENT_DATA_RATE) {
		3286	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
		3287	return;
		3288	}
		3289
		3290	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
		3291
1986	serge	3292	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) \|\|
		3293	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		3294
		3295	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3296	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3297	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3298
		3299	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3300	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		3301	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3302
		3303	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3304	speed_cntl \|= LC_CLR_FAILED_SPD_CHANGE_CNT;
		3305	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3306
		3307	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3308	speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
		3309	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3310
		3311	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3312	speed_cntl \|= LC_GEN2_EN_STRAP;
		3313	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3314
		3315	} else {
		3316	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3317	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		3318	if (1)
		3319	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3320	else
		3321	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3322	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3323	}
1430	serge	3324	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/evergreen.c – Rev 3031