WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/evergreen.c

Rev	Author	Line No.	Line
1430	serge	1	/*
		2	* Copyright 2010 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	//#include
1963	serge	26	#include
1430	serge	27	#include "drmP.h"
		28	#include "radeon.h"
1963	serge	29	#include "radeon_asic.h"
1430	serge	30	#include "radeon_drm.h"
1963	serge	31	#include "evergreend.h"
1430	serge	32	#include "atom.h"
		33	#include "avivod.h"
		34	#include "evergreen_reg.h"
1986	serge	35	#include "evergreen_blit_shaders.h"
1430	serge	36
1963	serge	37	#define EVERGREEN_PFP_UCODE_SIZE 1120
		38	#define EVERGREEN_PM4_UCODE_SIZE 1376
		39
1430	serge	40	static void evergreen_gpu_init(struct radeon_device *rdev);
		41	void evergreen_fini(struct radeon_device *rdev);
1986	serge	42	static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
1430	serge	43
1990	serge	44
		45
		46	u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
		47	{
		48	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
		49	u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
		50
		51	/* Lock the graphics update lock */
		52	tmp \|= EVERGREEN_GRPH_UPDATE_LOCK;
		53	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		54
		55	/* update the scanout addresses */
		56	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		57	upper_32_bits(crtc_base));
		58	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		59	(u32)crtc_base);
		60
		61	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		62	upper_32_bits(crtc_base));
		63	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		64	(u32)crtc_base);
		65
		66	/* Wait for update_pending to go high. */
		67	while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING));
		68	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
		69
		70	/* Unlock the lock, so double-buffering can take place inside vblank */
		71	tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
		72	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		73
		74	/* Return current update_pending status: */
		75	return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
		76	}
		77
		78	/* get temperature in millidegrees */
		79	int evergreen_get_temp(struct radeon_device *rdev)
		80	{
		81	u32 temp, toffset;
		82	int actual_temp = 0;
		83
		84	if (rdev->family == CHIP_JUNIPER) {
		85	toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
		86	TOFFSET_SHIFT;
		87	temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
		88	TS0_ADC_DOUT_SHIFT;
		89
		90	if (toffset & 0x100)
		91	actual_temp = temp / 2 - (0x200 - toffset);
		92	else
		93	actual_temp = temp / 2 + toffset;
		94
		95	actual_temp = actual_temp * 1000;
		96
		97	} else {
		98	temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
		99	ASIC_T_SHIFT;
		100
		101	if (temp & 0x400)
		102	actual_temp = -256;
		103	else if (temp & 0x200)
		104	actual_temp = 255;
		105	else if (temp & 0x100) {
		106	actual_temp = temp & 0x1ff;
		107	actual_temp \|= ~0x1ff;
		108	} else
		109	actual_temp = temp & 0xff;
		110
		111	actual_temp = (actual_temp * 1000) / 2;
		112	}
		113
		114	return actual_temp;
		115	}
		116
		117	int sumo_get_temp(struct radeon_device *rdev)
		118	{
		119	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
		120	int actual_temp = temp - 49;
		121
		122	return actual_temp * 1000;
		123	}
		124
		125	void evergreen_pm_misc(struct radeon_device *rdev)
		126	{
		127	int req_ps_idx = rdev->pm.requested_power_state_index;
		128	int req_cm_idx = rdev->pm.requested_clock_mode_index;
		129	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
		130	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
		131
		132	if (voltage->type == VOLTAGE_SW) {
		133	/* 0xff01 is a flag rather then an actual voltage */
		134	if (voltage->voltage == 0xff01)
		135	return;
		136	if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
		137	radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
		138	rdev->pm.current_vddc = voltage->voltage;
		139	DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
		140	}
		141	/* 0xff01 is a flag rather then an actual voltage */
		142	if (voltage->vddci == 0xff01)
		143	return;
		144	if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
		145	radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
		146	rdev->pm.current_vddci = voltage->vddci;
		147	DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
		148	}
		149	}
		150	}
		151
		152	void evergreen_pm_prepare(struct radeon_device *rdev)
		153	{
		154	struct drm_device *ddev = rdev->ddev;
		155	struct drm_crtc *crtc;
		156	struct radeon_crtc *radeon_crtc;
		157	u32 tmp;
		158
		159	/* disable any active CRTCs */
		160	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		161	radeon_crtc = to_radeon_crtc(crtc);
		162	if (radeon_crtc->enabled) {
		163	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		164	tmp \|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		165	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		166	}
		167	}
		168	}
		169
		170	void evergreen_pm_finish(struct radeon_device *rdev)
		171	{
		172	struct drm_device *ddev = rdev->ddev;
		173	struct drm_crtc *crtc;
		174	struct radeon_crtc *radeon_crtc;
		175	u32 tmp;
		176
		177	/* enable any active CRTCs */
		178	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		179	radeon_crtc = to_radeon_crtc(crtc);
		180	if (radeon_crtc->enabled) {
		181	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		182	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		183	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		184	}
		185	}
		186	}
		187
1430	serge	188	bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		189	{
		190	bool connected = false;
1963	serge	191
		192	switch (hpd) {
		193	case RADEON_HPD_1:
		194	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		195	connected = true;
		196	break;
		197	case RADEON_HPD_2:
		198	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		199	connected = true;
		200	break;
		201	case RADEON_HPD_3:
		202	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		203	connected = true;
		204	break;
		205	case RADEON_HPD_4:
		206	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		207	connected = true;
		208	break;
		209	case RADEON_HPD_5:
		210	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		211	connected = true;
		212	break;
		213	case RADEON_HPD_6:
		214	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		215	connected = true;
		216	break;
		217	default:
		218	break;
		219	}
		220
1430	serge	221	return connected;
		222	}
		223
		224	void evergreen_hpd_set_polarity(struct radeon_device *rdev,
		225	enum radeon_hpd_id hpd)
		226	{
1963	serge	227	u32 tmp;
		228	bool connected = evergreen_hpd_sense(rdev, hpd);
		229
		230	switch (hpd) {
		231	case RADEON_HPD_1:
		232	tmp = RREG32(DC_HPD1_INT_CONTROL);
		233	if (connected)
		234	tmp &= ~DC_HPDx_INT_POLARITY;
		235	else
		236	tmp \|= DC_HPDx_INT_POLARITY;
		237	WREG32(DC_HPD1_INT_CONTROL, tmp);
		238	break;
		239	case RADEON_HPD_2:
		240	tmp = RREG32(DC_HPD2_INT_CONTROL);
		241	if (connected)
		242	tmp &= ~DC_HPDx_INT_POLARITY;
		243	else
		244	tmp \|= DC_HPDx_INT_POLARITY;
		245	WREG32(DC_HPD2_INT_CONTROL, tmp);
		246	break;
		247	case RADEON_HPD_3:
		248	tmp = RREG32(DC_HPD3_INT_CONTROL);
		249	if (connected)
		250	tmp &= ~DC_HPDx_INT_POLARITY;
		251	else
		252	tmp \|= DC_HPDx_INT_POLARITY;
		253	WREG32(DC_HPD3_INT_CONTROL, tmp);
		254	break;
		255	case RADEON_HPD_4:
		256	tmp = RREG32(DC_HPD4_INT_CONTROL);
		257	if (connected)
		258	tmp &= ~DC_HPDx_INT_POLARITY;
		259	else
		260	tmp \|= DC_HPDx_INT_POLARITY;
		261	WREG32(DC_HPD4_INT_CONTROL, tmp);
		262	break;
		263	case RADEON_HPD_5:
		264	tmp = RREG32(DC_HPD5_INT_CONTROL);
		265	if (connected)
		266	tmp &= ~DC_HPDx_INT_POLARITY;
		267	else
		268	tmp \|= DC_HPDx_INT_POLARITY;
		269	WREG32(DC_HPD5_INT_CONTROL, tmp);
		270	break;
		271	case RADEON_HPD_6:
		272	tmp = RREG32(DC_HPD6_INT_CONTROL);
		273	if (connected)
		274	tmp &= ~DC_HPDx_INT_POLARITY;
		275	else
		276	tmp \|= DC_HPDx_INT_POLARITY;
		277	WREG32(DC_HPD6_INT_CONTROL, tmp);
		278	break;
		279	default:
		280	break;
		281	}
1430	serge	282	}
		283
		284	void evergreen_hpd_init(struct radeon_device *rdev)
		285	{
1963	serge	286	struct drm_device *dev = rdev->ddev;
		287	struct drm_connector *connector;
		288	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \|
		289	DC_HPDx_RX_INT_TIMER(0xfa) \| DC_HPDx_EN;
		290
		291	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		292	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		293	switch (radeon_connector->hpd.hpd) {
		294	case RADEON_HPD_1:
		295	WREG32(DC_HPD1_CONTROL, tmp);
		296	rdev->irq.hpd[0] = true;
		297	break;
		298	case RADEON_HPD_2:
		299	WREG32(DC_HPD2_CONTROL, tmp);
		300	rdev->irq.hpd[1] = true;
		301	break;
		302	case RADEON_HPD_3:
		303	WREG32(DC_HPD3_CONTROL, tmp);
		304	rdev->irq.hpd[2] = true;
		305	break;
		306	case RADEON_HPD_4:
		307	WREG32(DC_HPD4_CONTROL, tmp);
		308	rdev->irq.hpd[3] = true;
		309	break;
		310	case RADEON_HPD_5:
		311	WREG32(DC_HPD5_CONTROL, tmp);
		312	rdev->irq.hpd[4] = true;
		313	break;
		314	case RADEON_HPD_6:
		315	WREG32(DC_HPD6_CONTROL, tmp);
		316	rdev->irq.hpd[5] = true;
		317	break;
		318	default:
		319	break;
		320	}
		321	}
2004	serge	322	// if (rdev->irq.installed)
		323	// evergreen_irq_set(rdev);
1430	serge	324	}
		325
2004	serge	326	#if 0
		327
1963	serge	328	void evergreen_hpd_fini(struct radeon_device *rdev)
		329	{
		330	struct drm_device *dev = rdev->ddev;
		331	struct drm_connector *connector;
1430	serge	332
1963	serge	333	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		334	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		335	switch (radeon_connector->hpd.hpd) {
		336	case RADEON_HPD_1:
		337	WREG32(DC_HPD1_CONTROL, 0);
		338	rdev->irq.hpd[0] = false;
		339	break;
		340	case RADEON_HPD_2:
		341	WREG32(DC_HPD2_CONTROL, 0);
		342	rdev->irq.hpd[1] = false;
		343	break;
		344	case RADEON_HPD_3:
		345	WREG32(DC_HPD3_CONTROL, 0);
		346	rdev->irq.hpd[2] = false;
		347	break;
		348	case RADEON_HPD_4:
		349	WREG32(DC_HPD4_CONTROL, 0);
		350	rdev->irq.hpd[3] = false;
		351	break;
		352	case RADEON_HPD_5:
		353	WREG32(DC_HPD5_CONTROL, 0);
		354	rdev->irq.hpd[4] = false;
		355	break;
		356	case RADEON_HPD_6:
		357	WREG32(DC_HPD6_CONTROL, 0);
		358	rdev->irq.hpd[5] = false;
		359	break;
		360	default:
		361	break;
		362	}
		363	}
1430	serge	364	}
		365
1963	serge	366	#endif
1986	serge	367	/* watermark setup */
1963	serge	368
1986	serge	369	static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
		370	struct radeon_crtc *radeon_crtc,
		371	struct drm_display_mode *mode,
		372	struct drm_display_mode *other_mode)
		373	{
		374	u32 tmp;
		375	/*
		376	* Line Buffer Setup
		377	* There are 3 line buffers, each one shared by 2 display controllers.
		378	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
		379	* the display controllers. The paritioning is done via one of four
		380	* preset allocations specified in bits 2:0:
		381	* first display controller
		382	* 0 - first half of lb (3840 * 2)
		383	* 1 - first 3/4 of lb (5760 * 2)
		384	* 2 - whole lb (7680 * 2), other crtc must be disabled
		385	* 3 - first 1/4 of lb (1920 * 2)
		386	* second display controller
		387	* 4 - second half of lb (3840 * 2)
		388	* 5 - second 3/4 of lb (5760 * 2)
		389	* 6 - whole lb (7680 * 2), other crtc must be disabled
		390	* 7 - last 1/4 of lb (1920 * 2)
		391	*/
		392	/* this can get tricky if we have two large displays on a paired group
		393	* of crtcs. Ideally for multiple large displays we'd assign them to
		394	* non-linked crtcs for maximum line buffer allocation.
		395	*/
		396	if (radeon_crtc->base.enabled && mode) {
		397	if (other_mode)
		398	tmp = 0; /* 1/2 */
		399	else
		400	tmp = 2; /* whole */
		401	} else
		402	tmp = 0;
1963	serge	403
1986	serge	404	/* second controller of the pair uses second half of the lb */
		405	if (radeon_crtc->crtc_id % 2)
		406	tmp += 4;
		407	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
		408
		409	if (radeon_crtc->base.enabled && mode) {
		410	switch (tmp) {
		411	case 0:
		412	case 4:
		413	default:
		414	if (ASIC_IS_DCE5(rdev))
		415	return 4096 * 2;
		416	else
		417	return 3840 * 2;
		418	case 1:
		419	case 5:
		420	if (ASIC_IS_DCE5(rdev))
		421	return 6144 * 2;
		422	else
		423	return 5760 * 2;
		424	case 2:
		425	case 6:
		426	if (ASIC_IS_DCE5(rdev))
		427	return 8192 * 2;
		428	else
		429	return 7680 * 2;
		430	case 3:
		431	case 7:
		432	if (ASIC_IS_DCE5(rdev))
		433	return 2048 * 2;
		434	else
		435	return 1920 * 2;
		436	}
		437	}
		438
		439	/* controller not enabled, so no lb used */
		440	return 0;
		441	}
		442
		443	static u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
		444	{
		445	u32 tmp = RREG32(MC_SHARED_CHMAP);
		446
		447	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		448	case 0:
		449	default:
		450	return 1;
		451	case 1:
		452	return 2;
		453	case 2:
		454	return 4;
		455	case 3:
		456	return 8;
		457	}
		458	}
		459
		460	struct evergreen_wm_params {
		461	u32 dram_channels; /* number of dram channels */
		462	u32 yclk; /* bandwidth per dram data pin in kHz */
		463	u32 sclk; /* engine clock in kHz */
		464	u32 disp_clk; /* display clock in kHz */
		465	u32 src_width; /* viewport width */
		466	u32 active_time; /* active display time in ns */
		467	u32 blank_time; /* blank time in ns */
		468	bool interlaced; /* mode is interlaced */
		469	fixed20_12 vsc; /* vertical scale ratio */
		470	u32 num_heads; /* number of active crtcs */
		471	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
		472	u32 lb_size; /* line buffer allocated to pipe */
		473	u32 vtaps; /* vertical scaler taps */
		474	};
		475
		476	static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
		477	{
		478	/* Calculate DRAM Bandwidth and the part allocated to display. */
		479	fixed20_12 dram_efficiency; /* 0.7 */
		480	fixed20_12 yclk, dram_channels, bandwidth;
		481	fixed20_12 a;
		482
		483	a.full = dfixed_const(1000);
		484	yclk.full = dfixed_const(wm->yclk);
		485	yclk.full = dfixed_div(yclk, a);
		486	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		487	a.full = dfixed_const(10);
		488	dram_efficiency.full = dfixed_const(7);
		489	dram_efficiency.full = dfixed_div(dram_efficiency, a);
		490	bandwidth.full = dfixed_mul(dram_channels, yclk);
		491	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
		492
		493	return dfixed_trunc(bandwidth);
		494	}
		495
		496	static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		497	{
		498	/* Calculate DRAM Bandwidth and the part allocated to display. */
		499	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
		500	fixed20_12 yclk, dram_channels, bandwidth;
		501	fixed20_12 a;
		502
		503	a.full = dfixed_const(1000);
		504	yclk.full = dfixed_const(wm->yclk);
		505	yclk.full = dfixed_div(yclk, a);
		506	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		507	a.full = dfixed_const(10);
		508	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
		509	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
		510	bandwidth.full = dfixed_mul(dram_channels, yclk);
		511	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
		512
		513	return dfixed_trunc(bandwidth);
		514	}
		515
		516	static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
		517	{
		518	/* Calculate the display Data return Bandwidth */
		519	fixed20_12 return_efficiency; /* 0.8 */
		520	fixed20_12 sclk, bandwidth;
		521	fixed20_12 a;
		522
		523	a.full = dfixed_const(1000);
		524	sclk.full = dfixed_const(wm->sclk);
		525	sclk.full = dfixed_div(sclk, a);
		526	a.full = dfixed_const(10);
		527	return_efficiency.full = dfixed_const(8);
		528	return_efficiency.full = dfixed_div(return_efficiency, a);
		529	a.full = dfixed_const(32);
		530	bandwidth.full = dfixed_mul(a, sclk);
		531	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
		532
		533	return dfixed_trunc(bandwidth);
		534	}
		535
		536	static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
		537	{
		538	/* Calculate the DMIF Request Bandwidth */
		539	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
		540	fixed20_12 disp_clk, bandwidth;
		541	fixed20_12 a;
		542
		543	a.full = dfixed_const(1000);
		544	disp_clk.full = dfixed_const(wm->disp_clk);
		545	disp_clk.full = dfixed_div(disp_clk, a);
		546	a.full = dfixed_const(10);
		547	disp_clk_request_efficiency.full = dfixed_const(8);
		548	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
		549	a.full = dfixed_const(32);
		550	bandwidth.full = dfixed_mul(a, disp_clk);
		551	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
		552
		553	return dfixed_trunc(bandwidth);
		554	}
		555
		556	static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
		557	{
		558	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
		559	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
		560	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
		561	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
		562
		563	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
		564	}
		565
		566	static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
		567	{
		568	/* Calculate the display mode Average Bandwidth
		569	* DisplayMode should contain the source and destination dimensions,
		570	* timing, etc.
		571	*/
		572	fixed20_12 bpp;
		573	fixed20_12 line_time;
		574	fixed20_12 src_width;
		575	fixed20_12 bandwidth;
		576	fixed20_12 a;
		577
		578	a.full = dfixed_const(1000);
		579	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
		580	line_time.full = dfixed_div(line_time, a);
		581	bpp.full = dfixed_const(wm->bytes_per_pixel);
		582	src_width.full = dfixed_const(wm->src_width);
		583	bandwidth.full = dfixed_mul(src_width, bpp);
		584	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
		585	bandwidth.full = dfixed_div(bandwidth, line_time);
		586
		587	return dfixed_trunc(bandwidth);
		588	}
		589
		590	static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
		591	{
		592	/* First calcualte the latency in ns */
		593	u32 mc_latency = 2000; /* 2000 ns. */
		594	u32 available_bandwidth = evergreen_available_bandwidth(wm);
		595	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
		596	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
		597	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
		598	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
		599	(wm->num_heads * cursor_line_pair_return_time);
		600	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
		601	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
		602	fixed20_12 a, b, c;
		603
		604	if (wm->num_heads == 0)
		605	return 0;
		606
		607	a.full = dfixed_const(2);
		608	b.full = dfixed_const(1);
		609	if ((wm->vsc.full > a.full) \|\|
		610	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) \|\|
		611	(wm->vtaps >= 5) \|\|
		612	((wm->vsc.full >= a.full) && wm->interlaced))
		613	max_src_lines_per_dst_line = 4;
		614	else
		615	max_src_lines_per_dst_line = 2;
		616
		617	a.full = dfixed_const(available_bandwidth);
		618	b.full = dfixed_const(wm->num_heads);
		619	a.full = dfixed_div(a, b);
		620
		621	b.full = dfixed_const(1000);
		622	c.full = dfixed_const(wm->disp_clk);
		623	b.full = dfixed_div(c, b);
		624	c.full = dfixed_const(wm->bytes_per_pixel);
		625	b.full = dfixed_mul(b, c);
		626
		627	lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
		628
		629	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
		630	b.full = dfixed_const(1000);
		631	c.full = dfixed_const(lb_fill_bw);
		632	b.full = dfixed_div(c, b);
		633	a.full = dfixed_div(a, b);
		634	line_fill_time = dfixed_trunc(a);
		635
		636	if (line_fill_time < wm->active_time)
		637	return latency;
		638	else
		639	return latency + (line_fill_time - wm->active_time);
		640
		641	}
		642
		643	static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		644	{
		645	if (evergreen_average_bandwidth(wm) <=
		646	(evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
		647	return true;
		648	else
		649	return false;
		650	};
		651
		652	static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
		653	{
		654	if (evergreen_average_bandwidth(wm) <=
		655	(evergreen_available_bandwidth(wm) / wm->num_heads))
		656	return true;
		657	else
		658	return false;
		659	};
		660
		661	static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
		662	{
		663	u32 lb_partitions = wm->lb_size / wm->src_width;
		664	u32 line_time = wm->active_time + wm->blank_time;
		665	u32 latency_tolerant_lines;
		666	u32 latency_hiding;
		667	fixed20_12 a;
		668
		669	a.full = dfixed_const(1);
		670	if (wm->vsc.full > a.full)
		671	latency_tolerant_lines = 1;
		672	else {
		673	if (lb_partitions <= (wm->vtaps + 1))
		674	latency_tolerant_lines = 1;
		675	else
		676	latency_tolerant_lines = 2;
		677	}
		678
		679	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
		680
		681	if (evergreen_latency_watermark(wm) <= latency_hiding)
		682	return true;
		683	else
		684	return false;
		685	}
		686
		687	static void evergreen_program_watermarks(struct radeon_device *rdev,
		688	struct radeon_crtc *radeon_crtc,
		689	u32 lb_size, u32 num_heads)
		690	{
		691	struct drm_display_mode *mode = &radeon_crtc->base.mode;
		692	struct evergreen_wm_params wm;
		693	u32 pixel_period;
		694	u32 line_time = 0;
		695	u32 latency_watermark_a = 0, latency_watermark_b = 0;
		696	u32 priority_a_mark = 0, priority_b_mark = 0;
		697	u32 priority_a_cnt = PRIORITY_OFF;
		698	u32 priority_b_cnt = PRIORITY_OFF;
		699	u32 pipe_offset = radeon_crtc->crtc_id * 16;
		700	u32 tmp, arb_control3;
		701	fixed20_12 a, b, c;
		702
		703	if (radeon_crtc->base.enabled && num_heads && mode) {
		704	pixel_period = 1000000 / (u32)mode->clock;
		705	line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
		706	priority_a_cnt = 0;
		707	priority_b_cnt = 0;
		708
		709	wm.yclk = rdev->pm.current_mclk * 10;
		710	wm.sclk = rdev->pm.current_sclk * 10;
		711	wm.disp_clk = mode->clock;
		712	wm.src_width = mode->crtc_hdisplay;
		713	wm.active_time = mode->crtc_hdisplay * pixel_period;
		714	wm.blank_time = line_time - wm.active_time;
		715	wm.interlaced = false;
		716	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		717	wm.interlaced = true;
		718	wm.vsc = radeon_crtc->vsc;
		719	wm.vtaps = 1;
		720	if (radeon_crtc->rmx_type != RMX_OFF)
		721	wm.vtaps = 2;
		722	wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
		723	wm.lb_size = lb_size;
		724	wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
		725	wm.num_heads = num_heads;
		726
		727	/* set for high clocks */
		728	latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
		729	/* set for low clocks */
		730	/* wm.yclk = low clk; wm.sclk = low clk */
		731	latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
		732
		733	/* possibly force display priority to high */
		734	/* should really do this at mode validation time... */
		735	if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) \|\|
		736	!evergreen_average_bandwidth_vs_available_bandwidth(&wm) \|\|
		737	!evergreen_check_latency_hiding(&wm) \|\|
		738	(rdev->disp_priority == 2)) {
		739	DRM_INFO("force priority to high\n");
		740	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		741	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		742	}
		743
		744	a.full = dfixed_const(1000);
		745	b.full = dfixed_const(mode->clock);
		746	b.full = dfixed_div(b, a);
		747	c.full = dfixed_const(latency_watermark_a);
		748	c.full = dfixed_mul(c, b);
		749	c.full = dfixed_mul(c, radeon_crtc->hsc);
		750	c.full = dfixed_div(c, a);
		751	a.full = dfixed_const(16);
		752	c.full = dfixed_div(c, a);
		753	priority_a_mark = dfixed_trunc(c);
		754	priority_a_cnt \|= priority_a_mark & PRIORITY_MARK_MASK;
		755
		756	a.full = dfixed_const(1000);
		757	b.full = dfixed_const(mode->clock);
		758	b.full = dfixed_div(b, a);
		759	c.full = dfixed_const(latency_watermark_b);
		760	c.full = dfixed_mul(c, b);
		761	c.full = dfixed_mul(c, radeon_crtc->hsc);
		762	c.full = dfixed_div(c, a);
		763	a.full = dfixed_const(16);
		764	c.full = dfixed_div(c, a);
		765	priority_b_mark = dfixed_trunc(c);
		766	priority_b_cnt \|= priority_b_mark & PRIORITY_MARK_MASK;
		767	}
		768
		769	/* select wm A */
		770	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		771	tmp = arb_control3;
		772	tmp &= ~LATENCY_WATERMARK_MASK(3);
		773	tmp \|= LATENCY_WATERMARK_MASK(1);
		774	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		775	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		776	(LATENCY_LOW_WATERMARK(latency_watermark_a) \|
		777	LATENCY_HIGH_WATERMARK(line_time)));
		778	/* select wm B */
		779	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		780	tmp &= ~LATENCY_WATERMARK_MASK(3);
		781	tmp \|= LATENCY_WATERMARK_MASK(2);
		782	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		783	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		784	(LATENCY_LOW_WATERMARK(latency_watermark_b) \|
		785	LATENCY_HIGH_WATERMARK(line_time)));
		786	/* restore original selection */
		787	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
		788
		789	/* write the priority marks */
		790	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
		791	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
		792
		793	}
		794
1963	serge	795	void evergreen_bandwidth_update(struct radeon_device *rdev)
1430	serge	796	{
1986	serge	797	struct drm_display_mode *mode0 = NULL;
		798	struct drm_display_mode *mode1 = NULL;
		799	u32 num_heads = 0, lb_size;
		800	int i;
		801
		802	radeon_update_display_priority(rdev);
		803
		804	for (i = 0; i < rdev->num_crtc; i++) {
		805	if (rdev->mode_info.crtcs[i]->base.enabled)
		806	num_heads++;
		807	}
		808	for (i = 0; i < rdev->num_crtc; i += 2) {
		809	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
		810	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
		811	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
		812	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
		813	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
		814	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
		815	}
1430	serge	816	}
		817
1963	serge	818	int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1430	serge	819	{
		820	unsigned i;
		821	u32 tmp;
		822
		823	for (i = 0; i < rdev->usec_timeout; i++) {
		824	/* read MC_STATUS */
		825	tmp = RREG32(SRBM_STATUS) & 0x1F00;
		826	if (!tmp)
		827	return 0;
		828	udelay(1);
		829	}
		830	return -1;
		831	}
		832
		833	/*
		834	* GART
		835	*/
1963	serge	836	void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
		837	{
		838	unsigned i;
		839	u32 tmp;
		840
		841	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		842
		843	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		844	for (i = 0; i < rdev->usec_timeout; i++) {
		845	/* read MC_STATUS */
		846	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		847	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		848	if (tmp == 2) {
		849	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		850	return;
		851	}
		852	if (tmp) {
		853	return;
		854	}
		855	udelay(1);
		856	}
		857	}
		858
1430	serge	859	int evergreen_pcie_gart_enable(struct radeon_device *rdev)
		860	{
		861	u32 tmp;
1963	serge	862	int r;
1430	serge	863
		864	if (rdev->gart.table.vram.robj == NULL) {
		865	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		866	return -EINVAL;
		867	}
		868	r = radeon_gart_table_vram_pin(rdev);
		869	if (r)
		870	return r;
		871	radeon_gart_restore(rdev);
		872	/* Setup L2 cache */
		873	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		874	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		875	EFFECTIVE_L2_QUEUE_SIZE(7));
		876	WREG32(VM_L2_CNTL2, 0);
		877	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		878	/* Setup TLB control */
		879	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		880	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		881	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		882	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
1963	serge	883	if (rdev->flags & RADEON_IS_IGP) {
		884	WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
		885	WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
		886	WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
		887	} else {
1430	serge	888	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		889	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		890	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1963	serge	891	}
1430	serge	892	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		893	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		894	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		895	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
		896	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		897	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		898	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		899	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		900	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		901	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		902	(u32)(rdev->dummy_page.addr >> 12));
1963	serge	903	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	904
1963	serge	905	evergreen_pcie_gart_tlb_flush(rdev);
1430	serge	906	rdev->gart.ready = true;
		907	return 0;
		908	}
		909
		910	void evergreen_pcie_gart_disable(struct radeon_device *rdev)
		911	{
		912	u32 tmp;
1963	serge	913	int r;
1430	serge	914
		915	/* Disable all tables */
1963	serge	916	WREG32(VM_CONTEXT0_CNTL, 0);
		917	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	918
		919	/* Setup L2 cache */
		920	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		921	EFFECTIVE_L2_QUEUE_SIZE(7));
		922	WREG32(VM_L2_CNTL2, 0);
		923	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		924	/* Setup TLB control */
		925	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		926	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		927	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		928	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		929	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		930	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		931	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		932	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
		933	if (rdev->gart.table.vram.robj) {
		934	r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
		935	if (likely(r == 0)) {
		936	radeon_bo_kunmap(rdev->gart.table.vram.robj);
		937	radeon_bo_unpin(rdev->gart.table.vram.robj);
		938	radeon_bo_unreserve(rdev->gart.table.vram.robj);
		939	}
		940	}
		941	}
		942
		943	void evergreen_pcie_gart_fini(struct radeon_device *rdev)
		944	{
		945	evergreen_pcie_gart_disable(rdev);
		946	radeon_gart_table_vram_free(rdev);
		947	radeon_gart_fini(rdev);
		948	}
		949
		950
		951	void evergreen_agp_enable(struct radeon_device *rdev)
		952	{
		953	u32 tmp;
		954
		955	/* Setup L2 cache */
		956	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		957	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		958	EFFECTIVE_L2_QUEUE_SIZE(7));
		959	WREG32(VM_L2_CNTL2, 0);
		960	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		961	/* Setup TLB control */
		962	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		963	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		964	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		965	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		966	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		967	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		968	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		969	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		970	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		971	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		972	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1963	serge	973	WREG32(VM_CONTEXT0_CNTL, 0);
		974	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	975	}
		976
1963	serge	977	void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	978	{
		979	save->vga_control[0] = RREG32(D1VGA_CONTROL);
		980	save->vga_control[1] = RREG32(D2VGA_CONTROL);
		981	save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL);
		982	save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL);
		983	save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL);
		984	save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL);
		985	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
		986	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
		987	save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
		988	save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
1963	serge	989	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	990	save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
		991	save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
		992	save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
		993	save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
1963	serge	994	}
1430	serge	995
		996	/* Stop all video */
		997	WREG32(VGA_RENDER_CONTROL, 0);
		998	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
		999	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1963	serge	1000	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1001	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
		1002	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
		1003	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
		1004	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1963	serge	1005	}
1430	serge	1006	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		1007	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1963	serge	1008	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1009	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		1010	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		1011	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		1012	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	1013	}
1430	serge	1014	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		1015	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1963	serge	1016	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1017	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		1018	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		1019	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		1020	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	1021	}
1430	serge	1022
		1023	WREG32(D1VGA_CONTROL, 0);
		1024	WREG32(D2VGA_CONTROL, 0);
		1025	WREG32(EVERGREEN_D3VGA_CONTROL, 0);
		1026	WREG32(EVERGREEN_D4VGA_CONTROL, 0);
		1027	WREG32(EVERGREEN_D5VGA_CONTROL, 0);
		1028	WREG32(EVERGREEN_D6VGA_CONTROL, 0);
		1029	}
		1030
1963	serge	1031	void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	1032	{
		1033	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
		1034	upper_32_bits(rdev->mc.vram_start));
		1035	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
		1036	upper_32_bits(rdev->mc.vram_start));
		1037	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
		1038	(u32)rdev->mc.vram_start);
		1039	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
		1040	(u32)rdev->mc.vram_start);
		1041
		1042	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
		1043	upper_32_bits(rdev->mc.vram_start));
		1044	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
		1045	upper_32_bits(rdev->mc.vram_start));
		1046	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
		1047	(u32)rdev->mc.vram_start);
		1048	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
		1049	(u32)rdev->mc.vram_start);
		1050
1963	serge	1051	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1052	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
		1053	upper_32_bits(rdev->mc.vram_start));
		1054	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
		1055	upper_32_bits(rdev->mc.vram_start));
		1056	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
		1057	(u32)rdev->mc.vram_start);
		1058	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
		1059	(u32)rdev->mc.vram_start);
		1060
		1061	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
		1062	upper_32_bits(rdev->mc.vram_start));
		1063	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
		1064	upper_32_bits(rdev->mc.vram_start));
		1065	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
		1066	(u32)rdev->mc.vram_start);
		1067	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
		1068	(u32)rdev->mc.vram_start);
		1069
		1070	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
		1071	upper_32_bits(rdev->mc.vram_start));
		1072	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
		1073	upper_32_bits(rdev->mc.vram_start));
		1074	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
		1075	(u32)rdev->mc.vram_start);
		1076	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
		1077	(u32)rdev->mc.vram_start);
		1078
		1079	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
		1080	upper_32_bits(rdev->mc.vram_start));
		1081	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
		1082	upper_32_bits(rdev->mc.vram_start));
		1083	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
		1084	(u32)rdev->mc.vram_start);
		1085	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
		1086	(u32)rdev->mc.vram_start);
1963	serge	1087	}
1430	serge	1088
		1089	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
		1090	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
		1091	/* Unlock host access */
		1092	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
		1093	mdelay(1);
		1094	/* Restore video state */
		1095	WREG32(D1VGA_CONTROL, save->vga_control[0]);
		1096	WREG32(D2VGA_CONTROL, save->vga_control[1]);
		1097	WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]);
		1098	WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]);
		1099	WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]);
		1100	WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
		1101	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
		1102	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1963	serge	1103	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1104	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
		1105	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
		1106	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
		1107	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1963	serge	1108	}
1430	serge	1109	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]);
		1110	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]);
1963	serge	1111	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1112	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
		1113	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
		1114	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
		1115	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
1963	serge	1116	}
1430	serge	1117	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		1118	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1963	serge	1119	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1120	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		1121	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		1122	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		1123	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	1124	}
1430	serge	1125	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
		1126	}
		1127
1963	serge	1128	void evergreen_mc_program(struct radeon_device *rdev)
1430	serge	1129	{
		1130	struct evergreen_mc_save save;
		1131	u32 tmp;
		1132	int i, j;
		1133
		1134	/* Initialize HDP */
		1135	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1136	WREG32((0x2c14 + j), 0x00000000);
		1137	WREG32((0x2c18 + j), 0x00000000);
		1138	WREG32((0x2c1c + j), 0x00000000);
		1139	WREG32((0x2c20 + j), 0x00000000);
		1140	WREG32((0x2c24 + j), 0x00000000);
		1141	}
		1142	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		1143
		1144	evergreen_mc_stop(rdev, &save);
		1145	if (evergreen_mc_wait_for_idle(rdev)) {
		1146	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1147	}
		1148	/* Lockout access through VGA aperture*/
		1149	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		1150	/* Update configuration */
		1151	if (rdev->flags & RADEON_IS_AGP) {
		1152	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		1153	/* VRAM before AGP */
		1154	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1155	rdev->mc.vram_start >> 12);
		1156	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1157	rdev->mc.gtt_end >> 12);
		1158	} else {
		1159	/* VRAM after AGP */
		1160	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1161	rdev->mc.gtt_start >> 12);
		1162	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1163	rdev->mc.vram_end >> 12);
		1164	}
		1165	} else {
		1166	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1167	rdev->mc.vram_start >> 12);
		1168	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1169	rdev->mc.vram_end >> 12);
		1170	}
		1171	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
1963	serge	1172	if (rdev->flags & RADEON_IS_IGP) {
		1173	tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
		1174	tmp \|= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
		1175	tmp \|= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
		1176	WREG32(MC_FUS_VM_FB_OFFSET, tmp);
		1177	}
1430	serge	1178	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		1179	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		1180	WREG32(MC_VM_FB_LOCATION, tmp);
		1181	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1963	serge	1182	WREG32(HDP_NONSURFACE_INFO, (2 << 7) \| (1 << 30));
		1183	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1430	serge	1184	if (rdev->flags & RADEON_IS_AGP) {
		1185	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
		1186	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
		1187	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		1188	} else {
		1189	WREG32(MC_VM_AGP_BASE, 0);
		1190	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		1191	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		1192	}
		1193	if (evergreen_mc_wait_for_idle(rdev)) {
		1194	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1195	}
		1196	evergreen_mc_resume(rdev, &save);
		1197	/* we need to own VRAM, so turn off the VGA renderer here
		1198	* to stop it overwriting our objects */
		1199	rv515_vga_render_disable(rdev);
		1200	}
		1201
		1202	/*
		1203	* CP.
		1204	*/
1986	serge	1205	void evergreen_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		1206	{
		1207	/* set to DX10/11 mode */
		1208	radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
		1209	radeon_ring_write(rdev, 1);
		1210	/* FIXME: implement */
		1211	radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		1212	radeon_ring_write(rdev,
		1213	#ifdef __BIG_ENDIAN
		1214	(2 << 0) \|
		1215	#endif
		1216	(ib->gpu_addr & 0xFFFFFFFC));
		1217	radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
		1218	radeon_ring_write(rdev, ib->length_dw);
		1219	}
1963	serge	1220
1986	serge	1221
1963	serge	1222	static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1430	serge	1223	{
1963	serge	1224	const __be32 *fw_data;
		1225	int i;
		1226
		1227	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1228	return -EINVAL;
		1229
		1230	r700_cp_stop(rdev);
		1231	WREG32(CP_RB_CNTL,
		1232	#ifdef __BIG_ENDIAN
		1233	BUF_SWAP_32BIT \|
		1234	#endif
		1235	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
		1236
		1237	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1238	WREG32(CP_PFP_UCODE_ADDR, 0);
		1239	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
		1240	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		1241	WREG32(CP_PFP_UCODE_ADDR, 0);
		1242
		1243	fw_data = (const __be32 *)rdev->me_fw->data;
		1244	WREG32(CP_ME_RAM_WADDR, 0);
		1245	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
		1246	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		1247
		1248	WREG32(CP_PFP_UCODE_ADDR, 0);
		1249	WREG32(CP_ME_RAM_WADDR, 0);
		1250	WREG32(CP_ME_RAM_RADDR, 0);
		1251	return 0;
1430	serge	1252	}
		1253
1963	serge	1254	static int evergreen_cp_start(struct radeon_device *rdev)
		1255	{
		1256	int r, i;
		1257	uint32_t cp_me;
1430	serge	1258
1963	serge	1259	r = radeon_ring_lock(rdev, 7);
		1260	if (r) {
		1261	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1262	return r;
		1263	}
		1264	radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1265	radeon_ring_write(rdev, 0x1);
		1266	radeon_ring_write(rdev, 0x0);
		1267	radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
		1268	radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1269	radeon_ring_write(rdev, 0);
		1270	radeon_ring_write(rdev, 0);
		1271	radeon_ring_unlock_commit(rdev);
		1272
		1273	cp_me = 0xff;
		1274	WREG32(CP_ME_CNTL, cp_me);
		1275
		1276	r = radeon_ring_lock(rdev, evergreen_default_size + 19);
		1277	if (r) {
		1278	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1279	return r;
		1280	}
		1281
		1282	/* setup clear context state */
		1283	radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1284	radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
		1285
		1286	for (i = 0; i < evergreen_default_size; i++)
		1287	radeon_ring_write(rdev, evergreen_default_state[i]);
		1288
		1289	radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1290	radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
		1291
		1292	/* set clear context state */
		1293	radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
		1294	radeon_ring_write(rdev, 0);
		1295
		1296	/* SQ_VTX_BASE_VTX_LOC */
		1297	radeon_ring_write(rdev, 0xc0026f00);
		1298	radeon_ring_write(rdev, 0x00000000);
		1299	radeon_ring_write(rdev, 0x00000000);
		1300	radeon_ring_write(rdev, 0x00000000);
		1301
		1302	/* Clear consts */
		1303	radeon_ring_write(rdev, 0xc0036f00);
		1304	radeon_ring_write(rdev, 0x00000bc4);
		1305	radeon_ring_write(rdev, 0xffffffff);
		1306	radeon_ring_write(rdev, 0xffffffff);
		1307	radeon_ring_write(rdev, 0xffffffff);
		1308
		1309	radeon_ring_write(rdev, 0xc0026900);
		1310	radeon_ring_write(rdev, 0x00000316);
		1311	radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		1312	radeon_ring_write(rdev, 0x00000010); /* */
		1313
		1314	radeon_ring_unlock_commit(rdev);
		1315
		1316	return 0;
		1317	}
		1318
		1319	int evergreen_cp_resume(struct radeon_device *rdev)
1430	serge	1320	{
1963	serge	1321	u32 tmp;
		1322	u32 rb_bufsz;
		1323	int r;
1430	serge	1324
1963	serge	1325	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
		1326	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP \|
		1327	SOFT_RESET_PA \|
		1328	SOFT_RESET_SH \|
		1329	SOFT_RESET_VGT \|
		1330	SOFT_RESET_SX));
		1331	RREG32(GRBM_SOFT_RESET);
		1332	mdelay(15);
		1333	WREG32(GRBM_SOFT_RESET, 0);
		1334	RREG32(GRBM_SOFT_RESET);
		1335
		1336	/* Set ring buffer size */
		1337	rb_bufsz = drm_order(rdev->cp.ring_size / 8);
		1338	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
		1339	#ifdef __BIG_ENDIAN
		1340	tmp \|= BUF_SWAP_32BIT;
		1341	#endif
		1342	WREG32(CP_RB_CNTL, tmp);
		1343	WREG32(CP_SEM_WAIT_TIMER, 0x4);
		1344
		1345	/* Set the write pointer delay */
		1346	WREG32(CP_RB_WPTR_DELAY, 0);
		1347
		1348	/* Initialize the ring buffer's read and write pointers */
		1349	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1350	WREG32(CP_RB_RPTR_WR, 0);
		1351	WREG32(CP_RB_WPTR, 0);
		1352
		1353	/* set the wb address wether it's enabled or not */
		1354	WREG32(CP_RB_RPTR_ADDR,
		1355	#ifdef __BIG_ENDIAN
		1356	RB_RPTR_SWAP(2) \|
		1357	#endif
		1358	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1359	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1360	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1361
		1362	if (rdev->wb.enabled)
		1363	WREG32(SCRATCH_UMSK, 0xff);
		1364	else {
		1365	tmp \|= RB_NO_UPDATE;
		1366	WREG32(SCRATCH_UMSK, 0);
		1367	}
		1368
		1369	mdelay(1);
		1370	WREG32(CP_RB_CNTL, tmp);
		1371
		1372	WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
		1373	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1374
		1375	rdev->cp.rptr = RREG32(CP_RB_RPTR);
		1376	rdev->cp.wptr = RREG32(CP_RB_WPTR);
		1377
		1378	evergreen_cp_start(rdev);
		1379	rdev->cp.ready = true;
		1380	r = radeon_ring_test(rdev);
		1381	if (r) {
		1382	rdev->cp.ready = false;
		1383	return r;
		1384	}
1430	serge	1385	return 0;
		1386	}
		1387
		1388	/*
		1389	* Core functions
		1390	*/
1963	serge	1391	static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
		1392	u32 num_tile_pipes,
1430	serge	1393	u32 num_backends,
		1394	u32 backend_disable_mask)
		1395	{
		1396	u32 backend_map = 0;
1963	serge	1397	u32 enabled_backends_mask = 0;
		1398	u32 enabled_backends_count = 0;
		1399	u32 cur_pipe;
		1400	u32 swizzle_pipe[EVERGREEN_MAX_PIPES];
		1401	u32 cur_backend = 0;
		1402	u32 i;
		1403	bool force_no_swizzle;
1430	serge	1404
1963	serge	1405	if (num_tile_pipes > EVERGREEN_MAX_PIPES)
		1406	num_tile_pipes = EVERGREEN_MAX_PIPES;
		1407	if (num_tile_pipes < 1)
		1408	num_tile_pipes = 1;
		1409	if (num_backends > EVERGREEN_MAX_BACKENDS)
		1410	num_backends = EVERGREEN_MAX_BACKENDS;
		1411	if (num_backends < 1)
		1412	num_backends = 1;
		1413
		1414	for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
		1415	if (((backend_disable_mask >> i) & 1) == 0) {
		1416	enabled_backends_mask \|= (1 << i);
		1417	++enabled_backends_count;
		1418	}
		1419	if (enabled_backends_count == num_backends)
		1420	break;
		1421	}
		1422
		1423	if (enabled_backends_count == 0) {
		1424	enabled_backends_mask = 1;
		1425	enabled_backends_count = 1;
		1426	}
		1427
		1428	if (enabled_backends_count != num_backends)
		1429	num_backends = enabled_backends_count;
		1430
		1431	memset((uint8_t )&swizzle_pipe[0], 0, sizeof(u32) EVERGREEN_MAX_PIPES);
		1432	switch (rdev->family) {
		1433	case CHIP_CEDAR:
		1434	case CHIP_REDWOOD:
		1435	case CHIP_PALM:
		1436	case CHIP_SUMO:
		1437	case CHIP_SUMO2:
		1438	case CHIP_TURKS:
		1439	case CHIP_CAICOS:
		1440	force_no_swizzle = false;
		1441	break;
		1442	case CHIP_CYPRESS:
		1443	case CHIP_HEMLOCK:
		1444	case CHIP_JUNIPER:
		1445	case CHIP_BARTS:
		1446	default:
		1447	force_no_swizzle = true;
		1448	break;
		1449	}
		1450	if (force_no_swizzle) {
		1451	bool last_backend_enabled = false;
		1452
		1453	force_no_swizzle = false;
		1454	for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
		1455	if (((enabled_backends_mask >> i) & 1) == 1) {
		1456	if (last_backend_enabled)
		1457	force_no_swizzle = true;
		1458	last_backend_enabled = true;
		1459	} else
		1460	last_backend_enabled = false;
		1461	}
		1462	}
		1463
		1464	switch (num_tile_pipes) {
		1465	case 1:
		1466	case 3:
		1467	case 5:
		1468	case 7:
		1469	DRM_ERROR("odd number of pipes!\n");
		1470	break;
		1471	case 2:
		1472	swizzle_pipe[0] = 0;
		1473	swizzle_pipe[1] = 1;
		1474	break;
		1475	case 4:
		1476	if (force_no_swizzle) {
		1477	swizzle_pipe[0] = 0;
		1478	swizzle_pipe[1] = 1;
		1479	swizzle_pipe[2] = 2;
		1480	swizzle_pipe[3] = 3;
		1481	} else {
		1482	swizzle_pipe[0] = 0;
		1483	swizzle_pipe[1] = 2;
		1484	swizzle_pipe[2] = 1;
		1485	swizzle_pipe[3] = 3;
		1486	}
		1487	break;
		1488	case 6:
		1489	if (force_no_swizzle) {
		1490	swizzle_pipe[0] = 0;
		1491	swizzle_pipe[1] = 1;
		1492	swizzle_pipe[2] = 2;
		1493	swizzle_pipe[3] = 3;
		1494	swizzle_pipe[4] = 4;
		1495	swizzle_pipe[5] = 5;
		1496	} else {
		1497	swizzle_pipe[0] = 0;
		1498	swizzle_pipe[1] = 2;
		1499	swizzle_pipe[2] = 4;
		1500	swizzle_pipe[3] = 1;
		1501	swizzle_pipe[4] = 3;
		1502	swizzle_pipe[5] = 5;
		1503	}
		1504	break;
		1505	case 8:
		1506	if (force_no_swizzle) {
		1507	swizzle_pipe[0] = 0;
		1508	swizzle_pipe[1] = 1;
		1509	swizzle_pipe[2] = 2;
		1510	swizzle_pipe[3] = 3;
		1511	swizzle_pipe[4] = 4;
		1512	swizzle_pipe[5] = 5;
		1513	swizzle_pipe[6] = 6;
		1514	swizzle_pipe[7] = 7;
		1515	} else {
		1516	swizzle_pipe[0] = 0;
		1517	swizzle_pipe[1] = 2;
		1518	swizzle_pipe[2] = 4;
		1519	swizzle_pipe[3] = 6;
		1520	swizzle_pipe[4] = 1;
		1521	swizzle_pipe[5] = 3;
		1522	swizzle_pipe[6] = 5;
		1523	swizzle_pipe[7] = 7;
		1524	}
		1525	break;
		1526	}
		1527
		1528	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
		1529	while (((1 << cur_backend) & enabled_backends_mask) == 0)
		1530	cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
		1531
		1532	backend_map \|= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4)));
		1533
		1534	cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
		1535	}
		1536
1430	serge	1537	return backend_map;
		1538	}
		1539
1986	serge	1540	static void evergreen_program_channel_remap(struct radeon_device *rdev)
		1541	{
		1542	u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp;
		1543
		1544	tmp = RREG32(MC_SHARED_CHMAP);
		1545	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		1546	case 0:
		1547	case 1:
		1548	case 2:
		1549	case 3:
		1550	default:
		1551	/* default mapping */
		1552	mc_shared_chremap = 0x00fac688;
		1553	break;
		1554	}
		1555
		1556	switch (rdev->family) {
		1557	case CHIP_HEMLOCK:
		1558	case CHIP_CYPRESS:
		1559	case CHIP_BARTS:
		1560	tcp_chan_steer_lo = 0x54763210;
		1561	tcp_chan_steer_hi = 0x0000ba98;
		1562	break;
		1563	case CHIP_JUNIPER:
		1564	case CHIP_REDWOOD:
		1565	case CHIP_CEDAR:
		1566	case CHIP_PALM:
		1567	case CHIP_SUMO:
		1568	case CHIP_SUMO2:
		1569	case CHIP_TURKS:
		1570	case CHIP_CAICOS:
		1571	default:
		1572	tcp_chan_steer_lo = 0x76543210;
		1573	tcp_chan_steer_hi = 0x0000ba98;
		1574	break;
		1575	}
		1576
		1577	WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo);
		1578	WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi);
		1579	WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
		1580	}
		1581
1430	serge	1582	static void evergreen_gpu_init(struct radeon_device *rdev)
		1583	{
1963	serge	1584	u32 cc_rb_backend_disable = 0;
		1585	u32 cc_gc_shader_pipe_config;
		1586	u32 gb_addr_config = 0;
		1587	u32 mc_shared_chmap, mc_arb_ramcfg;
		1588	u32 gb_backend_map;
		1589	u32 grbm_gfx_index;
		1590	u32 sx_debug_1;
		1591	u32 smx_dc_ctl0;
		1592	u32 sq_config;
		1593	u32 sq_lds_resource_mgmt;
		1594	u32 sq_gpr_resource_mgmt_1;
		1595	u32 sq_gpr_resource_mgmt_2;
		1596	u32 sq_gpr_resource_mgmt_3;
		1597	u32 sq_thread_resource_mgmt;
		1598	u32 sq_thread_resource_mgmt_2;
		1599	u32 sq_stack_resource_mgmt_1;
		1600	u32 sq_stack_resource_mgmt_2;
		1601	u32 sq_stack_resource_mgmt_3;
		1602	u32 vgt_cache_invalidation;
		1603	u32 hdp_host_path_cntl, tmp;
		1604	int i, j, num_shader_engines, ps_thread_count;
		1605
		1606	switch (rdev->family) {
		1607	case CHIP_CYPRESS:
		1608	case CHIP_HEMLOCK:
		1609	rdev->config.evergreen.num_ses = 2;
		1610	rdev->config.evergreen.max_pipes = 4;
		1611	rdev->config.evergreen.max_tile_pipes = 8;
		1612	rdev->config.evergreen.max_simds = 10;
		1613	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1614	rdev->config.evergreen.max_gprs = 256;
		1615	rdev->config.evergreen.max_threads = 248;
		1616	rdev->config.evergreen.max_gs_threads = 32;
		1617	rdev->config.evergreen.max_stack_entries = 512;
		1618	rdev->config.evergreen.sx_num_of_sets = 4;
		1619	rdev->config.evergreen.sx_max_export_size = 256;
		1620	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1621	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1622	rdev->config.evergreen.max_hw_contexts = 8;
		1623	rdev->config.evergreen.sq_num_cf_insts = 2;
		1624
		1625	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1626	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1627	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1628	break;
		1629	case CHIP_JUNIPER:
		1630	rdev->config.evergreen.num_ses = 1;
		1631	rdev->config.evergreen.max_pipes = 4;
		1632	rdev->config.evergreen.max_tile_pipes = 4;
		1633	rdev->config.evergreen.max_simds = 10;
		1634	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1635	rdev->config.evergreen.max_gprs = 256;
		1636	rdev->config.evergreen.max_threads = 248;
		1637	rdev->config.evergreen.max_gs_threads = 32;
		1638	rdev->config.evergreen.max_stack_entries = 512;
		1639	rdev->config.evergreen.sx_num_of_sets = 4;
		1640	rdev->config.evergreen.sx_max_export_size = 256;
		1641	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1642	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1643	rdev->config.evergreen.max_hw_contexts = 8;
		1644	rdev->config.evergreen.sq_num_cf_insts = 2;
		1645
		1646	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1647	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1648	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1649	break;
		1650	case CHIP_REDWOOD:
		1651	rdev->config.evergreen.num_ses = 1;
		1652	rdev->config.evergreen.max_pipes = 4;
		1653	rdev->config.evergreen.max_tile_pipes = 4;
		1654	rdev->config.evergreen.max_simds = 5;
		1655	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1656	rdev->config.evergreen.max_gprs = 256;
		1657	rdev->config.evergreen.max_threads = 248;
		1658	rdev->config.evergreen.max_gs_threads = 32;
		1659	rdev->config.evergreen.max_stack_entries = 256;
		1660	rdev->config.evergreen.sx_num_of_sets = 4;
		1661	rdev->config.evergreen.sx_max_export_size = 256;
		1662	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1663	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1664	rdev->config.evergreen.max_hw_contexts = 8;
		1665	rdev->config.evergreen.sq_num_cf_insts = 2;
		1666
		1667	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1668	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1669	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1670	break;
		1671	case CHIP_CEDAR:
		1672	default:
		1673	rdev->config.evergreen.num_ses = 1;
		1674	rdev->config.evergreen.max_pipes = 2;
		1675	rdev->config.evergreen.max_tile_pipes = 2;
		1676	rdev->config.evergreen.max_simds = 2;
		1677	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1678	rdev->config.evergreen.max_gprs = 256;
		1679	rdev->config.evergreen.max_threads = 192;
		1680	rdev->config.evergreen.max_gs_threads = 16;
		1681	rdev->config.evergreen.max_stack_entries = 256;
		1682	rdev->config.evergreen.sx_num_of_sets = 4;
		1683	rdev->config.evergreen.sx_max_export_size = 128;
		1684	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1685	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1686	rdev->config.evergreen.max_hw_contexts = 4;
		1687	rdev->config.evergreen.sq_num_cf_insts = 1;
		1688
		1689	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1690	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1691	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1692	break;
		1693	case CHIP_PALM:
		1694	rdev->config.evergreen.num_ses = 1;
		1695	rdev->config.evergreen.max_pipes = 2;
		1696	rdev->config.evergreen.max_tile_pipes = 2;
		1697	rdev->config.evergreen.max_simds = 2;
		1698	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1699	rdev->config.evergreen.max_gprs = 256;
		1700	rdev->config.evergreen.max_threads = 192;
		1701	rdev->config.evergreen.max_gs_threads = 16;
		1702	rdev->config.evergreen.max_stack_entries = 256;
		1703	rdev->config.evergreen.sx_num_of_sets = 4;
		1704	rdev->config.evergreen.sx_max_export_size = 128;
		1705	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1706	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1707	rdev->config.evergreen.max_hw_contexts = 4;
		1708	rdev->config.evergreen.sq_num_cf_insts = 1;
		1709
		1710	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1711	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1712	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1713	break;
		1714	case CHIP_SUMO:
		1715	rdev->config.evergreen.num_ses = 1;
		1716	rdev->config.evergreen.max_pipes = 4;
		1717	rdev->config.evergreen.max_tile_pipes = 2;
		1718	if (rdev->pdev->device == 0x9648)
		1719	rdev->config.evergreen.max_simds = 3;
		1720	else if ((rdev->pdev->device == 0x9647) \|\|
		1721	(rdev->pdev->device == 0x964a))
		1722	rdev->config.evergreen.max_simds = 4;
		1723	else
		1724	rdev->config.evergreen.max_simds = 5;
		1725	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1726	rdev->config.evergreen.max_gprs = 256;
		1727	rdev->config.evergreen.max_threads = 248;
		1728	rdev->config.evergreen.max_gs_threads = 32;
		1729	rdev->config.evergreen.max_stack_entries = 256;
		1730	rdev->config.evergreen.sx_num_of_sets = 4;
		1731	rdev->config.evergreen.sx_max_export_size = 256;
		1732	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1733	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1734	rdev->config.evergreen.max_hw_contexts = 8;
		1735	rdev->config.evergreen.sq_num_cf_insts = 2;
		1736
		1737	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1738	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1739	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1740	break;
		1741	case CHIP_SUMO2:
		1742	rdev->config.evergreen.num_ses = 1;
		1743	rdev->config.evergreen.max_pipes = 4;
		1744	rdev->config.evergreen.max_tile_pipes = 4;
		1745	rdev->config.evergreen.max_simds = 2;
		1746	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1747	rdev->config.evergreen.max_gprs = 256;
		1748	rdev->config.evergreen.max_threads = 248;
		1749	rdev->config.evergreen.max_gs_threads = 32;
		1750	rdev->config.evergreen.max_stack_entries = 512;
		1751	rdev->config.evergreen.sx_num_of_sets = 4;
		1752	rdev->config.evergreen.sx_max_export_size = 256;
		1753	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1754	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1755	rdev->config.evergreen.max_hw_contexts = 8;
		1756	rdev->config.evergreen.sq_num_cf_insts = 2;
		1757
		1758	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1759	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1760	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1761	break;
		1762	case CHIP_BARTS:
		1763	rdev->config.evergreen.num_ses = 2;
		1764	rdev->config.evergreen.max_pipes = 4;
		1765	rdev->config.evergreen.max_tile_pipes = 8;
		1766	rdev->config.evergreen.max_simds = 7;
		1767	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1768	rdev->config.evergreen.max_gprs = 256;
		1769	rdev->config.evergreen.max_threads = 248;
		1770	rdev->config.evergreen.max_gs_threads = 32;
		1771	rdev->config.evergreen.max_stack_entries = 512;
		1772	rdev->config.evergreen.sx_num_of_sets = 4;
		1773	rdev->config.evergreen.sx_max_export_size = 256;
		1774	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1775	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1776	rdev->config.evergreen.max_hw_contexts = 8;
		1777	rdev->config.evergreen.sq_num_cf_insts = 2;
		1778
		1779	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1780	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1781	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1782	break;
		1783	case CHIP_TURKS:
		1784	rdev->config.evergreen.num_ses = 1;
		1785	rdev->config.evergreen.max_pipes = 4;
		1786	rdev->config.evergreen.max_tile_pipes = 4;
		1787	rdev->config.evergreen.max_simds = 6;
		1788	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1789	rdev->config.evergreen.max_gprs = 256;
		1790	rdev->config.evergreen.max_threads = 248;
		1791	rdev->config.evergreen.max_gs_threads = 32;
		1792	rdev->config.evergreen.max_stack_entries = 256;
		1793	rdev->config.evergreen.sx_num_of_sets = 4;
		1794	rdev->config.evergreen.sx_max_export_size = 256;
		1795	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1796	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1797	rdev->config.evergreen.max_hw_contexts = 8;
		1798	rdev->config.evergreen.sq_num_cf_insts = 2;
		1799
		1800	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1801	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1802	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1803	break;
		1804	case CHIP_CAICOS:
		1805	rdev->config.evergreen.num_ses = 1;
		1806	rdev->config.evergreen.max_pipes = 4;
		1807	rdev->config.evergreen.max_tile_pipes = 2;
		1808	rdev->config.evergreen.max_simds = 2;
		1809	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1810	rdev->config.evergreen.max_gprs = 256;
		1811	rdev->config.evergreen.max_threads = 192;
		1812	rdev->config.evergreen.max_gs_threads = 16;
		1813	rdev->config.evergreen.max_stack_entries = 256;
		1814	rdev->config.evergreen.sx_num_of_sets = 4;
		1815	rdev->config.evergreen.sx_max_export_size = 128;
		1816	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1817	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1818	rdev->config.evergreen.max_hw_contexts = 4;
		1819	rdev->config.evergreen.sq_num_cf_insts = 1;
		1820
		1821	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1822	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1823	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1824	break;
		1825	}
		1826
		1827	/* Initialize HDP */
		1828	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1829	WREG32((0x2c14 + j), 0x00000000);
		1830	WREG32((0x2c18 + j), 0x00000000);
		1831	WREG32((0x2c1c + j), 0x00000000);
		1832	WREG32((0x2c20 + j), 0x00000000);
		1833	WREG32((0x2c24 + j), 0x00000000);
		1834	}
		1835
		1836	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1837
		1838	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & ~2;
		1839
		1840	cc_gc_shader_pipe_config \|=
		1841	INACTIVE_QD_PIPES((EVERGREEN_MAX_PIPES_MASK << rdev->config.evergreen.max_pipes)
		1842	& EVERGREEN_MAX_PIPES_MASK);
		1843	cc_gc_shader_pipe_config \|=
		1844	INACTIVE_SIMDS((EVERGREEN_MAX_SIMDS_MASK << rdev->config.evergreen.max_simds)
		1845	& EVERGREEN_MAX_SIMDS_MASK);
		1846
		1847	cc_rb_backend_disable =
		1848	BACKEND_DISABLE((EVERGREEN_MAX_BACKENDS_MASK << rdev->config.evergreen.max_backends)
		1849	& EVERGREEN_MAX_BACKENDS_MASK);
		1850
		1851
		1852	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
		1853	if (rdev->flags & RADEON_IS_IGP)
		1854	mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
		1855	else
		1856	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
		1857
		1858	switch (rdev->config.evergreen.max_tile_pipes) {
		1859	case 1:
		1860	default:
		1861	gb_addr_config \|= NUM_PIPES(0);
		1862	break;
		1863	case 2:
		1864	gb_addr_config \|= NUM_PIPES(1);
		1865	break;
		1866	case 4:
		1867	gb_addr_config \|= NUM_PIPES(2);
		1868	break;
		1869	case 8:
		1870	gb_addr_config \|= NUM_PIPES(3);
		1871	break;
		1872	}
		1873
		1874	gb_addr_config \|= PIPE_INTERLEAVE_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
		1875	gb_addr_config \|= BANK_INTERLEAVE_SIZE(0);
		1876	gb_addr_config \|= NUM_SHADER_ENGINES(rdev->config.evergreen.num_ses - 1);
		1877	gb_addr_config \|= SHADER_ENGINE_TILE_SIZE(1);
		1878	gb_addr_config \|= NUM_GPUS(0); /* Hemlock? */
		1879	gb_addr_config \|= MULTI_GPU_TILE_SIZE(2);
		1880
		1881	if (((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) > 2)
		1882	gb_addr_config \|= ROW_SIZE(2);
		1883	else
		1884	gb_addr_config \|= ROW_SIZE((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT);
		1885
		1886	if (rdev->ddev->pdev->device == 0x689e) {
		1887	u32 efuse_straps_4;
		1888	u32 efuse_straps_3;
		1889	u8 efuse_box_bit_131_124;
		1890
		1891	WREG32(RCU_IND_INDEX, 0x204);
		1892	efuse_straps_4 = RREG32(RCU_IND_DATA);
		1893	WREG32(RCU_IND_INDEX, 0x203);
		1894	efuse_straps_3 = RREG32(RCU_IND_DATA);
		1895	efuse_box_bit_131_124 = (u8)(((efuse_straps_4 & 0xf) << 4) \| ((efuse_straps_3 & 0xf0000000) >> 28));
		1896
		1897	switch(efuse_box_bit_131_124) {
		1898	case 0x00:
		1899	gb_backend_map = 0x76543210;
		1900	break;
		1901	case 0x55:
		1902	gb_backend_map = 0x77553311;
		1903	break;
		1904	case 0x56:
		1905	gb_backend_map = 0x77553300;
		1906	break;
		1907	case 0x59:
		1908	gb_backend_map = 0x77552211;
		1909	break;
		1910	case 0x66:
		1911	gb_backend_map = 0x77443300;
		1912	break;
		1913	case 0x99:
		1914	gb_backend_map = 0x66552211;
		1915	break;
		1916	case 0x5a:
		1917	gb_backend_map = 0x77552200;
		1918	break;
		1919	case 0xaa:
		1920	gb_backend_map = 0x66442200;
		1921	break;
		1922	case 0x95:
		1923	gb_backend_map = 0x66553311;
		1924	break;
		1925	default:
		1926	DRM_ERROR("bad backend map, using default\n");
		1927	gb_backend_map =
		1928	evergreen_get_tile_pipe_to_backend_map(rdev,
		1929	rdev->config.evergreen.max_tile_pipes,
		1930	rdev->config.evergreen.max_backends,
		1931	((EVERGREEN_MAX_BACKENDS_MASK <<
		1932	rdev->config.evergreen.max_backends) &
		1933	EVERGREEN_MAX_BACKENDS_MASK));
		1934	break;
		1935	}
		1936	} else if (rdev->ddev->pdev->device == 0x68b9) {
		1937	u32 efuse_straps_3;
		1938	u8 efuse_box_bit_127_124;
		1939
		1940	WREG32(RCU_IND_INDEX, 0x203);
		1941	efuse_straps_3 = RREG32(RCU_IND_DATA);
		1942	efuse_box_bit_127_124 = (u8)((efuse_straps_3 & 0xF0000000) >> 28);
		1943
		1944	switch(efuse_box_bit_127_124) {
		1945	case 0x0:
		1946	gb_backend_map = 0x00003210;
		1947	break;
		1948	case 0x5:
		1949	case 0x6:
		1950	case 0x9:
		1951	case 0xa:
		1952	gb_backend_map = 0x00003311;
		1953	break;
		1954	default:
		1955	DRM_ERROR("bad backend map, using default\n");
		1956	gb_backend_map =
		1957	evergreen_get_tile_pipe_to_backend_map(rdev,
		1958	rdev->config.evergreen.max_tile_pipes,
		1959	rdev->config.evergreen.max_backends,
		1960	((EVERGREEN_MAX_BACKENDS_MASK <<
		1961	rdev->config.evergreen.max_backends) &
		1962	EVERGREEN_MAX_BACKENDS_MASK));
		1963	break;
		1964	}
		1965	} else {
		1966	switch (rdev->family) {
		1967	case CHIP_CYPRESS:
		1968	case CHIP_HEMLOCK:
		1969	case CHIP_BARTS:
		1970	gb_backend_map = 0x66442200;
		1971	break;
		1972	case CHIP_JUNIPER:
		1973	gb_backend_map = 0x00006420;
		1974	break;
		1975	default:
		1976	gb_backend_map =
		1977	evergreen_get_tile_pipe_to_backend_map(rdev,
		1978	rdev->config.evergreen.max_tile_pipes,
		1979	rdev->config.evergreen.max_backends,
		1980	((EVERGREEN_MAX_BACKENDS_MASK <<
		1981	rdev->config.evergreen.max_backends) &
		1982	EVERGREEN_MAX_BACKENDS_MASK));
		1983	}
		1984	}
		1985
		1986	/* setup tiling info dword. gb_addr_config is not adequate since it does
		1987	* not have bank info, so create a custom tiling dword.
		1988	* bits 3:0 num_pipes
		1989	* bits 7:4 num_banks
		1990	* bits 11:8 group_size
		1991	* bits 15:12 row_size
		1992	*/
		1993	rdev->config.evergreen.tile_config = 0;
		1994	switch (rdev->config.evergreen.max_tile_pipes) {
		1995	case 1:
		1996	default:
		1997	rdev->config.evergreen.tile_config \|= (0 << 0);
		1998	break;
		1999	case 2:
		2000	rdev->config.evergreen.tile_config \|= (1 << 0);
		2001	break;
		2002	case 4:
		2003	rdev->config.evergreen.tile_config \|= (2 << 0);
		2004	break;
		2005	case 8:
		2006	rdev->config.evergreen.tile_config \|= (3 << 0);
		2007	break;
		2008	}
1986	serge	2009	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1963	serge	2010	if (rdev->flags & RADEON_IS_IGP)
1986	serge	2011	rdev->config.evergreen.tile_config \|= 1 << 4;
1963	serge	2012	else
		2013	rdev->config.evergreen.tile_config \|=
		2014	((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
		2015	rdev->config.evergreen.tile_config \|=
		2016	((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT) << 8;
		2017	rdev->config.evergreen.tile_config \|=
		2018	((gb_addr_config & 0x30000000) >> 28) << 12;
		2019
		2020	WREG32(GB_BACKEND_MAP, gb_backend_map);
		2021	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		2022	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
		2023	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
		2024
		2025	evergreen_program_channel_remap(rdev);
		2026
		2027	num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
		2028	grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
		2029
		2030	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
		2031	u32 rb = cc_rb_backend_disable \| (0xf0 << 16);
		2032	u32 sp = cc_gc_shader_pipe_config;
		2033	u32 gfx = grbm_gfx_index \| SE_INDEX(i);
		2034
		2035	if (i == num_shader_engines) {
		2036	rb \|= BACKEND_DISABLE(EVERGREEN_MAX_BACKENDS_MASK);
		2037	sp \|= INACTIVE_SIMDS(EVERGREEN_MAX_SIMDS_MASK);
		2038	}
		2039
		2040	WREG32(GRBM_GFX_INDEX, gfx);
		2041	WREG32(RLC_GFX_INDEX, gfx);
		2042
		2043	WREG32(CC_RB_BACKEND_DISABLE, rb);
		2044	WREG32(CC_SYS_RB_BACKEND_DISABLE, rb);
		2045	WREG32(GC_USER_RB_BACKEND_DISABLE, rb);
		2046	WREG32(CC_GC_SHADER_PIPE_CONFIG, sp);
		2047	}
		2048
		2049	grbm_gfx_index \|= SE_BROADCAST_WRITES;
		2050	WREG32(GRBM_GFX_INDEX, grbm_gfx_index);
		2051	WREG32(RLC_GFX_INDEX, grbm_gfx_index);
		2052
		2053	WREG32(CGTS_SYS_TCC_DISABLE, 0);
		2054	WREG32(CGTS_TCC_DISABLE, 0);
		2055	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
		2056	WREG32(CGTS_USER_TCC_DISABLE, 0);
		2057
		2058	/* set HW defaults for 3D engine */
		2059	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \|
		2060	ROQ_IB2_START(0x2b)));
		2061
		2062	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
		2063
		2064	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \|
		2065	SYNC_GRADIENT \|
		2066	SYNC_WALKER \|
		2067	SYNC_ALIGNER));
		2068
		2069	sx_debug_1 = RREG32(SX_DEBUG_1);
		2070	sx_debug_1 \|= ENABLE_NEW_SMX_ADDRESS;
		2071	WREG32(SX_DEBUG_1, sx_debug_1);
		2072
		2073
		2074	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
		2075	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
		2076	smx_dc_ctl0 \|= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
		2077	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
		2078
		2079	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) \|
		2080	POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) \|
		2081	SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
		2082
		2083	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) \|
		2084	SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) \|
		2085	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
		2086
		2087	WREG32(VGT_NUM_INSTANCES, 1);
		2088	WREG32(SPI_CONFIG_CNTL, 0);
		2089	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
		2090	WREG32(CP_PERFMON_CNTL, 0);
		2091
		2092	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) \|
		2093	FETCH_FIFO_HIWATER(0x4) \|
		2094	DONE_FIFO_HIWATER(0xe0) \|
		2095	ALU_UPDATE_FIFO_HIWATER(0x8)));
		2096
		2097	sq_config = RREG32(SQ_CONFIG);
		2098	sq_config &= ~(PS_PRIO(3) \|
		2099	VS_PRIO(3) \|
		2100	GS_PRIO(3) \|
		2101	ES_PRIO(3));
		2102	sq_config \|= (VC_ENABLE \|
		2103	EXPORT_SRC_C \|
		2104	PS_PRIO(0) \|
		2105	VS_PRIO(1) \|
		2106	GS_PRIO(2) \|
		2107	ES_PRIO(3));
		2108
		2109	switch (rdev->family) {
		2110	case CHIP_CEDAR:
		2111	case CHIP_PALM:
		2112	case CHIP_SUMO:
		2113	case CHIP_SUMO2:
		2114	case CHIP_CAICOS:
		2115	/* no vertex cache */
		2116	sq_config &= ~VC_ENABLE;
		2117	break;
		2118	default:
		2119	break;
		2120	}
		2121
		2122	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
		2123
		2124	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
		2125	sq_gpr_resource_mgmt_1 \|= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
		2126	sq_gpr_resource_mgmt_1 \|= NUM_CLAUSE_TEMP_GPRS(4);
		2127	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		2128	sq_gpr_resource_mgmt_2 \|= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		2129	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		2130	sq_gpr_resource_mgmt_3 \|= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		2131
		2132	switch (rdev->family) {
		2133	case CHIP_CEDAR:
		2134	case CHIP_PALM:
		2135	case CHIP_SUMO:
		2136	case CHIP_SUMO2:
		2137	ps_thread_count = 96;
		2138	break;
		2139	default:
		2140	ps_thread_count = 128;
		2141	break;
		2142	}
		2143
		2144	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
		2145	sq_thread_resource_mgmt \|= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2146	sq_thread_resource_mgmt \|= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2147	sq_thread_resource_mgmt \|= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2148	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2149	sq_thread_resource_mgmt_2 \|= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2150
		2151	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2152	sq_stack_resource_mgmt_1 \|= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2153	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2154	sq_stack_resource_mgmt_2 \|= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2155	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2156	sq_stack_resource_mgmt_3 \|= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2157
		2158	WREG32(SQ_CONFIG, sq_config);
		2159	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		2160	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		2161	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
		2162	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		2163	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
		2164	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		2165	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		2166	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
		2167	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
		2168	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
		2169
		2170	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		2171	FORCE_EOV_MAX_REZ_CNT(255)));
		2172
		2173	switch (rdev->family) {
		2174	case CHIP_CEDAR:
		2175	case CHIP_PALM:
		2176	case CHIP_SUMO:
		2177	case CHIP_SUMO2:
		2178	case CHIP_CAICOS:
		2179	vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
		2180	break;
		2181	default:
		2182	vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
		2183	break;
		2184	}
		2185	vgt_cache_invalidation \|= AUTO_INVLD_EN(ES_AND_GS_AUTO);
		2186	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
		2187
		2188	WREG32(VGT_GS_VERTEX_REUSE, 16);
		2189	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
		2190	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		2191
		2192	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
		2193	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
		2194
		2195	WREG32(CB_PERF_CTR0_SEL_0, 0);
		2196	WREG32(CB_PERF_CTR0_SEL_1, 0);
		2197	WREG32(CB_PERF_CTR1_SEL_0, 0);
		2198	WREG32(CB_PERF_CTR1_SEL_1, 0);
		2199	WREG32(CB_PERF_CTR2_SEL_0, 0);
		2200	WREG32(CB_PERF_CTR2_SEL_1, 0);
		2201	WREG32(CB_PERF_CTR3_SEL_0, 0);
		2202	WREG32(CB_PERF_CTR3_SEL_1, 0);
		2203
		2204	/* clear render buffer base addresses */
		2205	WREG32(CB_COLOR0_BASE, 0);
		2206	WREG32(CB_COLOR1_BASE, 0);
		2207	WREG32(CB_COLOR2_BASE, 0);
		2208	WREG32(CB_COLOR3_BASE, 0);
		2209	WREG32(CB_COLOR4_BASE, 0);
		2210	WREG32(CB_COLOR5_BASE, 0);
		2211	WREG32(CB_COLOR6_BASE, 0);
		2212	WREG32(CB_COLOR7_BASE, 0);
		2213	WREG32(CB_COLOR8_BASE, 0);
		2214	WREG32(CB_COLOR9_BASE, 0);
		2215	WREG32(CB_COLOR10_BASE, 0);
		2216	WREG32(CB_COLOR11_BASE, 0);
		2217
		2218	/* set the shader const cache sizes to 0 */
		2219	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
		2220	WREG32(i, 0);
		2221	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
		2222	WREG32(i, 0);
		2223
		2224	tmp = RREG32(HDP_MISC_CNTL);
		2225	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		2226	WREG32(HDP_MISC_CNTL, tmp);
		2227
		2228	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		2229	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		2230
		2231	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		2232
		2233	udelay(50);
		2234
1430	serge	2235	}
		2236
		2237	int evergreen_mc_init(struct radeon_device *rdev)
		2238	{
		2239	u32 tmp;
		2240	int chansize, numchan;
		2241
		2242	/* Get VRAM informations */
		2243	rdev->mc.vram_is_ddr = true;
2004	serge	2244	if (rdev->flags & RADEON_IS_IGP)
		2245	tmp = RREG32(FUS_MC_ARB_RAMCFG);
		2246	else
1430	serge	2247	tmp = RREG32(MC_ARB_RAMCFG);
		2248	if (tmp & CHANSIZE_OVERRIDE) {
		2249	chansize = 16;
		2250	} else if (tmp & CHANSIZE_MASK) {
		2251	chansize = 64;
		2252	} else {
		2253	chansize = 32;
		2254	}
		2255	tmp = RREG32(MC_SHARED_CHMAP);
		2256	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		2257	case 0:
		2258	default:
		2259	numchan = 1;
		2260	break;
		2261	case 1:
		2262	numchan = 2;
		2263	break;
		2264	case 2:
		2265	numchan = 4;
		2266	break;
		2267	case 3:
		2268	numchan = 8;
		2269	break;
		2270	}
		2271	rdev->mc.vram_width = numchan * chansize;
		2272	/* Could aper size report 0 ? */
1963	serge	2273	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		2274	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1430	serge	2275	/* Setup GPU memory space */
1963	serge	2276	if (rdev->flags & RADEON_IS_IGP) {
		2277	/* size in bytes on fusion */
		2278	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		2279	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
		2280	} else {
1430	serge	2281	/* size in MB on evergreen */
		2282	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
		2283	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
1963	serge	2284	}
1430	serge	2285	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1963	serge	2286	r700_vram_gtt_location(rdev, &rdev->mc);
		2287	radeon_update_bandwidth_info(rdev);
		2288
1430	serge	2289	return 0;
		2290	}
		2291
1963	serge	2292	bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
1430	serge	2293	{
1986	serge	2294	u32 srbm_status;
		2295	u32 grbm_status;
		2296	u32 grbm_status_se0, grbm_status_se1;
		2297	struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
		2298	int r;
		2299
		2300	srbm_status = RREG32(SRBM_STATUS);
		2301	grbm_status = RREG32(GRBM_STATUS);
		2302	grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
		2303	grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
		2304	if (!(grbm_status & GUI_ACTIVE)) {
		2305	r100_gpu_lockup_update(lockup, &rdev->cp);
1963	serge	2306	return false;
1986	serge	2307	}
		2308	/* force CP activities */
		2309	r = radeon_ring_lock(rdev, 2);
		2310	if (!r) {
		2311	/* PACKET2 NOP */
		2312	radeon_ring_write(rdev, 0x80000000);
		2313	radeon_ring_write(rdev, 0x80000000);
		2314	radeon_ring_unlock_commit(rdev);
		2315	}
		2316	rdev->cp.rptr = RREG32(CP_RB_RPTR);
		2317	return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
1963	serge	2318	}
		2319
		2320	static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
		2321	{
		2322	struct evergreen_mc_save save;
		2323	u32 grbm_reset = 0;
		2324
		2325	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
		2326	return 0;
		2327
		2328	dev_info(rdev->dev, "GPU softreset \n");
		2329	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2330	RREG32(GRBM_STATUS));
		2331	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2332	RREG32(GRBM_STATUS_SE0));
		2333	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2334	RREG32(GRBM_STATUS_SE1));
		2335	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2336	RREG32(SRBM_STATUS));
		2337	evergreen_mc_stop(rdev, &save);
		2338	if (evergreen_mc_wait_for_idle(rdev)) {
		2339	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		2340	}
		2341	/* Disable CP parsing/prefetching */
		2342	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT);
		2343
		2344	/* reset all the gfx blocks */
		2345	grbm_reset = (SOFT_RESET_CP \|
		2346	SOFT_RESET_CB \|
		2347	SOFT_RESET_DB \|
		2348	SOFT_RESET_PA \|
		2349	SOFT_RESET_SC \|
		2350	SOFT_RESET_SPI \|
		2351	SOFT_RESET_SH \|
		2352	SOFT_RESET_SX \|
		2353	SOFT_RESET_TC \|
		2354	SOFT_RESET_TA \|
		2355	SOFT_RESET_VC \|
		2356	SOFT_RESET_VGT);
		2357
		2358	dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
		2359	WREG32(GRBM_SOFT_RESET, grbm_reset);
		2360	(void)RREG32(GRBM_SOFT_RESET);
		2361	udelay(50);
		2362	WREG32(GRBM_SOFT_RESET, 0);
		2363	(void)RREG32(GRBM_SOFT_RESET);
		2364	/* Wait a little for things to settle down */
		2365	udelay(50);
		2366	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2367	RREG32(GRBM_STATUS));
		2368	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2369	RREG32(GRBM_STATUS_SE0));
		2370	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2371	RREG32(GRBM_STATUS_SE1));
		2372	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2373	RREG32(SRBM_STATUS));
		2374	evergreen_mc_resume(rdev, &save);
1430	serge	2375	return 0;
		2376	}
		2377
1963	serge	2378	int evergreen_asic_reset(struct radeon_device *rdev)
		2379	{
		2380	return evergreen_gpu_soft_reset(rdev);
		2381	}
		2382
		2383	/* Interrupts */
		2384
		2385	u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
		2386	{
		2387	switch (crtc) {
		2388	case 0:
		2389	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2390	case 1:
		2391	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2392	case 2:
		2393	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2394	case 3:
		2395	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2396	case 4:
		2397	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2398	case 5:
		2399	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2400	default:
		2401	return 0;
		2402	}
		2403	}
		2404
		2405	void evergreen_disable_interrupt_state(struct radeon_device *rdev)
		2406	{
		2407	u32 tmp;
		2408
		2409	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		2410	WREG32(GRBM_INT_CNTL, 0);
		2411	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2412	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
		2413	if (!(rdev->flags & RADEON_IS_IGP)) {
		2414	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2415	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		2416	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2417	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		2418	}
		2419
		2420	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2421	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
		2422	if (!(rdev->flags & RADEON_IS_IGP)) {
		2423	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2424	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		2425	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2426	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		2427	}
		2428
		2429	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		2430	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2431
		2432	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2433	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2434	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2435	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2436	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2437	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2438	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2439	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2440	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2441	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2442	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2443	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2444
		2445	}
1430	serge	2446	static int evergreen_startup(struct radeon_device *rdev)
		2447	{
		2448	int r;
		2449
1990	serge	2450	/* enable pcie gen2 link */
		2451	if (!ASIC_IS_DCE5(rdev))
		2452	evergreen_pcie_gen2_enable(rdev);
		2453
		2454	if (ASIC_IS_DCE5(rdev)) {
		2455	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw \|\| !rdev->mc_fw) {
		2456	r = ni_init_microcode(rdev);
		2457	if (r) {
		2458	DRM_ERROR("Failed to load firmware!\n");
		2459	return r;
		2460	}
		2461	}
		2462	r = ni_mc_load_microcode(rdev);
		2463	if (r) {
		2464	DRM_ERROR("Failed to load MC firmware!\n");
		2465	return r;
		2466	}
		2467	} else {
1430	serge	2468	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		2469	r = r600_init_microcode(rdev);
		2470	if (r) {
		2471	DRM_ERROR("Failed to load firmware!\n");
		2472	return r;
		2473	}
		2474	}
1990	serge	2475	}
1963	serge	2476
1430	serge	2477	evergreen_mc_program(rdev);
		2478	if (rdev->flags & RADEON_IS_AGP) {
1963	serge	2479	evergreen_agp_enable(rdev);
1430	serge	2480	} else {
		2481	r = evergreen_pcie_gart_enable(rdev);
		2482	if (r)
		2483	return r;
		2484	}
		2485	evergreen_gpu_init(rdev);
		2486	#if 0
1963	serge	2487	r = evergreen_blit_init(rdev);
		2488	if (r) {
		2489	evergreen_blit_fini(rdev);
		2490	rdev->asic->copy = NULL;
		2491	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		2492	}
1430	serge	2493
1963	serge	2494	/* allocate wb buffer */
		2495	r = radeon_wb_init(rdev);
		2496	if (r)
		2497	return r;
		2498	#endif
		2499
		2500	/* Enable IRQ */
		2501
1430	serge	2502	r = radeon_ring_init(rdev, rdev->cp.ring_size);
		2503	if (r)
		2504	return r;
		2505	r = evergreen_cp_load_microcode(rdev);
		2506	if (r)
		2507	return r;
1963	serge	2508	r = evergreen_cp_resume(rdev);
1430	serge	2509	if (r)
		2510	return r;
1963	serge	2511
1430	serge	2512	return 0;
		2513	}
		2514
		2515
		2516
		2517
		2518
		2519
		2520
		2521	/* Plan is to move initialization in that function and use
		2522	* helper function so that radeon_device_init pretty much
		2523	* do nothing more than calling asic specific function. This
		2524	* should also allow to remove a bunch of callback function
		2525	* like vram_info.
		2526	*/
		2527	int evergreen_init(struct radeon_device *rdev)
		2528	{
		2529	int r;
		2530
		2531	/* This don't do much */
		2532	r = radeon_gem_init(rdev);
		2533	if (r)
		2534	return r;
		2535	/* Read BIOS */
		2536	if (!radeon_get_bios(rdev)) {
		2537	if (ASIC_IS_AVIVO(rdev))
		2538	return -EINVAL;
		2539	}
		2540	/* Must be an ATOMBIOS */
		2541	if (!rdev->is_atom_bios) {
1986	serge	2542	dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
1430	serge	2543	return -EINVAL;
		2544	}
		2545	r = radeon_atombios_init(rdev);
		2546	if (r)
		2547	return r;
1986	serge	2548	/* reset the asic, the gfx blocks are often in a bad state
		2549	* after the driver is unloaded or after a resume
		2550	*/
		2551	if (radeon_asic_reset(rdev))
		2552	dev_warn(rdev->dev, "GPU reset failed !\n");
1430	serge	2553	/* Post card if necessary */
1986	serge	2554	if (!radeon_card_posted(rdev)) {
1430	serge	2555	if (!rdev->bios) {
		2556	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		2557	return -EINVAL;
		2558	}
		2559	DRM_INFO("GPU not posted. posting now...\n");
		2560	atom_asic_init(rdev->mode_info.atom_context);
		2561	}
		2562	/* Initialize scratch registers */
		2563	r600_scratch_init(rdev);
		2564	/* Initialize surface registers */
		2565	radeon_surface_init(rdev);
		2566	/* Initialize clocks */
		2567	radeon_get_clock_info(rdev->ddev);
		2568	/* Fence driver */
		2569	/* initialize AGP */
		2570	if (rdev->flags & RADEON_IS_AGP) {
		2571	r = radeon_agp_init(rdev);
		2572	if (r)
		2573	radeon_agp_disable(rdev);
		2574	}
		2575	/* initialize memory controller */
		2576	r = evergreen_mc_init(rdev);
		2577	if (r)
		2578	return r;
		2579	/* Memory manager */
		2580	r = radeon_bo_init(rdev);
		2581	if (r)
		2582	return r;
1963	serge	2583
1430	serge	2584
		2585	rdev->cp.ring_obj = NULL;
		2586	r600_ring_init(rdev, 1024 * 1024);
		2587
1963	serge	2588	// rdev->ih.ring_obj = NULL;
		2589	// r600_ih_ring_init(rdev, 64 * 1024);
1430	serge	2590
		2591	r = r600_pcie_gart_init(rdev);
		2592	if (r)
		2593	return r;
1963	serge	2594
		2595	rdev->accel_working = true;
1430	serge	2596	r = evergreen_startup(rdev);
		2597	if (r) {
1963	serge	2598	dev_err(rdev->dev, "disabling GPU acceleration\n");
1430	serge	2599	rdev->accel_working = false;
		2600	}
		2601	if (rdev->accel_working) {
		2602	}
		2603	return 0;
		2604	}
		2605
1986	serge	2606
		2607	static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
1430	serge	2608	{
1986	serge	2609	u32 link_width_cntl, speed_cntl;
		2610
		2611	if (radeon_pcie_gen2 == 0)
		2612	return;
		2613
		2614	if (rdev->flags & RADEON_IS_IGP)
		2615	return;
		2616
		2617	if (!(rdev->flags & RADEON_IS_PCIE))
		2618	return;
		2619
		2620	/* x2 cards have a special sequence */
		2621	if (ASIC_IS_X2(rdev))
		2622	return;
		2623
		2624	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2625	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) \|\|
		2626	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		2627
		2628	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		2629	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		2630	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		2631
		2632	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2633	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		2634	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2635
		2636	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2637	speed_cntl \|= LC_CLR_FAILED_SPD_CHANGE_CNT;
		2638	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2639
		2640	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2641	speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
		2642	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2643
		2644	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2645	speed_cntl \|= LC_GEN2_EN_STRAP;
		2646	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2647
		2648	} else {
		2649	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		2650	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		2651	if (1)
		2652	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		2653	else
		2654	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		2655	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		2656	}
1430	serge	2657	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/evergreen.c – Rev 2004