WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/evergreen.c

Rev	Author	Line No.	Line
1430	serge	1	/*
		2	* Copyright 2010 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	//#include
1963	serge	26	#include
1430	serge	27	#include "drmP.h"
		28	#include "radeon.h"
1963	serge	29	#include "radeon_asic.h"
1430	serge	30	#include "radeon_drm.h"
1963	serge	31	#include "evergreend.h"
1430	serge	32	#include "atom.h"
		33	#include "avivod.h"
		34	#include "evergreen_reg.h"
1986	serge	35	#include "evergreen_blit_shaders.h"
1430	serge	36
1963	serge	37	#define EVERGREEN_PFP_UCODE_SIZE 1120
		38	#define EVERGREEN_PM4_UCODE_SIZE 1376
		39
1430	serge	40	static void evergreen_gpu_init(struct radeon_device *rdev);
		41	void evergreen_fini(struct radeon_device *rdev);
1986	serge	42	static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
1430	serge	43
1990	serge	44
		45
		46	u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
		47	{
		48	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
		49	u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
		50
		51	/* Lock the graphics update lock */
		52	tmp \|= EVERGREEN_GRPH_UPDATE_LOCK;
		53	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		54
		55	/* update the scanout addresses */
		56	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		57	upper_32_bits(crtc_base));
		58	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		59	(u32)crtc_base);
		60
		61	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		62	upper_32_bits(crtc_base));
		63	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		64	(u32)crtc_base);
		65
		66	/* Wait for update_pending to go high. */
		67	while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING));
		68	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
		69
		70	/* Unlock the lock, so double-buffering can take place inside vblank */
		71	tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
		72	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		73
		74	/* Return current update_pending status: */
		75	return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
		76	}
		77
		78	/* get temperature in millidegrees */
		79	int evergreen_get_temp(struct radeon_device *rdev)
		80	{
		81	u32 temp, toffset;
		82	int actual_temp = 0;
		83
		84	if (rdev->family == CHIP_JUNIPER) {
		85	toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
		86	TOFFSET_SHIFT;
		87	temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
		88	TS0_ADC_DOUT_SHIFT;
		89
		90	if (toffset & 0x100)
		91	actual_temp = temp / 2 - (0x200 - toffset);
		92	else
		93	actual_temp = temp / 2 + toffset;
		94
		95	actual_temp = actual_temp * 1000;
		96
		97	} else {
		98	temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
		99	ASIC_T_SHIFT;
		100
		101	if (temp & 0x400)
		102	actual_temp = -256;
		103	else if (temp & 0x200)
		104	actual_temp = 255;
		105	else if (temp & 0x100) {
		106	actual_temp = temp & 0x1ff;
		107	actual_temp \|= ~0x1ff;
		108	} else
		109	actual_temp = temp & 0xff;
		110
		111	actual_temp = (actual_temp * 1000) / 2;
		112	}
		113
		114	return actual_temp;
		115	}
		116
		117	int sumo_get_temp(struct radeon_device *rdev)
		118	{
		119	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
		120	int actual_temp = temp - 49;
		121
		122	return actual_temp * 1000;
		123	}
		124
		125	void evergreen_pm_misc(struct radeon_device *rdev)
		126	{
		127	int req_ps_idx = rdev->pm.requested_power_state_index;
		128	int req_cm_idx = rdev->pm.requested_clock_mode_index;
		129	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
		130	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
		131
		132	if (voltage->type == VOLTAGE_SW) {
		133	/* 0xff01 is a flag rather then an actual voltage */
		134	if (voltage->voltage == 0xff01)
		135	return;
		136	if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
		137	radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
		138	rdev->pm.current_vddc = voltage->voltage;
		139	DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
		140	}
		141	/* 0xff01 is a flag rather then an actual voltage */
		142	if (voltage->vddci == 0xff01)
		143	return;
		144	if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
		145	radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
		146	rdev->pm.current_vddci = voltage->vddci;
		147	DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
		148	}
		149	}
		150	}
		151
		152	void evergreen_pm_prepare(struct radeon_device *rdev)
		153	{
		154	struct drm_device *ddev = rdev->ddev;
		155	struct drm_crtc *crtc;
		156	struct radeon_crtc *radeon_crtc;
		157	u32 tmp;
		158
		159	/* disable any active CRTCs */
		160	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		161	radeon_crtc = to_radeon_crtc(crtc);
		162	if (radeon_crtc->enabled) {
		163	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		164	tmp \|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		165	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		166	}
		167	}
		168	}
		169
		170	void evergreen_pm_finish(struct radeon_device *rdev)
		171	{
		172	struct drm_device *ddev = rdev->ddev;
		173	struct drm_crtc *crtc;
		174	struct radeon_crtc *radeon_crtc;
		175	u32 tmp;
		176
		177	/* enable any active CRTCs */
		178	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		179	radeon_crtc = to_radeon_crtc(crtc);
		180	if (radeon_crtc->enabled) {
		181	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		182	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		183	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		184	}
		185	}
		186	}
		187
1430	serge	188	bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		189	{
		190	bool connected = false;
1963	serge	191
		192	switch (hpd) {
		193	case RADEON_HPD_1:
		194	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		195	connected = true;
		196	break;
		197	case RADEON_HPD_2:
		198	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		199	connected = true;
		200	break;
		201	case RADEON_HPD_3:
		202	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		203	connected = true;
		204	break;
		205	case RADEON_HPD_4:
		206	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		207	connected = true;
		208	break;
		209	case RADEON_HPD_5:
		210	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		211	connected = true;
		212	break;
		213	case RADEON_HPD_6:
		214	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		215	connected = true;
		216	break;
		217	default:
		218	break;
		219	}
		220
1430	serge	221	return connected;
		222	}
		223
		224	void evergreen_hpd_set_polarity(struct radeon_device *rdev,
		225	enum radeon_hpd_id hpd)
		226	{
1963	serge	227	u32 tmp;
		228	bool connected = evergreen_hpd_sense(rdev, hpd);
		229
		230	switch (hpd) {
		231	case RADEON_HPD_1:
		232	tmp = RREG32(DC_HPD1_INT_CONTROL);
		233	if (connected)
		234	tmp &= ~DC_HPDx_INT_POLARITY;
		235	else
		236	tmp \|= DC_HPDx_INT_POLARITY;
		237	WREG32(DC_HPD1_INT_CONTROL, tmp);
		238	break;
		239	case RADEON_HPD_2:
		240	tmp = RREG32(DC_HPD2_INT_CONTROL);
		241	if (connected)
		242	tmp &= ~DC_HPDx_INT_POLARITY;
		243	else
		244	tmp \|= DC_HPDx_INT_POLARITY;
		245	WREG32(DC_HPD2_INT_CONTROL, tmp);
		246	break;
		247	case RADEON_HPD_3:
		248	tmp = RREG32(DC_HPD3_INT_CONTROL);
		249	if (connected)
		250	tmp &= ~DC_HPDx_INT_POLARITY;
		251	else
		252	tmp \|= DC_HPDx_INT_POLARITY;
		253	WREG32(DC_HPD3_INT_CONTROL, tmp);
		254	break;
		255	case RADEON_HPD_4:
		256	tmp = RREG32(DC_HPD4_INT_CONTROL);
		257	if (connected)
		258	tmp &= ~DC_HPDx_INT_POLARITY;
		259	else
		260	tmp \|= DC_HPDx_INT_POLARITY;
		261	WREG32(DC_HPD4_INT_CONTROL, tmp);
		262	break;
		263	case RADEON_HPD_5:
		264	tmp = RREG32(DC_HPD5_INT_CONTROL);
		265	if (connected)
		266	tmp &= ~DC_HPDx_INT_POLARITY;
		267	else
		268	tmp \|= DC_HPDx_INT_POLARITY;
		269	WREG32(DC_HPD5_INT_CONTROL, tmp);
		270	break;
		271	case RADEON_HPD_6:
		272	tmp = RREG32(DC_HPD6_INT_CONTROL);
		273	if (connected)
		274	tmp &= ~DC_HPDx_INT_POLARITY;
		275	else
		276	tmp \|= DC_HPDx_INT_POLARITY;
		277	WREG32(DC_HPD6_INT_CONTROL, tmp);
		278	break;
		279	default:
		280	break;
		281	}
1430	serge	282	}
		283
1963	serge	284	#if 0
1430	serge	285	void evergreen_hpd_init(struct radeon_device *rdev)
		286	{
1963	serge	287	struct drm_device *dev = rdev->ddev;
		288	struct drm_connector *connector;
		289	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \|
		290	DC_HPDx_RX_INT_TIMER(0xfa) \| DC_HPDx_EN;
		291
		292	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		293	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		294	switch (radeon_connector->hpd.hpd) {
		295	case RADEON_HPD_1:
		296	WREG32(DC_HPD1_CONTROL, tmp);
		297	rdev->irq.hpd[0] = true;
		298	break;
		299	case RADEON_HPD_2:
		300	WREG32(DC_HPD2_CONTROL, tmp);
		301	rdev->irq.hpd[1] = true;
		302	break;
		303	case RADEON_HPD_3:
		304	WREG32(DC_HPD3_CONTROL, tmp);
		305	rdev->irq.hpd[2] = true;
		306	break;
		307	case RADEON_HPD_4:
		308	WREG32(DC_HPD4_CONTROL, tmp);
		309	rdev->irq.hpd[3] = true;
		310	break;
		311	case RADEON_HPD_5:
		312	WREG32(DC_HPD5_CONTROL, tmp);
		313	rdev->irq.hpd[4] = true;
		314	break;
		315	case RADEON_HPD_6:
		316	WREG32(DC_HPD6_CONTROL, tmp);
		317	rdev->irq.hpd[5] = true;
		318	break;
		319	default:
		320	break;
		321	}
		322	}
		323	if (rdev->irq.installed)
		324	evergreen_irq_set(rdev);
1430	serge	325	}
		326
1963	serge	327	void evergreen_hpd_fini(struct radeon_device *rdev)
		328	{
		329	struct drm_device *dev = rdev->ddev;
		330	struct drm_connector *connector;
1430	serge	331
1963	serge	332	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		333	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		334	switch (radeon_connector->hpd.hpd) {
		335	case RADEON_HPD_1:
		336	WREG32(DC_HPD1_CONTROL, 0);
		337	rdev->irq.hpd[0] = false;
		338	break;
		339	case RADEON_HPD_2:
		340	WREG32(DC_HPD2_CONTROL, 0);
		341	rdev->irq.hpd[1] = false;
		342	break;
		343	case RADEON_HPD_3:
		344	WREG32(DC_HPD3_CONTROL, 0);
		345	rdev->irq.hpd[2] = false;
		346	break;
		347	case RADEON_HPD_4:
		348	WREG32(DC_HPD4_CONTROL, 0);
		349	rdev->irq.hpd[3] = false;
		350	break;
		351	case RADEON_HPD_5:
		352	WREG32(DC_HPD5_CONTROL, 0);
		353	rdev->irq.hpd[4] = false;
		354	break;
		355	case RADEON_HPD_6:
		356	WREG32(DC_HPD6_CONTROL, 0);
		357	rdev->irq.hpd[5] = false;
		358	break;
		359	default:
		360	break;
		361	}
		362	}
1430	serge	363	}
		364
1963	serge	365	#endif
1986	serge	366	/* watermark setup */
1963	serge	367
1986	serge	368	static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
		369	struct radeon_crtc *radeon_crtc,
		370	struct drm_display_mode *mode,
		371	struct drm_display_mode *other_mode)
		372	{
		373	u32 tmp;
		374	/*
		375	* Line Buffer Setup
		376	* There are 3 line buffers, each one shared by 2 display controllers.
		377	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
		378	* the display controllers. The paritioning is done via one of four
		379	* preset allocations specified in bits 2:0:
		380	* first display controller
		381	* 0 - first half of lb (3840 * 2)
		382	* 1 - first 3/4 of lb (5760 * 2)
		383	* 2 - whole lb (7680 * 2), other crtc must be disabled
		384	* 3 - first 1/4 of lb (1920 * 2)
		385	* second display controller
		386	* 4 - second half of lb (3840 * 2)
		387	* 5 - second 3/4 of lb (5760 * 2)
		388	* 6 - whole lb (7680 * 2), other crtc must be disabled
		389	* 7 - last 1/4 of lb (1920 * 2)
		390	*/
		391	/* this can get tricky if we have two large displays on a paired group
		392	* of crtcs. Ideally for multiple large displays we'd assign them to
		393	* non-linked crtcs for maximum line buffer allocation.
		394	*/
		395	if (radeon_crtc->base.enabled && mode) {
		396	if (other_mode)
		397	tmp = 0; /* 1/2 */
		398	else
		399	tmp = 2; /* whole */
		400	} else
		401	tmp = 0;
1963	serge	402
1986	serge	403	/* second controller of the pair uses second half of the lb */
		404	if (radeon_crtc->crtc_id % 2)
		405	tmp += 4;
		406	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
		407
		408	if (radeon_crtc->base.enabled && mode) {
		409	switch (tmp) {
		410	case 0:
		411	case 4:
		412	default:
		413	if (ASIC_IS_DCE5(rdev))
		414	return 4096 * 2;
		415	else
		416	return 3840 * 2;
		417	case 1:
		418	case 5:
		419	if (ASIC_IS_DCE5(rdev))
		420	return 6144 * 2;
		421	else
		422	return 5760 * 2;
		423	case 2:
		424	case 6:
		425	if (ASIC_IS_DCE5(rdev))
		426	return 8192 * 2;
		427	else
		428	return 7680 * 2;
		429	case 3:
		430	case 7:
		431	if (ASIC_IS_DCE5(rdev))
		432	return 2048 * 2;
		433	else
		434	return 1920 * 2;
		435	}
		436	}
		437
		438	/* controller not enabled, so no lb used */
		439	return 0;
		440	}
		441
		442	static u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
		443	{
		444	u32 tmp = RREG32(MC_SHARED_CHMAP);
		445
		446	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		447	case 0:
		448	default:
		449	return 1;
		450	case 1:
		451	return 2;
		452	case 2:
		453	return 4;
		454	case 3:
		455	return 8;
		456	}
		457	}
		458
		459	struct evergreen_wm_params {
		460	u32 dram_channels; /* number of dram channels */
		461	u32 yclk; /* bandwidth per dram data pin in kHz */
		462	u32 sclk; /* engine clock in kHz */
		463	u32 disp_clk; /* display clock in kHz */
		464	u32 src_width; /* viewport width */
		465	u32 active_time; /* active display time in ns */
		466	u32 blank_time; /* blank time in ns */
		467	bool interlaced; /* mode is interlaced */
		468	fixed20_12 vsc; /* vertical scale ratio */
		469	u32 num_heads; /* number of active crtcs */
		470	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
		471	u32 lb_size; /* line buffer allocated to pipe */
		472	u32 vtaps; /* vertical scaler taps */
		473	};
		474
		475	static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
		476	{
		477	/* Calculate DRAM Bandwidth and the part allocated to display. */
		478	fixed20_12 dram_efficiency; /* 0.7 */
		479	fixed20_12 yclk, dram_channels, bandwidth;
		480	fixed20_12 a;
		481
		482	a.full = dfixed_const(1000);
		483	yclk.full = dfixed_const(wm->yclk);
		484	yclk.full = dfixed_div(yclk, a);
		485	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		486	a.full = dfixed_const(10);
		487	dram_efficiency.full = dfixed_const(7);
		488	dram_efficiency.full = dfixed_div(dram_efficiency, a);
		489	bandwidth.full = dfixed_mul(dram_channels, yclk);
		490	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
		491
		492	return dfixed_trunc(bandwidth);
		493	}
		494
		495	static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		496	{
		497	/* Calculate DRAM Bandwidth and the part allocated to display. */
		498	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
		499	fixed20_12 yclk, dram_channels, bandwidth;
		500	fixed20_12 a;
		501
		502	a.full = dfixed_const(1000);
		503	yclk.full = dfixed_const(wm->yclk);
		504	yclk.full = dfixed_div(yclk, a);
		505	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		506	a.full = dfixed_const(10);
		507	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
		508	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
		509	bandwidth.full = dfixed_mul(dram_channels, yclk);
		510	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
		511
		512	return dfixed_trunc(bandwidth);
		513	}
		514
		515	static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
		516	{
		517	/* Calculate the display Data return Bandwidth */
		518	fixed20_12 return_efficiency; /* 0.8 */
		519	fixed20_12 sclk, bandwidth;
		520	fixed20_12 a;
		521
		522	a.full = dfixed_const(1000);
		523	sclk.full = dfixed_const(wm->sclk);
		524	sclk.full = dfixed_div(sclk, a);
		525	a.full = dfixed_const(10);
		526	return_efficiency.full = dfixed_const(8);
		527	return_efficiency.full = dfixed_div(return_efficiency, a);
		528	a.full = dfixed_const(32);
		529	bandwidth.full = dfixed_mul(a, sclk);
		530	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
		531
		532	return dfixed_trunc(bandwidth);
		533	}
		534
		535	static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
		536	{
		537	/* Calculate the DMIF Request Bandwidth */
		538	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
		539	fixed20_12 disp_clk, bandwidth;
		540	fixed20_12 a;
		541
		542	a.full = dfixed_const(1000);
		543	disp_clk.full = dfixed_const(wm->disp_clk);
		544	disp_clk.full = dfixed_div(disp_clk, a);
		545	a.full = dfixed_const(10);
		546	disp_clk_request_efficiency.full = dfixed_const(8);
		547	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
		548	a.full = dfixed_const(32);
		549	bandwidth.full = dfixed_mul(a, disp_clk);
		550	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
		551
		552	return dfixed_trunc(bandwidth);
		553	}
		554
		555	static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
		556	{
		557	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
		558	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
		559	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
		560	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
		561
		562	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
		563	}
		564
		565	static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
		566	{
		567	/* Calculate the display mode Average Bandwidth
		568	* DisplayMode should contain the source and destination dimensions,
		569	* timing, etc.
		570	*/
		571	fixed20_12 bpp;
		572	fixed20_12 line_time;
		573	fixed20_12 src_width;
		574	fixed20_12 bandwidth;
		575	fixed20_12 a;
		576
		577	a.full = dfixed_const(1000);
		578	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
		579	line_time.full = dfixed_div(line_time, a);
		580	bpp.full = dfixed_const(wm->bytes_per_pixel);
		581	src_width.full = dfixed_const(wm->src_width);
		582	bandwidth.full = dfixed_mul(src_width, bpp);
		583	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
		584	bandwidth.full = dfixed_div(bandwidth, line_time);
		585
		586	return dfixed_trunc(bandwidth);
		587	}
		588
		589	static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
		590	{
		591	/* First calcualte the latency in ns */
		592	u32 mc_latency = 2000; /* 2000 ns. */
		593	u32 available_bandwidth = evergreen_available_bandwidth(wm);
		594	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
		595	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
		596	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
		597	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
		598	(wm->num_heads * cursor_line_pair_return_time);
		599	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
		600	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
		601	fixed20_12 a, b, c;
		602
		603	if (wm->num_heads == 0)
		604	return 0;
		605
		606	a.full = dfixed_const(2);
		607	b.full = dfixed_const(1);
		608	if ((wm->vsc.full > a.full) \|\|
		609	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) \|\|
		610	(wm->vtaps >= 5) \|\|
		611	((wm->vsc.full >= a.full) && wm->interlaced))
		612	max_src_lines_per_dst_line = 4;
		613	else
		614	max_src_lines_per_dst_line = 2;
		615
		616	a.full = dfixed_const(available_bandwidth);
		617	b.full = dfixed_const(wm->num_heads);
		618	a.full = dfixed_div(a, b);
		619
		620	b.full = dfixed_const(1000);
		621	c.full = dfixed_const(wm->disp_clk);
		622	b.full = dfixed_div(c, b);
		623	c.full = dfixed_const(wm->bytes_per_pixel);
		624	b.full = dfixed_mul(b, c);
		625
		626	lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
		627
		628	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
		629	b.full = dfixed_const(1000);
		630	c.full = dfixed_const(lb_fill_bw);
		631	b.full = dfixed_div(c, b);
		632	a.full = dfixed_div(a, b);
		633	line_fill_time = dfixed_trunc(a);
		634
		635	if (line_fill_time < wm->active_time)
		636	return latency;
		637	else
		638	return latency + (line_fill_time - wm->active_time);
		639
		640	}
		641
		642	static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		643	{
		644	if (evergreen_average_bandwidth(wm) <=
		645	(evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
		646	return true;
		647	else
		648	return false;
		649	};
		650
		651	static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
		652	{
		653	if (evergreen_average_bandwidth(wm) <=
		654	(evergreen_available_bandwidth(wm) / wm->num_heads))
		655	return true;
		656	else
		657	return false;
		658	};
		659
		660	static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
		661	{
		662	u32 lb_partitions = wm->lb_size / wm->src_width;
		663	u32 line_time = wm->active_time + wm->blank_time;
		664	u32 latency_tolerant_lines;
		665	u32 latency_hiding;
		666	fixed20_12 a;
		667
		668	a.full = dfixed_const(1);
		669	if (wm->vsc.full > a.full)
		670	latency_tolerant_lines = 1;
		671	else {
		672	if (lb_partitions <= (wm->vtaps + 1))
		673	latency_tolerant_lines = 1;
		674	else
		675	latency_tolerant_lines = 2;
		676	}
		677
		678	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
		679
		680	if (evergreen_latency_watermark(wm) <= latency_hiding)
		681	return true;
		682	else
		683	return false;
		684	}
		685
		686	static void evergreen_program_watermarks(struct radeon_device *rdev,
		687	struct radeon_crtc *radeon_crtc,
		688	u32 lb_size, u32 num_heads)
		689	{
		690	struct drm_display_mode *mode = &radeon_crtc->base.mode;
		691	struct evergreen_wm_params wm;
		692	u32 pixel_period;
		693	u32 line_time = 0;
		694	u32 latency_watermark_a = 0, latency_watermark_b = 0;
		695	u32 priority_a_mark = 0, priority_b_mark = 0;
		696	u32 priority_a_cnt = PRIORITY_OFF;
		697	u32 priority_b_cnt = PRIORITY_OFF;
		698	u32 pipe_offset = radeon_crtc->crtc_id * 16;
		699	u32 tmp, arb_control3;
		700	fixed20_12 a, b, c;
		701
		702	if (radeon_crtc->base.enabled && num_heads && mode) {
		703	pixel_period = 1000000 / (u32)mode->clock;
		704	line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
		705	priority_a_cnt = 0;
		706	priority_b_cnt = 0;
		707
		708	wm.yclk = rdev->pm.current_mclk * 10;
		709	wm.sclk = rdev->pm.current_sclk * 10;
		710	wm.disp_clk = mode->clock;
		711	wm.src_width = mode->crtc_hdisplay;
		712	wm.active_time = mode->crtc_hdisplay * pixel_period;
		713	wm.blank_time = line_time - wm.active_time;
		714	wm.interlaced = false;
		715	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		716	wm.interlaced = true;
		717	wm.vsc = radeon_crtc->vsc;
		718	wm.vtaps = 1;
		719	if (radeon_crtc->rmx_type != RMX_OFF)
		720	wm.vtaps = 2;
		721	wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
		722	wm.lb_size = lb_size;
		723	wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
		724	wm.num_heads = num_heads;
		725
		726	/* set for high clocks */
		727	latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
		728	/* set for low clocks */
		729	/* wm.yclk = low clk; wm.sclk = low clk */
		730	latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
		731
		732	/* possibly force display priority to high */
		733	/* should really do this at mode validation time... */
		734	if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) \|\|
		735	!evergreen_average_bandwidth_vs_available_bandwidth(&wm) \|\|
		736	!evergreen_check_latency_hiding(&wm) \|\|
		737	(rdev->disp_priority == 2)) {
		738	DRM_INFO("force priority to high\n");
		739	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		740	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		741	}
		742
		743	a.full = dfixed_const(1000);
		744	b.full = dfixed_const(mode->clock);
		745	b.full = dfixed_div(b, a);
		746	c.full = dfixed_const(latency_watermark_a);
		747	c.full = dfixed_mul(c, b);
		748	c.full = dfixed_mul(c, radeon_crtc->hsc);
		749	c.full = dfixed_div(c, a);
		750	a.full = dfixed_const(16);
		751	c.full = dfixed_div(c, a);
		752	priority_a_mark = dfixed_trunc(c);
		753	priority_a_cnt \|= priority_a_mark & PRIORITY_MARK_MASK;
		754
		755	a.full = dfixed_const(1000);
		756	b.full = dfixed_const(mode->clock);
		757	b.full = dfixed_div(b, a);
		758	c.full = dfixed_const(latency_watermark_b);
		759	c.full = dfixed_mul(c, b);
		760	c.full = dfixed_mul(c, radeon_crtc->hsc);
		761	c.full = dfixed_div(c, a);
		762	a.full = dfixed_const(16);
		763	c.full = dfixed_div(c, a);
		764	priority_b_mark = dfixed_trunc(c);
		765	priority_b_cnt \|= priority_b_mark & PRIORITY_MARK_MASK;
		766	}
		767
		768	/* select wm A */
		769	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		770	tmp = arb_control3;
		771	tmp &= ~LATENCY_WATERMARK_MASK(3);
		772	tmp \|= LATENCY_WATERMARK_MASK(1);
		773	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		774	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		775	(LATENCY_LOW_WATERMARK(latency_watermark_a) \|
		776	LATENCY_HIGH_WATERMARK(line_time)));
		777	/* select wm B */
		778	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		779	tmp &= ~LATENCY_WATERMARK_MASK(3);
		780	tmp \|= LATENCY_WATERMARK_MASK(2);
		781	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		782	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		783	(LATENCY_LOW_WATERMARK(latency_watermark_b) \|
		784	LATENCY_HIGH_WATERMARK(line_time)));
		785	/* restore original selection */
		786	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
		787
		788	/* write the priority marks */
		789	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
		790	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
		791
		792	}
		793
1963	serge	794	void evergreen_bandwidth_update(struct radeon_device *rdev)
1430	serge	795	{
1986	serge	796	struct drm_display_mode *mode0 = NULL;
		797	struct drm_display_mode *mode1 = NULL;
		798	u32 num_heads = 0, lb_size;
		799	int i;
		800
		801	radeon_update_display_priority(rdev);
		802
		803	for (i = 0; i < rdev->num_crtc; i++) {
		804	if (rdev->mode_info.crtcs[i]->base.enabled)
		805	num_heads++;
		806	}
		807	for (i = 0; i < rdev->num_crtc; i += 2) {
		808	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
		809	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
		810	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
		811	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
		812	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
		813	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
		814	}
1430	serge	815	}
		816
1963	serge	817	int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1430	serge	818	{
		819	unsigned i;
		820	u32 tmp;
		821
		822	for (i = 0; i < rdev->usec_timeout; i++) {
		823	/* read MC_STATUS */
		824	tmp = RREG32(SRBM_STATUS) & 0x1F00;
		825	if (!tmp)
		826	return 0;
		827	udelay(1);
		828	}
		829	return -1;
		830	}
		831
		832	/*
		833	* GART
		834	*/
1963	serge	835	void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
		836	{
		837	unsigned i;
		838	u32 tmp;
		839
		840	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		841
		842	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		843	for (i = 0; i < rdev->usec_timeout; i++) {
		844	/* read MC_STATUS */
		845	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		846	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		847	if (tmp == 2) {
		848	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		849	return;
		850	}
		851	if (tmp) {
		852	return;
		853	}
		854	udelay(1);
		855	}
		856	}
		857
1430	serge	858	int evergreen_pcie_gart_enable(struct radeon_device *rdev)
		859	{
		860	u32 tmp;
1963	serge	861	int r;
1430	serge	862
		863	if (rdev->gart.table.vram.robj == NULL) {
		864	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		865	return -EINVAL;
		866	}
		867	r = radeon_gart_table_vram_pin(rdev);
		868	if (r)
		869	return r;
		870	radeon_gart_restore(rdev);
		871	/* Setup L2 cache */
		872	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		873	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		874	EFFECTIVE_L2_QUEUE_SIZE(7));
		875	WREG32(VM_L2_CNTL2, 0);
		876	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		877	/* Setup TLB control */
		878	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		879	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		880	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		881	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
1963	serge	882	if (rdev->flags & RADEON_IS_IGP) {
		883	WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
		884	WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
		885	WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
		886	} else {
1430	serge	887	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		888	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		889	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1963	serge	890	}
1430	serge	891	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		892	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		893	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		894	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
		895	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		896	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		897	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		898	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		899	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		900	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		901	(u32)(rdev->dummy_page.addr >> 12));
1963	serge	902	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	903
1963	serge	904	evergreen_pcie_gart_tlb_flush(rdev);
1430	serge	905	rdev->gart.ready = true;
		906	return 0;
		907	}
		908
		909	void evergreen_pcie_gart_disable(struct radeon_device *rdev)
		910	{
		911	u32 tmp;
1963	serge	912	int r;
1430	serge	913
		914	/* Disable all tables */
1963	serge	915	WREG32(VM_CONTEXT0_CNTL, 0);
		916	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	917
		918	/* Setup L2 cache */
		919	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		920	EFFECTIVE_L2_QUEUE_SIZE(7));
		921	WREG32(VM_L2_CNTL2, 0);
		922	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		923	/* Setup TLB control */
		924	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		925	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		926	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		927	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		928	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		929	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		930	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		931	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
		932	if (rdev->gart.table.vram.robj) {
		933	r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
		934	if (likely(r == 0)) {
		935	radeon_bo_kunmap(rdev->gart.table.vram.robj);
		936	radeon_bo_unpin(rdev->gart.table.vram.robj);
		937	radeon_bo_unreserve(rdev->gart.table.vram.robj);
		938	}
		939	}
		940	}
		941
		942	void evergreen_pcie_gart_fini(struct radeon_device *rdev)
		943	{
		944	evergreen_pcie_gart_disable(rdev);
		945	radeon_gart_table_vram_free(rdev);
		946	radeon_gart_fini(rdev);
		947	}
		948
		949
		950	void evergreen_agp_enable(struct radeon_device *rdev)
		951	{
		952	u32 tmp;
		953
		954	/* Setup L2 cache */
		955	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		956	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		957	EFFECTIVE_L2_QUEUE_SIZE(7));
		958	WREG32(VM_L2_CNTL2, 0);
		959	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		960	/* Setup TLB control */
		961	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		962	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		963	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		964	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		965	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		966	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		967	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		968	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		969	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		970	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		971	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1963	serge	972	WREG32(VM_CONTEXT0_CNTL, 0);
		973	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	974	}
		975
1963	serge	976	void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	977	{
		978	save->vga_control[0] = RREG32(D1VGA_CONTROL);
		979	save->vga_control[1] = RREG32(D2VGA_CONTROL);
		980	save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL);
		981	save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL);
		982	save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL);
		983	save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL);
		984	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
		985	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
		986	save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
		987	save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
1963	serge	988	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	989	save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
		990	save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
		991	save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
		992	save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
1963	serge	993	}
1430	serge	994
		995	/* Stop all video */
		996	WREG32(VGA_RENDER_CONTROL, 0);
		997	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
		998	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1963	serge	999	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1000	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
		1001	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
		1002	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
		1003	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1963	serge	1004	}
1430	serge	1005	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		1006	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1963	serge	1007	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1008	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		1009	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		1010	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		1011	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	1012	}
1430	serge	1013	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		1014	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1963	serge	1015	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1016	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		1017	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		1018	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		1019	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	1020	}
1430	serge	1021
		1022	WREG32(D1VGA_CONTROL, 0);
		1023	WREG32(D2VGA_CONTROL, 0);
		1024	WREG32(EVERGREEN_D3VGA_CONTROL, 0);
		1025	WREG32(EVERGREEN_D4VGA_CONTROL, 0);
		1026	WREG32(EVERGREEN_D5VGA_CONTROL, 0);
		1027	WREG32(EVERGREEN_D6VGA_CONTROL, 0);
		1028	}
		1029
1963	serge	1030	void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	1031	{
		1032	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
		1033	upper_32_bits(rdev->mc.vram_start));
		1034	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
		1035	upper_32_bits(rdev->mc.vram_start));
		1036	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
		1037	(u32)rdev->mc.vram_start);
		1038	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
		1039	(u32)rdev->mc.vram_start);
		1040
		1041	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
		1042	upper_32_bits(rdev->mc.vram_start));
		1043	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
		1044	upper_32_bits(rdev->mc.vram_start));
		1045	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
		1046	(u32)rdev->mc.vram_start);
		1047	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
		1048	(u32)rdev->mc.vram_start);
		1049
1963	serge	1050	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1051	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
		1052	upper_32_bits(rdev->mc.vram_start));
		1053	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
		1054	upper_32_bits(rdev->mc.vram_start));
		1055	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
		1056	(u32)rdev->mc.vram_start);
		1057	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
		1058	(u32)rdev->mc.vram_start);
		1059
		1060	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
		1061	upper_32_bits(rdev->mc.vram_start));
		1062	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
		1063	upper_32_bits(rdev->mc.vram_start));
		1064	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
		1065	(u32)rdev->mc.vram_start);
		1066	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
		1067	(u32)rdev->mc.vram_start);
		1068
		1069	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
		1070	upper_32_bits(rdev->mc.vram_start));
		1071	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
		1072	upper_32_bits(rdev->mc.vram_start));
		1073	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
		1074	(u32)rdev->mc.vram_start);
		1075	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
		1076	(u32)rdev->mc.vram_start);
		1077
		1078	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
		1079	upper_32_bits(rdev->mc.vram_start));
		1080	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
		1081	upper_32_bits(rdev->mc.vram_start));
		1082	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
		1083	(u32)rdev->mc.vram_start);
		1084	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
		1085	(u32)rdev->mc.vram_start);
1963	serge	1086	}
1430	serge	1087
		1088	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
		1089	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
		1090	/* Unlock host access */
		1091	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
		1092	mdelay(1);
		1093	/* Restore video state */
		1094	WREG32(D1VGA_CONTROL, save->vga_control[0]);
		1095	WREG32(D2VGA_CONTROL, save->vga_control[1]);
		1096	WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]);
		1097	WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]);
		1098	WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]);
		1099	WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
		1100	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
		1101	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1963	serge	1102	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1103	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
		1104	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
		1105	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
		1106	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1963	serge	1107	}
1430	serge	1108	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]);
		1109	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]);
1963	serge	1110	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1111	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
		1112	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
		1113	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
		1114	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
1963	serge	1115	}
1430	serge	1116	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		1117	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1963	serge	1118	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	1119	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		1120	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		1121	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		1122	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	1123	}
1430	serge	1124	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
		1125	}
		1126
1963	serge	1127	void evergreen_mc_program(struct radeon_device *rdev)
1430	serge	1128	{
		1129	struct evergreen_mc_save save;
		1130	u32 tmp;
		1131	int i, j;
		1132
		1133	/* Initialize HDP */
		1134	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1135	WREG32((0x2c14 + j), 0x00000000);
		1136	WREG32((0x2c18 + j), 0x00000000);
		1137	WREG32((0x2c1c + j), 0x00000000);
		1138	WREG32((0x2c20 + j), 0x00000000);
		1139	WREG32((0x2c24 + j), 0x00000000);
		1140	}
		1141	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		1142
		1143	evergreen_mc_stop(rdev, &save);
		1144	if (evergreen_mc_wait_for_idle(rdev)) {
		1145	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1146	}
		1147	/* Lockout access through VGA aperture*/
		1148	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		1149	/* Update configuration */
		1150	if (rdev->flags & RADEON_IS_AGP) {
		1151	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		1152	/* VRAM before AGP */
		1153	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1154	rdev->mc.vram_start >> 12);
		1155	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1156	rdev->mc.gtt_end >> 12);
		1157	} else {
		1158	/* VRAM after AGP */
		1159	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1160	rdev->mc.gtt_start >> 12);
		1161	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1162	rdev->mc.vram_end >> 12);
		1163	}
		1164	} else {
		1165	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1166	rdev->mc.vram_start >> 12);
		1167	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1168	rdev->mc.vram_end >> 12);
		1169	}
		1170	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
1963	serge	1171	if (rdev->flags & RADEON_IS_IGP) {
		1172	tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
		1173	tmp \|= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
		1174	tmp \|= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
		1175	WREG32(MC_FUS_VM_FB_OFFSET, tmp);
		1176	}
1430	serge	1177	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		1178	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		1179	WREG32(MC_VM_FB_LOCATION, tmp);
		1180	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1963	serge	1181	WREG32(HDP_NONSURFACE_INFO, (2 << 7) \| (1 << 30));
		1182	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1430	serge	1183	if (rdev->flags & RADEON_IS_AGP) {
		1184	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
		1185	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
		1186	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		1187	} else {
		1188	WREG32(MC_VM_AGP_BASE, 0);
		1189	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		1190	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		1191	}
		1192	if (evergreen_mc_wait_for_idle(rdev)) {
		1193	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1194	}
		1195	evergreen_mc_resume(rdev, &save);
		1196	/* we need to own VRAM, so turn off the VGA renderer here
		1197	* to stop it overwriting our objects */
		1198	rv515_vga_render_disable(rdev);
		1199	}
		1200
		1201	/*
		1202	* CP.
		1203	*/
1986	serge	1204	void evergreen_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		1205	{
		1206	/* set to DX10/11 mode */
		1207	radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
		1208	radeon_ring_write(rdev, 1);
		1209	/* FIXME: implement */
		1210	radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		1211	radeon_ring_write(rdev,
		1212	#ifdef __BIG_ENDIAN
		1213	(2 << 0) \|
		1214	#endif
		1215	(ib->gpu_addr & 0xFFFFFFFC));
		1216	radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
		1217	radeon_ring_write(rdev, ib->length_dw);
		1218	}
1963	serge	1219
1986	serge	1220
1963	serge	1221	static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1430	serge	1222	{
1963	serge	1223	const __be32 *fw_data;
		1224	int i;
		1225
		1226	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1227	return -EINVAL;
		1228
		1229	r700_cp_stop(rdev);
		1230	WREG32(CP_RB_CNTL,
		1231	#ifdef __BIG_ENDIAN
		1232	BUF_SWAP_32BIT \|
		1233	#endif
		1234	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
		1235
		1236	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1237	WREG32(CP_PFP_UCODE_ADDR, 0);
		1238	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
		1239	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		1240	WREG32(CP_PFP_UCODE_ADDR, 0);
		1241
		1242	fw_data = (const __be32 *)rdev->me_fw->data;
		1243	WREG32(CP_ME_RAM_WADDR, 0);
		1244	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
		1245	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		1246
		1247	WREG32(CP_PFP_UCODE_ADDR, 0);
		1248	WREG32(CP_ME_RAM_WADDR, 0);
		1249	WREG32(CP_ME_RAM_RADDR, 0);
		1250	return 0;
1430	serge	1251	}
		1252
1963	serge	1253	static int evergreen_cp_start(struct radeon_device *rdev)
		1254	{
		1255	int r, i;
		1256	uint32_t cp_me;
1430	serge	1257
1963	serge	1258	r = radeon_ring_lock(rdev, 7);
		1259	if (r) {
		1260	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1261	return r;
		1262	}
		1263	radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1264	radeon_ring_write(rdev, 0x1);
		1265	radeon_ring_write(rdev, 0x0);
		1266	radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
		1267	radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1268	radeon_ring_write(rdev, 0);
		1269	radeon_ring_write(rdev, 0);
		1270	radeon_ring_unlock_commit(rdev);
		1271
		1272	cp_me = 0xff;
		1273	WREG32(CP_ME_CNTL, cp_me);
		1274
		1275	r = radeon_ring_lock(rdev, evergreen_default_size + 19);
		1276	if (r) {
		1277	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1278	return r;
		1279	}
		1280
		1281	/* setup clear context state */
		1282	radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1283	radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
		1284
		1285	for (i = 0; i < evergreen_default_size; i++)
		1286	radeon_ring_write(rdev, evergreen_default_state[i]);
		1287
		1288	radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1289	radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
		1290
		1291	/* set clear context state */
		1292	radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
		1293	radeon_ring_write(rdev, 0);
		1294
		1295	/* SQ_VTX_BASE_VTX_LOC */
		1296	radeon_ring_write(rdev, 0xc0026f00);
		1297	radeon_ring_write(rdev, 0x00000000);
		1298	radeon_ring_write(rdev, 0x00000000);
		1299	radeon_ring_write(rdev, 0x00000000);
		1300
		1301	/* Clear consts */
		1302	radeon_ring_write(rdev, 0xc0036f00);
		1303	radeon_ring_write(rdev, 0x00000bc4);
		1304	radeon_ring_write(rdev, 0xffffffff);
		1305	radeon_ring_write(rdev, 0xffffffff);
		1306	radeon_ring_write(rdev, 0xffffffff);
		1307
		1308	radeon_ring_write(rdev, 0xc0026900);
		1309	radeon_ring_write(rdev, 0x00000316);
		1310	radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		1311	radeon_ring_write(rdev, 0x00000010); /* */
		1312
		1313	radeon_ring_unlock_commit(rdev);
		1314
		1315	return 0;
		1316	}
		1317
		1318	int evergreen_cp_resume(struct radeon_device *rdev)
1430	serge	1319	{
1963	serge	1320	u32 tmp;
		1321	u32 rb_bufsz;
		1322	int r;
1430	serge	1323
1963	serge	1324	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
		1325	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP \|
		1326	SOFT_RESET_PA \|
		1327	SOFT_RESET_SH \|
		1328	SOFT_RESET_VGT \|
		1329	SOFT_RESET_SX));
		1330	RREG32(GRBM_SOFT_RESET);
		1331	mdelay(15);
		1332	WREG32(GRBM_SOFT_RESET, 0);
		1333	RREG32(GRBM_SOFT_RESET);
		1334
		1335	/* Set ring buffer size */
		1336	rb_bufsz = drm_order(rdev->cp.ring_size / 8);
		1337	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
		1338	#ifdef __BIG_ENDIAN
		1339	tmp \|= BUF_SWAP_32BIT;
		1340	#endif
		1341	WREG32(CP_RB_CNTL, tmp);
		1342	WREG32(CP_SEM_WAIT_TIMER, 0x4);
		1343
		1344	/* Set the write pointer delay */
		1345	WREG32(CP_RB_WPTR_DELAY, 0);
		1346
		1347	/* Initialize the ring buffer's read and write pointers */
		1348	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1349	WREG32(CP_RB_RPTR_WR, 0);
		1350	WREG32(CP_RB_WPTR, 0);
		1351
		1352	/* set the wb address wether it's enabled or not */
		1353	WREG32(CP_RB_RPTR_ADDR,
		1354	#ifdef __BIG_ENDIAN
		1355	RB_RPTR_SWAP(2) \|
		1356	#endif
		1357	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1358	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1359	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1360
		1361	if (rdev->wb.enabled)
		1362	WREG32(SCRATCH_UMSK, 0xff);
		1363	else {
		1364	tmp \|= RB_NO_UPDATE;
		1365	WREG32(SCRATCH_UMSK, 0);
		1366	}
		1367
		1368	mdelay(1);
		1369	WREG32(CP_RB_CNTL, tmp);
		1370
		1371	WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
		1372	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1373
		1374	rdev->cp.rptr = RREG32(CP_RB_RPTR);
		1375	rdev->cp.wptr = RREG32(CP_RB_WPTR);
		1376
		1377	evergreen_cp_start(rdev);
		1378	rdev->cp.ready = true;
		1379	r = radeon_ring_test(rdev);
		1380	if (r) {
		1381	rdev->cp.ready = false;
		1382	return r;
		1383	}
1430	serge	1384	return 0;
		1385	}
		1386
		1387	/*
		1388	* Core functions
		1389	*/
1963	serge	1390	static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
		1391	u32 num_tile_pipes,
1430	serge	1392	u32 num_backends,
		1393	u32 backend_disable_mask)
		1394	{
		1395	u32 backend_map = 0;
1963	serge	1396	u32 enabled_backends_mask = 0;
		1397	u32 enabled_backends_count = 0;
		1398	u32 cur_pipe;
		1399	u32 swizzle_pipe[EVERGREEN_MAX_PIPES];
		1400	u32 cur_backend = 0;
		1401	u32 i;
		1402	bool force_no_swizzle;
1430	serge	1403
1963	serge	1404	if (num_tile_pipes > EVERGREEN_MAX_PIPES)
		1405	num_tile_pipes = EVERGREEN_MAX_PIPES;
		1406	if (num_tile_pipes < 1)
		1407	num_tile_pipes = 1;
		1408	if (num_backends > EVERGREEN_MAX_BACKENDS)
		1409	num_backends = EVERGREEN_MAX_BACKENDS;
		1410	if (num_backends < 1)
		1411	num_backends = 1;
		1412
		1413	for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
		1414	if (((backend_disable_mask >> i) & 1) == 0) {
		1415	enabled_backends_mask \|= (1 << i);
		1416	++enabled_backends_count;
		1417	}
		1418	if (enabled_backends_count == num_backends)
		1419	break;
		1420	}
		1421
		1422	if (enabled_backends_count == 0) {
		1423	enabled_backends_mask = 1;
		1424	enabled_backends_count = 1;
		1425	}
		1426
		1427	if (enabled_backends_count != num_backends)
		1428	num_backends = enabled_backends_count;
		1429
		1430	memset((uint8_t )&swizzle_pipe[0], 0, sizeof(u32) EVERGREEN_MAX_PIPES);
		1431	switch (rdev->family) {
		1432	case CHIP_CEDAR:
		1433	case CHIP_REDWOOD:
		1434	case CHIP_PALM:
		1435	case CHIP_SUMO:
		1436	case CHIP_SUMO2:
		1437	case CHIP_TURKS:
		1438	case CHIP_CAICOS:
		1439	force_no_swizzle = false;
		1440	break;
		1441	case CHIP_CYPRESS:
		1442	case CHIP_HEMLOCK:
		1443	case CHIP_JUNIPER:
		1444	case CHIP_BARTS:
		1445	default:
		1446	force_no_swizzle = true;
		1447	break;
		1448	}
		1449	if (force_no_swizzle) {
		1450	bool last_backend_enabled = false;
		1451
		1452	force_no_swizzle = false;
		1453	for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
		1454	if (((enabled_backends_mask >> i) & 1) == 1) {
		1455	if (last_backend_enabled)
		1456	force_no_swizzle = true;
		1457	last_backend_enabled = true;
		1458	} else
		1459	last_backend_enabled = false;
		1460	}
		1461	}
		1462
		1463	switch (num_tile_pipes) {
		1464	case 1:
		1465	case 3:
		1466	case 5:
		1467	case 7:
		1468	DRM_ERROR("odd number of pipes!\n");
		1469	break;
		1470	case 2:
		1471	swizzle_pipe[0] = 0;
		1472	swizzle_pipe[1] = 1;
		1473	break;
		1474	case 4:
		1475	if (force_no_swizzle) {
		1476	swizzle_pipe[0] = 0;
		1477	swizzle_pipe[1] = 1;
		1478	swizzle_pipe[2] = 2;
		1479	swizzle_pipe[3] = 3;
		1480	} else {
		1481	swizzle_pipe[0] = 0;
		1482	swizzle_pipe[1] = 2;
		1483	swizzle_pipe[2] = 1;
		1484	swizzle_pipe[3] = 3;
		1485	}
		1486	break;
		1487	case 6:
		1488	if (force_no_swizzle) {
		1489	swizzle_pipe[0] = 0;
		1490	swizzle_pipe[1] = 1;
		1491	swizzle_pipe[2] = 2;
		1492	swizzle_pipe[3] = 3;
		1493	swizzle_pipe[4] = 4;
		1494	swizzle_pipe[5] = 5;
		1495	} else {
		1496	swizzle_pipe[0] = 0;
		1497	swizzle_pipe[1] = 2;
		1498	swizzle_pipe[2] = 4;
		1499	swizzle_pipe[3] = 1;
		1500	swizzle_pipe[4] = 3;
		1501	swizzle_pipe[5] = 5;
		1502	}
		1503	break;
		1504	case 8:
		1505	if (force_no_swizzle) {
		1506	swizzle_pipe[0] = 0;
		1507	swizzle_pipe[1] = 1;
		1508	swizzle_pipe[2] = 2;
		1509	swizzle_pipe[3] = 3;
		1510	swizzle_pipe[4] = 4;
		1511	swizzle_pipe[5] = 5;
		1512	swizzle_pipe[6] = 6;
		1513	swizzle_pipe[7] = 7;
		1514	} else {
		1515	swizzle_pipe[0] = 0;
		1516	swizzle_pipe[1] = 2;
		1517	swizzle_pipe[2] = 4;
		1518	swizzle_pipe[3] = 6;
		1519	swizzle_pipe[4] = 1;
		1520	swizzle_pipe[5] = 3;
		1521	swizzle_pipe[6] = 5;
		1522	swizzle_pipe[7] = 7;
		1523	}
		1524	break;
		1525	}
		1526
		1527	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
		1528	while (((1 << cur_backend) & enabled_backends_mask) == 0)
		1529	cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
		1530
		1531	backend_map \|= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4)));
		1532
		1533	cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
		1534	}
		1535
1430	serge	1536	return backend_map;
		1537	}
		1538
1986	serge	1539	static void evergreen_program_channel_remap(struct radeon_device *rdev)
		1540	{
		1541	u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp;
		1542
		1543	tmp = RREG32(MC_SHARED_CHMAP);
		1544	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		1545	case 0:
		1546	case 1:
		1547	case 2:
		1548	case 3:
		1549	default:
		1550	/* default mapping */
		1551	mc_shared_chremap = 0x00fac688;
		1552	break;
		1553	}
		1554
		1555	switch (rdev->family) {
		1556	case CHIP_HEMLOCK:
		1557	case CHIP_CYPRESS:
		1558	case CHIP_BARTS:
		1559	tcp_chan_steer_lo = 0x54763210;
		1560	tcp_chan_steer_hi = 0x0000ba98;
		1561	break;
		1562	case CHIP_JUNIPER:
		1563	case CHIP_REDWOOD:
		1564	case CHIP_CEDAR:
		1565	case CHIP_PALM:
		1566	case CHIP_SUMO:
		1567	case CHIP_SUMO2:
		1568	case CHIP_TURKS:
		1569	case CHIP_CAICOS:
		1570	default:
		1571	tcp_chan_steer_lo = 0x76543210;
		1572	tcp_chan_steer_hi = 0x0000ba98;
		1573	break;
		1574	}
		1575
		1576	WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo);
		1577	WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi);
		1578	WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
		1579	}
		1580
1430	serge	1581	static void evergreen_gpu_init(struct radeon_device *rdev)
		1582	{
1963	serge	1583	u32 cc_rb_backend_disable = 0;
		1584	u32 cc_gc_shader_pipe_config;
		1585	u32 gb_addr_config = 0;
		1586	u32 mc_shared_chmap, mc_arb_ramcfg;
		1587	u32 gb_backend_map;
		1588	u32 grbm_gfx_index;
		1589	u32 sx_debug_1;
		1590	u32 smx_dc_ctl0;
		1591	u32 sq_config;
		1592	u32 sq_lds_resource_mgmt;
		1593	u32 sq_gpr_resource_mgmt_1;
		1594	u32 sq_gpr_resource_mgmt_2;
		1595	u32 sq_gpr_resource_mgmt_3;
		1596	u32 sq_thread_resource_mgmt;
		1597	u32 sq_thread_resource_mgmt_2;
		1598	u32 sq_stack_resource_mgmt_1;
		1599	u32 sq_stack_resource_mgmt_2;
		1600	u32 sq_stack_resource_mgmt_3;
		1601	u32 vgt_cache_invalidation;
		1602	u32 hdp_host_path_cntl, tmp;
		1603	int i, j, num_shader_engines, ps_thread_count;
		1604
		1605	switch (rdev->family) {
		1606	case CHIP_CYPRESS:
		1607	case CHIP_HEMLOCK:
		1608	rdev->config.evergreen.num_ses = 2;
		1609	rdev->config.evergreen.max_pipes = 4;
		1610	rdev->config.evergreen.max_tile_pipes = 8;
		1611	rdev->config.evergreen.max_simds = 10;
		1612	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1613	rdev->config.evergreen.max_gprs = 256;
		1614	rdev->config.evergreen.max_threads = 248;
		1615	rdev->config.evergreen.max_gs_threads = 32;
		1616	rdev->config.evergreen.max_stack_entries = 512;
		1617	rdev->config.evergreen.sx_num_of_sets = 4;
		1618	rdev->config.evergreen.sx_max_export_size = 256;
		1619	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1620	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1621	rdev->config.evergreen.max_hw_contexts = 8;
		1622	rdev->config.evergreen.sq_num_cf_insts = 2;
		1623
		1624	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1625	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1626	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1627	break;
		1628	case CHIP_JUNIPER:
		1629	rdev->config.evergreen.num_ses = 1;
		1630	rdev->config.evergreen.max_pipes = 4;
		1631	rdev->config.evergreen.max_tile_pipes = 4;
		1632	rdev->config.evergreen.max_simds = 10;
		1633	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1634	rdev->config.evergreen.max_gprs = 256;
		1635	rdev->config.evergreen.max_threads = 248;
		1636	rdev->config.evergreen.max_gs_threads = 32;
		1637	rdev->config.evergreen.max_stack_entries = 512;
		1638	rdev->config.evergreen.sx_num_of_sets = 4;
		1639	rdev->config.evergreen.sx_max_export_size = 256;
		1640	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1641	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1642	rdev->config.evergreen.max_hw_contexts = 8;
		1643	rdev->config.evergreen.sq_num_cf_insts = 2;
		1644
		1645	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1646	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1647	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1648	break;
		1649	case CHIP_REDWOOD:
		1650	rdev->config.evergreen.num_ses = 1;
		1651	rdev->config.evergreen.max_pipes = 4;
		1652	rdev->config.evergreen.max_tile_pipes = 4;
		1653	rdev->config.evergreen.max_simds = 5;
		1654	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1655	rdev->config.evergreen.max_gprs = 256;
		1656	rdev->config.evergreen.max_threads = 248;
		1657	rdev->config.evergreen.max_gs_threads = 32;
		1658	rdev->config.evergreen.max_stack_entries = 256;
		1659	rdev->config.evergreen.sx_num_of_sets = 4;
		1660	rdev->config.evergreen.sx_max_export_size = 256;
		1661	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1662	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1663	rdev->config.evergreen.max_hw_contexts = 8;
		1664	rdev->config.evergreen.sq_num_cf_insts = 2;
		1665
		1666	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1667	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1668	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1669	break;
		1670	case CHIP_CEDAR:
		1671	default:
		1672	rdev->config.evergreen.num_ses = 1;
		1673	rdev->config.evergreen.max_pipes = 2;
		1674	rdev->config.evergreen.max_tile_pipes = 2;
		1675	rdev->config.evergreen.max_simds = 2;
		1676	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1677	rdev->config.evergreen.max_gprs = 256;
		1678	rdev->config.evergreen.max_threads = 192;
		1679	rdev->config.evergreen.max_gs_threads = 16;
		1680	rdev->config.evergreen.max_stack_entries = 256;
		1681	rdev->config.evergreen.sx_num_of_sets = 4;
		1682	rdev->config.evergreen.sx_max_export_size = 128;
		1683	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1684	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1685	rdev->config.evergreen.max_hw_contexts = 4;
		1686	rdev->config.evergreen.sq_num_cf_insts = 1;
		1687
		1688	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1689	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1690	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1691	break;
		1692	case CHIP_PALM:
		1693	rdev->config.evergreen.num_ses = 1;
		1694	rdev->config.evergreen.max_pipes = 2;
		1695	rdev->config.evergreen.max_tile_pipes = 2;
		1696	rdev->config.evergreen.max_simds = 2;
		1697	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1698	rdev->config.evergreen.max_gprs = 256;
		1699	rdev->config.evergreen.max_threads = 192;
		1700	rdev->config.evergreen.max_gs_threads = 16;
		1701	rdev->config.evergreen.max_stack_entries = 256;
		1702	rdev->config.evergreen.sx_num_of_sets = 4;
		1703	rdev->config.evergreen.sx_max_export_size = 128;
		1704	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1705	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1706	rdev->config.evergreen.max_hw_contexts = 4;
		1707	rdev->config.evergreen.sq_num_cf_insts = 1;
		1708
		1709	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1710	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1711	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1712	break;
		1713	case CHIP_SUMO:
		1714	rdev->config.evergreen.num_ses = 1;
		1715	rdev->config.evergreen.max_pipes = 4;
		1716	rdev->config.evergreen.max_tile_pipes = 2;
		1717	if (rdev->pdev->device == 0x9648)
		1718	rdev->config.evergreen.max_simds = 3;
		1719	else if ((rdev->pdev->device == 0x9647) \|\|
		1720	(rdev->pdev->device == 0x964a))
		1721	rdev->config.evergreen.max_simds = 4;
		1722	else
		1723	rdev->config.evergreen.max_simds = 5;
		1724	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1725	rdev->config.evergreen.max_gprs = 256;
		1726	rdev->config.evergreen.max_threads = 248;
		1727	rdev->config.evergreen.max_gs_threads = 32;
		1728	rdev->config.evergreen.max_stack_entries = 256;
		1729	rdev->config.evergreen.sx_num_of_sets = 4;
		1730	rdev->config.evergreen.sx_max_export_size = 256;
		1731	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1732	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1733	rdev->config.evergreen.max_hw_contexts = 8;
		1734	rdev->config.evergreen.sq_num_cf_insts = 2;
		1735
		1736	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1737	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1738	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1739	break;
		1740	case CHIP_SUMO2:
		1741	rdev->config.evergreen.num_ses = 1;
		1742	rdev->config.evergreen.max_pipes = 4;
		1743	rdev->config.evergreen.max_tile_pipes = 4;
		1744	rdev->config.evergreen.max_simds = 2;
		1745	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1746	rdev->config.evergreen.max_gprs = 256;
		1747	rdev->config.evergreen.max_threads = 248;
		1748	rdev->config.evergreen.max_gs_threads = 32;
		1749	rdev->config.evergreen.max_stack_entries = 512;
		1750	rdev->config.evergreen.sx_num_of_sets = 4;
		1751	rdev->config.evergreen.sx_max_export_size = 256;
		1752	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1753	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1754	rdev->config.evergreen.max_hw_contexts = 8;
		1755	rdev->config.evergreen.sq_num_cf_insts = 2;
		1756
		1757	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1758	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1759	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1760	break;
		1761	case CHIP_BARTS:
		1762	rdev->config.evergreen.num_ses = 2;
		1763	rdev->config.evergreen.max_pipes = 4;
		1764	rdev->config.evergreen.max_tile_pipes = 8;
		1765	rdev->config.evergreen.max_simds = 7;
		1766	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1767	rdev->config.evergreen.max_gprs = 256;
		1768	rdev->config.evergreen.max_threads = 248;
		1769	rdev->config.evergreen.max_gs_threads = 32;
		1770	rdev->config.evergreen.max_stack_entries = 512;
		1771	rdev->config.evergreen.sx_num_of_sets = 4;
		1772	rdev->config.evergreen.sx_max_export_size = 256;
		1773	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1774	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1775	rdev->config.evergreen.max_hw_contexts = 8;
		1776	rdev->config.evergreen.sq_num_cf_insts = 2;
		1777
		1778	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1779	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1780	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1781	break;
		1782	case CHIP_TURKS:
		1783	rdev->config.evergreen.num_ses = 1;
		1784	rdev->config.evergreen.max_pipes = 4;
		1785	rdev->config.evergreen.max_tile_pipes = 4;
		1786	rdev->config.evergreen.max_simds = 6;
		1787	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1788	rdev->config.evergreen.max_gprs = 256;
		1789	rdev->config.evergreen.max_threads = 248;
		1790	rdev->config.evergreen.max_gs_threads = 32;
		1791	rdev->config.evergreen.max_stack_entries = 256;
		1792	rdev->config.evergreen.sx_num_of_sets = 4;
		1793	rdev->config.evergreen.sx_max_export_size = 256;
		1794	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1795	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1796	rdev->config.evergreen.max_hw_contexts = 8;
		1797	rdev->config.evergreen.sq_num_cf_insts = 2;
		1798
		1799	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1800	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1801	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1802	break;
		1803	case CHIP_CAICOS:
		1804	rdev->config.evergreen.num_ses = 1;
		1805	rdev->config.evergreen.max_pipes = 4;
		1806	rdev->config.evergreen.max_tile_pipes = 2;
		1807	rdev->config.evergreen.max_simds = 2;
		1808	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1809	rdev->config.evergreen.max_gprs = 256;
		1810	rdev->config.evergreen.max_threads = 192;
		1811	rdev->config.evergreen.max_gs_threads = 16;
		1812	rdev->config.evergreen.max_stack_entries = 256;
		1813	rdev->config.evergreen.sx_num_of_sets = 4;
		1814	rdev->config.evergreen.sx_max_export_size = 128;
		1815	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1816	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1817	rdev->config.evergreen.max_hw_contexts = 4;
		1818	rdev->config.evergreen.sq_num_cf_insts = 1;
		1819
		1820	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1821	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1822	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1823	break;
		1824	}
		1825
		1826	/* Initialize HDP */
		1827	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1828	WREG32((0x2c14 + j), 0x00000000);
		1829	WREG32((0x2c18 + j), 0x00000000);
		1830	WREG32((0x2c1c + j), 0x00000000);
		1831	WREG32((0x2c20 + j), 0x00000000);
		1832	WREG32((0x2c24 + j), 0x00000000);
		1833	}
		1834
		1835	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1836
		1837	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & ~2;
		1838
		1839	cc_gc_shader_pipe_config \|=
		1840	INACTIVE_QD_PIPES((EVERGREEN_MAX_PIPES_MASK << rdev->config.evergreen.max_pipes)
		1841	& EVERGREEN_MAX_PIPES_MASK);
		1842	cc_gc_shader_pipe_config \|=
		1843	INACTIVE_SIMDS((EVERGREEN_MAX_SIMDS_MASK << rdev->config.evergreen.max_simds)
		1844	& EVERGREEN_MAX_SIMDS_MASK);
		1845
		1846	cc_rb_backend_disable =
		1847	BACKEND_DISABLE((EVERGREEN_MAX_BACKENDS_MASK << rdev->config.evergreen.max_backends)
		1848	& EVERGREEN_MAX_BACKENDS_MASK);
		1849
		1850
		1851	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
		1852	if (rdev->flags & RADEON_IS_IGP)
		1853	mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
		1854	else
		1855	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
		1856
		1857	switch (rdev->config.evergreen.max_tile_pipes) {
		1858	case 1:
		1859	default:
		1860	gb_addr_config \|= NUM_PIPES(0);
		1861	break;
		1862	case 2:
		1863	gb_addr_config \|= NUM_PIPES(1);
		1864	break;
		1865	case 4:
		1866	gb_addr_config \|= NUM_PIPES(2);
		1867	break;
		1868	case 8:
		1869	gb_addr_config \|= NUM_PIPES(3);
		1870	break;
		1871	}
		1872
		1873	gb_addr_config \|= PIPE_INTERLEAVE_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
		1874	gb_addr_config \|= BANK_INTERLEAVE_SIZE(0);
		1875	gb_addr_config \|= NUM_SHADER_ENGINES(rdev->config.evergreen.num_ses - 1);
		1876	gb_addr_config \|= SHADER_ENGINE_TILE_SIZE(1);
		1877	gb_addr_config \|= NUM_GPUS(0); /* Hemlock? */
		1878	gb_addr_config \|= MULTI_GPU_TILE_SIZE(2);
		1879
		1880	if (((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) > 2)
		1881	gb_addr_config \|= ROW_SIZE(2);
		1882	else
		1883	gb_addr_config \|= ROW_SIZE((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT);
		1884
		1885	if (rdev->ddev->pdev->device == 0x689e) {
		1886	u32 efuse_straps_4;
		1887	u32 efuse_straps_3;
		1888	u8 efuse_box_bit_131_124;
		1889
		1890	WREG32(RCU_IND_INDEX, 0x204);
		1891	efuse_straps_4 = RREG32(RCU_IND_DATA);
		1892	WREG32(RCU_IND_INDEX, 0x203);
		1893	efuse_straps_3 = RREG32(RCU_IND_DATA);
		1894	efuse_box_bit_131_124 = (u8)(((efuse_straps_4 & 0xf) << 4) \| ((efuse_straps_3 & 0xf0000000) >> 28));
		1895
		1896	switch(efuse_box_bit_131_124) {
		1897	case 0x00:
		1898	gb_backend_map = 0x76543210;
		1899	break;
		1900	case 0x55:
		1901	gb_backend_map = 0x77553311;
		1902	break;
		1903	case 0x56:
		1904	gb_backend_map = 0x77553300;
		1905	break;
		1906	case 0x59:
		1907	gb_backend_map = 0x77552211;
		1908	break;
		1909	case 0x66:
		1910	gb_backend_map = 0x77443300;
		1911	break;
		1912	case 0x99:
		1913	gb_backend_map = 0x66552211;
		1914	break;
		1915	case 0x5a:
		1916	gb_backend_map = 0x77552200;
		1917	break;
		1918	case 0xaa:
		1919	gb_backend_map = 0x66442200;
		1920	break;
		1921	case 0x95:
		1922	gb_backend_map = 0x66553311;
		1923	break;
		1924	default:
		1925	DRM_ERROR("bad backend map, using default\n");
		1926	gb_backend_map =
		1927	evergreen_get_tile_pipe_to_backend_map(rdev,
		1928	rdev->config.evergreen.max_tile_pipes,
		1929	rdev->config.evergreen.max_backends,
		1930	((EVERGREEN_MAX_BACKENDS_MASK <<
		1931	rdev->config.evergreen.max_backends) &
		1932	EVERGREEN_MAX_BACKENDS_MASK));
		1933	break;
		1934	}
		1935	} else if (rdev->ddev->pdev->device == 0x68b9) {
		1936	u32 efuse_straps_3;
		1937	u8 efuse_box_bit_127_124;
		1938
		1939	WREG32(RCU_IND_INDEX, 0x203);
		1940	efuse_straps_3 = RREG32(RCU_IND_DATA);
		1941	efuse_box_bit_127_124 = (u8)((efuse_straps_3 & 0xF0000000) >> 28);
		1942
		1943	switch(efuse_box_bit_127_124) {
		1944	case 0x0:
		1945	gb_backend_map = 0x00003210;
		1946	break;
		1947	case 0x5:
		1948	case 0x6:
		1949	case 0x9:
		1950	case 0xa:
		1951	gb_backend_map = 0x00003311;
		1952	break;
		1953	default:
		1954	DRM_ERROR("bad backend map, using default\n");
		1955	gb_backend_map =
		1956	evergreen_get_tile_pipe_to_backend_map(rdev,
		1957	rdev->config.evergreen.max_tile_pipes,
		1958	rdev->config.evergreen.max_backends,
		1959	((EVERGREEN_MAX_BACKENDS_MASK <<
		1960	rdev->config.evergreen.max_backends) &
		1961	EVERGREEN_MAX_BACKENDS_MASK));
		1962	break;
		1963	}
		1964	} else {
		1965	switch (rdev->family) {
		1966	case CHIP_CYPRESS:
		1967	case CHIP_HEMLOCK:
		1968	case CHIP_BARTS:
		1969	gb_backend_map = 0x66442200;
		1970	break;
		1971	case CHIP_JUNIPER:
		1972	gb_backend_map = 0x00006420;
		1973	break;
		1974	default:
		1975	gb_backend_map =
		1976	evergreen_get_tile_pipe_to_backend_map(rdev,
		1977	rdev->config.evergreen.max_tile_pipes,
		1978	rdev->config.evergreen.max_backends,
		1979	((EVERGREEN_MAX_BACKENDS_MASK <<
		1980	rdev->config.evergreen.max_backends) &
		1981	EVERGREEN_MAX_BACKENDS_MASK));
		1982	}
		1983	}
		1984
		1985	/* setup tiling info dword. gb_addr_config is not adequate since it does
		1986	* not have bank info, so create a custom tiling dword.
		1987	* bits 3:0 num_pipes
		1988	* bits 7:4 num_banks
		1989	* bits 11:8 group_size
		1990	* bits 15:12 row_size
		1991	*/
		1992	rdev->config.evergreen.tile_config = 0;
		1993	switch (rdev->config.evergreen.max_tile_pipes) {
		1994	case 1:
		1995	default:
		1996	rdev->config.evergreen.tile_config \|= (0 << 0);
		1997	break;
		1998	case 2:
		1999	rdev->config.evergreen.tile_config \|= (1 << 0);
		2000	break;
		2001	case 4:
		2002	rdev->config.evergreen.tile_config \|= (2 << 0);
		2003	break;
		2004	case 8:
		2005	rdev->config.evergreen.tile_config \|= (3 << 0);
		2006	break;
		2007	}
1986	serge	2008	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1963	serge	2009	if (rdev->flags & RADEON_IS_IGP)
1986	serge	2010	rdev->config.evergreen.tile_config \|= 1 << 4;
1963	serge	2011	else
		2012	rdev->config.evergreen.tile_config \|=
		2013	((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
		2014	rdev->config.evergreen.tile_config \|=
		2015	((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT) << 8;
		2016	rdev->config.evergreen.tile_config \|=
		2017	((gb_addr_config & 0x30000000) >> 28) << 12;
		2018
		2019	WREG32(GB_BACKEND_MAP, gb_backend_map);
		2020	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		2021	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
		2022	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
		2023
		2024	evergreen_program_channel_remap(rdev);
		2025
		2026	num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
		2027	grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
		2028
		2029	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
		2030	u32 rb = cc_rb_backend_disable \| (0xf0 << 16);
		2031	u32 sp = cc_gc_shader_pipe_config;
		2032	u32 gfx = grbm_gfx_index \| SE_INDEX(i);
		2033
		2034	if (i == num_shader_engines) {
		2035	rb \|= BACKEND_DISABLE(EVERGREEN_MAX_BACKENDS_MASK);
		2036	sp \|= INACTIVE_SIMDS(EVERGREEN_MAX_SIMDS_MASK);
		2037	}
		2038
		2039	WREG32(GRBM_GFX_INDEX, gfx);
		2040	WREG32(RLC_GFX_INDEX, gfx);
		2041
		2042	WREG32(CC_RB_BACKEND_DISABLE, rb);
		2043	WREG32(CC_SYS_RB_BACKEND_DISABLE, rb);
		2044	WREG32(GC_USER_RB_BACKEND_DISABLE, rb);
		2045	WREG32(CC_GC_SHADER_PIPE_CONFIG, sp);
		2046	}
		2047
		2048	grbm_gfx_index \|= SE_BROADCAST_WRITES;
		2049	WREG32(GRBM_GFX_INDEX, grbm_gfx_index);
		2050	WREG32(RLC_GFX_INDEX, grbm_gfx_index);
		2051
		2052	WREG32(CGTS_SYS_TCC_DISABLE, 0);
		2053	WREG32(CGTS_TCC_DISABLE, 0);
		2054	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
		2055	WREG32(CGTS_USER_TCC_DISABLE, 0);
		2056
		2057	/* set HW defaults for 3D engine */
		2058	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \|
		2059	ROQ_IB2_START(0x2b)));
		2060
		2061	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
		2062
		2063	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \|
		2064	SYNC_GRADIENT \|
		2065	SYNC_WALKER \|
		2066	SYNC_ALIGNER));
		2067
		2068	sx_debug_1 = RREG32(SX_DEBUG_1);
		2069	sx_debug_1 \|= ENABLE_NEW_SMX_ADDRESS;
		2070	WREG32(SX_DEBUG_1, sx_debug_1);
		2071
		2072
		2073	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
		2074	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
		2075	smx_dc_ctl0 \|= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
		2076	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
		2077
		2078	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) \|
		2079	POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) \|
		2080	SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
		2081
		2082	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) \|
		2083	SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) \|
		2084	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
		2085
		2086	WREG32(VGT_NUM_INSTANCES, 1);
		2087	WREG32(SPI_CONFIG_CNTL, 0);
		2088	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
		2089	WREG32(CP_PERFMON_CNTL, 0);
		2090
		2091	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) \|
		2092	FETCH_FIFO_HIWATER(0x4) \|
		2093	DONE_FIFO_HIWATER(0xe0) \|
		2094	ALU_UPDATE_FIFO_HIWATER(0x8)));
		2095
		2096	sq_config = RREG32(SQ_CONFIG);
		2097	sq_config &= ~(PS_PRIO(3) \|
		2098	VS_PRIO(3) \|
		2099	GS_PRIO(3) \|
		2100	ES_PRIO(3));
		2101	sq_config \|= (VC_ENABLE \|
		2102	EXPORT_SRC_C \|
		2103	PS_PRIO(0) \|
		2104	VS_PRIO(1) \|
		2105	GS_PRIO(2) \|
		2106	ES_PRIO(3));
		2107
		2108	switch (rdev->family) {
		2109	case CHIP_CEDAR:
		2110	case CHIP_PALM:
		2111	case CHIP_SUMO:
		2112	case CHIP_SUMO2:
		2113	case CHIP_CAICOS:
		2114	/* no vertex cache */
		2115	sq_config &= ~VC_ENABLE;
		2116	break;
		2117	default:
		2118	break;
		2119	}
		2120
		2121	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
		2122
		2123	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
		2124	sq_gpr_resource_mgmt_1 \|= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
		2125	sq_gpr_resource_mgmt_1 \|= NUM_CLAUSE_TEMP_GPRS(4);
		2126	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		2127	sq_gpr_resource_mgmt_2 \|= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		2128	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		2129	sq_gpr_resource_mgmt_3 \|= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		2130
		2131	switch (rdev->family) {
		2132	case CHIP_CEDAR:
		2133	case CHIP_PALM:
		2134	case CHIP_SUMO:
		2135	case CHIP_SUMO2:
		2136	ps_thread_count = 96;
		2137	break;
		2138	default:
		2139	ps_thread_count = 128;
		2140	break;
		2141	}
		2142
		2143	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
		2144	sq_thread_resource_mgmt \|= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2145	sq_thread_resource_mgmt \|= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2146	sq_thread_resource_mgmt \|= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2147	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2148	sq_thread_resource_mgmt_2 \|= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2149
		2150	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2151	sq_stack_resource_mgmt_1 \|= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2152	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2153	sq_stack_resource_mgmt_2 \|= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2154	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2155	sq_stack_resource_mgmt_3 \|= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2156
		2157	WREG32(SQ_CONFIG, sq_config);
		2158	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		2159	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		2160	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
		2161	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		2162	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
		2163	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		2164	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		2165	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
		2166	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
		2167	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
		2168
		2169	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		2170	FORCE_EOV_MAX_REZ_CNT(255)));
		2171
		2172	switch (rdev->family) {
		2173	case CHIP_CEDAR:
		2174	case CHIP_PALM:
		2175	case CHIP_SUMO:
		2176	case CHIP_SUMO2:
		2177	case CHIP_CAICOS:
		2178	vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
		2179	break;
		2180	default:
		2181	vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
		2182	break;
		2183	}
		2184	vgt_cache_invalidation \|= AUTO_INVLD_EN(ES_AND_GS_AUTO);
		2185	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
		2186
		2187	WREG32(VGT_GS_VERTEX_REUSE, 16);
		2188	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
		2189	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		2190
		2191	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
		2192	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
		2193
		2194	WREG32(CB_PERF_CTR0_SEL_0, 0);
		2195	WREG32(CB_PERF_CTR0_SEL_1, 0);
		2196	WREG32(CB_PERF_CTR1_SEL_0, 0);
		2197	WREG32(CB_PERF_CTR1_SEL_1, 0);
		2198	WREG32(CB_PERF_CTR2_SEL_0, 0);
		2199	WREG32(CB_PERF_CTR2_SEL_1, 0);
		2200	WREG32(CB_PERF_CTR3_SEL_0, 0);
		2201	WREG32(CB_PERF_CTR3_SEL_1, 0);
		2202
		2203	/* clear render buffer base addresses */
		2204	WREG32(CB_COLOR0_BASE, 0);
		2205	WREG32(CB_COLOR1_BASE, 0);
		2206	WREG32(CB_COLOR2_BASE, 0);
		2207	WREG32(CB_COLOR3_BASE, 0);
		2208	WREG32(CB_COLOR4_BASE, 0);
		2209	WREG32(CB_COLOR5_BASE, 0);
		2210	WREG32(CB_COLOR6_BASE, 0);
		2211	WREG32(CB_COLOR7_BASE, 0);
		2212	WREG32(CB_COLOR8_BASE, 0);
		2213	WREG32(CB_COLOR9_BASE, 0);
		2214	WREG32(CB_COLOR10_BASE, 0);
		2215	WREG32(CB_COLOR11_BASE, 0);
		2216
		2217	/* set the shader const cache sizes to 0 */
		2218	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
		2219	WREG32(i, 0);
		2220	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
		2221	WREG32(i, 0);
		2222
		2223	tmp = RREG32(HDP_MISC_CNTL);
		2224	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		2225	WREG32(HDP_MISC_CNTL, tmp);
		2226
		2227	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		2228	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		2229
		2230	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		2231
		2232	udelay(50);
		2233
1430	serge	2234	}
		2235
		2236	int evergreen_mc_init(struct radeon_device *rdev)
		2237	{
		2238	u32 tmp;
		2239	int chansize, numchan;
		2240
		2241	/* Get VRAM informations */
		2242	rdev->mc.vram_is_ddr = true;
		2243	tmp = RREG32(MC_ARB_RAMCFG);
		2244	if (tmp & CHANSIZE_OVERRIDE) {
		2245	chansize = 16;
		2246	} else if (tmp & CHANSIZE_MASK) {
		2247	chansize = 64;
		2248	} else {
		2249	chansize = 32;
		2250	}
		2251	tmp = RREG32(MC_SHARED_CHMAP);
		2252	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		2253	case 0:
		2254	default:
		2255	numchan = 1;
		2256	break;
		2257	case 1:
		2258	numchan = 2;
		2259	break;
		2260	case 2:
		2261	numchan = 4;
		2262	break;
		2263	case 3:
		2264	numchan = 8;
		2265	break;
		2266	}
		2267	rdev->mc.vram_width = numchan * chansize;
		2268	/* Could aper size report 0 ? */
1963	serge	2269	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		2270	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1430	serge	2271	/* Setup GPU memory space */
1963	serge	2272	if (rdev->flags & RADEON_IS_IGP) {
		2273	/* size in bytes on fusion */
		2274	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		2275	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
		2276	} else {
1430	serge	2277	/* size in MB on evergreen */
		2278	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
		2279	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
1963	serge	2280	}
1430	serge	2281	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1963	serge	2282	r700_vram_gtt_location(rdev, &rdev->mc);
		2283	radeon_update_bandwidth_info(rdev);
		2284
1430	serge	2285	return 0;
		2286	}
		2287
1963	serge	2288	bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
1430	serge	2289	{
1986	serge	2290	u32 srbm_status;
		2291	u32 grbm_status;
		2292	u32 grbm_status_se0, grbm_status_se1;
		2293	struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
		2294	int r;
		2295
		2296	srbm_status = RREG32(SRBM_STATUS);
		2297	grbm_status = RREG32(GRBM_STATUS);
		2298	grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
		2299	grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
		2300	if (!(grbm_status & GUI_ACTIVE)) {
		2301	r100_gpu_lockup_update(lockup, &rdev->cp);
1963	serge	2302	return false;
1986	serge	2303	}
		2304	/* force CP activities */
		2305	r = radeon_ring_lock(rdev, 2);
		2306	if (!r) {
		2307	/* PACKET2 NOP */
		2308	radeon_ring_write(rdev, 0x80000000);
		2309	radeon_ring_write(rdev, 0x80000000);
		2310	radeon_ring_unlock_commit(rdev);
		2311	}
		2312	rdev->cp.rptr = RREG32(CP_RB_RPTR);
		2313	return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
1963	serge	2314	}
		2315
		2316	static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
		2317	{
		2318	struct evergreen_mc_save save;
		2319	u32 grbm_reset = 0;
		2320
		2321	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
		2322	return 0;
		2323
		2324	dev_info(rdev->dev, "GPU softreset \n");
		2325	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2326	RREG32(GRBM_STATUS));
		2327	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2328	RREG32(GRBM_STATUS_SE0));
		2329	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2330	RREG32(GRBM_STATUS_SE1));
		2331	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2332	RREG32(SRBM_STATUS));
		2333	evergreen_mc_stop(rdev, &save);
		2334	if (evergreen_mc_wait_for_idle(rdev)) {
		2335	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		2336	}
		2337	/* Disable CP parsing/prefetching */
		2338	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT);
		2339
		2340	/* reset all the gfx blocks */
		2341	grbm_reset = (SOFT_RESET_CP \|
		2342	SOFT_RESET_CB \|
		2343	SOFT_RESET_DB \|
		2344	SOFT_RESET_PA \|
		2345	SOFT_RESET_SC \|
		2346	SOFT_RESET_SPI \|
		2347	SOFT_RESET_SH \|
		2348	SOFT_RESET_SX \|
		2349	SOFT_RESET_TC \|
		2350	SOFT_RESET_TA \|
		2351	SOFT_RESET_VC \|
		2352	SOFT_RESET_VGT);
		2353
		2354	dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
		2355	WREG32(GRBM_SOFT_RESET, grbm_reset);
		2356	(void)RREG32(GRBM_SOFT_RESET);
		2357	udelay(50);
		2358	WREG32(GRBM_SOFT_RESET, 0);
		2359	(void)RREG32(GRBM_SOFT_RESET);
		2360	/* Wait a little for things to settle down */
		2361	udelay(50);
		2362	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2363	RREG32(GRBM_STATUS));
		2364	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2365	RREG32(GRBM_STATUS_SE0));
		2366	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2367	RREG32(GRBM_STATUS_SE1));
		2368	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2369	RREG32(SRBM_STATUS));
		2370	evergreen_mc_resume(rdev, &save);
1430	serge	2371	return 0;
		2372	}
		2373
1963	serge	2374	int evergreen_asic_reset(struct radeon_device *rdev)
		2375	{
		2376	return evergreen_gpu_soft_reset(rdev);
		2377	}
		2378
		2379	/* Interrupts */
		2380
		2381	u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
		2382	{
		2383	switch (crtc) {
		2384	case 0:
		2385	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2386	case 1:
		2387	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2388	case 2:
		2389	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2390	case 3:
		2391	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2392	case 4:
		2393	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2394	case 5:
		2395	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2396	default:
		2397	return 0;
		2398	}
		2399	}
		2400
		2401	void evergreen_disable_interrupt_state(struct radeon_device *rdev)
		2402	{
		2403	u32 tmp;
		2404
		2405	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		2406	WREG32(GRBM_INT_CNTL, 0);
		2407	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2408	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
		2409	if (!(rdev->flags & RADEON_IS_IGP)) {
		2410	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2411	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		2412	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2413	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		2414	}
		2415
		2416	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2417	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
		2418	if (!(rdev->flags & RADEON_IS_IGP)) {
		2419	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2420	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		2421	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2422	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		2423	}
		2424
		2425	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		2426	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2427
		2428	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2429	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2430	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2431	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2432	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2433	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2434	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2435	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2436	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2437	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2438	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2439	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2440
		2441	}
1430	serge	2442	static int evergreen_startup(struct radeon_device *rdev)
		2443	{
		2444	int r;
		2445
1990	serge	2446	/* enable pcie gen2 link */
		2447	if (!ASIC_IS_DCE5(rdev))
		2448	evergreen_pcie_gen2_enable(rdev);
		2449
		2450	if (ASIC_IS_DCE5(rdev)) {
		2451	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw \|\| !rdev->mc_fw) {
		2452	r = ni_init_microcode(rdev);
		2453	if (r) {
		2454	DRM_ERROR("Failed to load firmware!\n");
		2455	return r;
		2456	}
		2457	}
		2458	r = ni_mc_load_microcode(rdev);
		2459	if (r) {
		2460	DRM_ERROR("Failed to load MC firmware!\n");
		2461	return r;
		2462	}
		2463	} else {
1430	serge	2464	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		2465	r = r600_init_microcode(rdev);
		2466	if (r) {
		2467	DRM_ERROR("Failed to load firmware!\n");
		2468	return r;
		2469	}
		2470	}
1990	serge	2471	}
1963	serge	2472
1430	serge	2473	evergreen_mc_program(rdev);
		2474	if (rdev->flags & RADEON_IS_AGP) {
1963	serge	2475	evergreen_agp_enable(rdev);
1430	serge	2476	} else {
		2477	r = evergreen_pcie_gart_enable(rdev);
		2478	if (r)
		2479	return r;
		2480	}
		2481	evergreen_gpu_init(rdev);
		2482	#if 0
1963	serge	2483	r = evergreen_blit_init(rdev);
		2484	if (r) {
		2485	evergreen_blit_fini(rdev);
		2486	rdev->asic->copy = NULL;
		2487	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		2488	}
1430	serge	2489
1963	serge	2490	/* allocate wb buffer */
		2491	r = radeon_wb_init(rdev);
		2492	if (r)
		2493	return r;
		2494	#endif
		2495
		2496	/* Enable IRQ */
		2497
1430	serge	2498	r = radeon_ring_init(rdev, rdev->cp.ring_size);
		2499	if (r)
		2500	return r;
		2501	r = evergreen_cp_load_microcode(rdev);
		2502	if (r)
		2503	return r;
1963	serge	2504	r = evergreen_cp_resume(rdev);
1430	serge	2505	if (r)
		2506	return r;
1963	serge	2507
1430	serge	2508	return 0;
		2509	}
		2510
		2511
		2512
		2513
		2514
		2515
		2516
		2517	/* Plan is to move initialization in that function and use
		2518	* helper function so that radeon_device_init pretty much
		2519	* do nothing more than calling asic specific function. This
		2520	* should also allow to remove a bunch of callback function
		2521	* like vram_info.
		2522	*/
		2523	int evergreen_init(struct radeon_device *rdev)
		2524	{
		2525	int r;
		2526
		2527	/* This don't do much */
		2528	r = radeon_gem_init(rdev);
		2529	if (r)
		2530	return r;
		2531	/* Read BIOS */
		2532	if (!radeon_get_bios(rdev)) {
		2533	if (ASIC_IS_AVIVO(rdev))
		2534	return -EINVAL;
		2535	}
		2536	/* Must be an ATOMBIOS */
		2537	if (!rdev->is_atom_bios) {
1986	serge	2538	dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
1430	serge	2539	return -EINVAL;
		2540	}
		2541	r = radeon_atombios_init(rdev);
		2542	if (r)
		2543	return r;
1986	serge	2544	/* reset the asic, the gfx blocks are often in a bad state
		2545	* after the driver is unloaded or after a resume
		2546	*/
		2547	if (radeon_asic_reset(rdev))
		2548	dev_warn(rdev->dev, "GPU reset failed !\n");
1430	serge	2549	/* Post card if necessary */
1986	serge	2550	if (!radeon_card_posted(rdev)) {
1430	serge	2551	if (!rdev->bios) {
		2552	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		2553	return -EINVAL;
		2554	}
		2555	DRM_INFO("GPU not posted. posting now...\n");
		2556	atom_asic_init(rdev->mode_info.atom_context);
		2557	}
		2558	/* Initialize scratch registers */
		2559	r600_scratch_init(rdev);
		2560	/* Initialize surface registers */
		2561	radeon_surface_init(rdev);
		2562	/* Initialize clocks */
		2563	radeon_get_clock_info(rdev->ddev);
		2564	/* Fence driver */
		2565	/* initialize AGP */
		2566	if (rdev->flags & RADEON_IS_AGP) {
		2567	r = radeon_agp_init(rdev);
		2568	if (r)
		2569	radeon_agp_disable(rdev);
		2570	}
		2571	/* initialize memory controller */
		2572	r = evergreen_mc_init(rdev);
		2573	if (r)
		2574	return r;
		2575	/* Memory manager */
		2576	r = radeon_bo_init(rdev);
		2577	if (r)
		2578	return r;
1963	serge	2579
1430	serge	2580
		2581	rdev->cp.ring_obj = NULL;
		2582	r600_ring_init(rdev, 1024 * 1024);
		2583
1963	serge	2584	// rdev->ih.ring_obj = NULL;
		2585	// r600_ih_ring_init(rdev, 64 * 1024);
1430	serge	2586
		2587	r = r600_pcie_gart_init(rdev);
		2588	if (r)
		2589	return r;
1963	serge	2590
		2591	rdev->accel_working = true;
1430	serge	2592	r = evergreen_startup(rdev);
		2593	if (r) {
1963	serge	2594	dev_err(rdev->dev, "disabling GPU acceleration\n");
1430	serge	2595	rdev->accel_working = false;
		2596	}
		2597	if (rdev->accel_working) {
		2598	}
		2599	return 0;
		2600	}
		2601
1986	serge	2602
		2603	static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
1430	serge	2604	{
1986	serge	2605	u32 link_width_cntl, speed_cntl;
		2606
		2607	if (radeon_pcie_gen2 == 0)
		2608	return;
		2609
		2610	if (rdev->flags & RADEON_IS_IGP)
		2611	return;
		2612
		2613	if (!(rdev->flags & RADEON_IS_PCIE))
		2614	return;
		2615
		2616	/* x2 cards have a special sequence */
		2617	if (ASIC_IS_X2(rdev))
		2618	return;
		2619
		2620	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2621	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) \|\|
		2622	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		2623
		2624	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		2625	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		2626	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		2627
		2628	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2629	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		2630	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2631
		2632	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2633	speed_cntl \|= LC_CLR_FAILED_SPD_CHANGE_CNT;
		2634	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2635
		2636	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2637	speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
		2638	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2639
		2640	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2641	speed_cntl \|= LC_GEN2_EN_STRAP;
		2642	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2643
		2644	} else {
		2645	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		2646	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		2647	if (1)
		2648	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		2649	else
		2650	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		2651	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		2652	}
1430	serge	2653	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/evergreen.c @ 5179 – Rev 1990