WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/evergreen.c

Rev	Author	Line No.	Line
1430	serge	1	/*
		2	* Copyright 2010 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	//#include
1963	serge	26	#include
1430	serge	27	#include "drmP.h"
		28	#include "radeon.h"
1963	serge	29	#include "radeon_asic.h"
1430	serge	30	#include "radeon_drm.h"
1963	serge	31	#include "evergreend.h"
1430	serge	32	#include "atom.h"
		33	#include "avivod.h"
		34	#include "evergreen_reg.h"
1986	serge	35	#include "evergreen_blit_shaders.h"
1430	serge	36
1963	serge	37	#define EVERGREEN_PFP_UCODE_SIZE 1120
		38	#define EVERGREEN_PM4_UCODE_SIZE 1376
		39
1430	serge	40	static void evergreen_gpu_init(struct radeon_device *rdev);
		41	void evergreen_fini(struct radeon_device *rdev);
1986	serge	42	static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
1430	serge	43
		44	bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		45	{
		46	bool connected = false;
1963	serge	47
		48	switch (hpd) {
		49	case RADEON_HPD_1:
		50	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		51	connected = true;
		52	break;
		53	case RADEON_HPD_2:
		54	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		55	connected = true;
		56	break;
		57	case RADEON_HPD_3:
		58	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		59	connected = true;
		60	break;
		61	case RADEON_HPD_4:
		62	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		63	connected = true;
		64	break;
		65	case RADEON_HPD_5:
		66	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		67	connected = true;
		68	break;
		69	case RADEON_HPD_6:
		70	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		71	connected = true;
		72	break;
		73	default:
		74	break;
		75	}
		76
1430	serge	77	return connected;
		78	}
		79
		80	void evergreen_hpd_set_polarity(struct radeon_device *rdev,
		81	enum radeon_hpd_id hpd)
		82	{
1963	serge	83	u32 tmp;
		84	bool connected = evergreen_hpd_sense(rdev, hpd);
		85
		86	switch (hpd) {
		87	case RADEON_HPD_1:
		88	tmp = RREG32(DC_HPD1_INT_CONTROL);
		89	if (connected)
		90	tmp &= ~DC_HPDx_INT_POLARITY;
		91	else
		92	tmp \|= DC_HPDx_INT_POLARITY;
		93	WREG32(DC_HPD1_INT_CONTROL, tmp);
		94	break;
		95	case RADEON_HPD_2:
		96	tmp = RREG32(DC_HPD2_INT_CONTROL);
		97	if (connected)
		98	tmp &= ~DC_HPDx_INT_POLARITY;
		99	else
		100	tmp \|= DC_HPDx_INT_POLARITY;
		101	WREG32(DC_HPD2_INT_CONTROL, tmp);
		102	break;
		103	case RADEON_HPD_3:
		104	tmp = RREG32(DC_HPD3_INT_CONTROL);
		105	if (connected)
		106	tmp &= ~DC_HPDx_INT_POLARITY;
		107	else
		108	tmp \|= DC_HPDx_INT_POLARITY;
		109	WREG32(DC_HPD3_INT_CONTROL, tmp);
		110	break;
		111	case RADEON_HPD_4:
		112	tmp = RREG32(DC_HPD4_INT_CONTROL);
		113	if (connected)
		114	tmp &= ~DC_HPDx_INT_POLARITY;
		115	else
		116	tmp \|= DC_HPDx_INT_POLARITY;
		117	WREG32(DC_HPD4_INT_CONTROL, tmp);
		118	break;
		119	case RADEON_HPD_5:
		120	tmp = RREG32(DC_HPD5_INT_CONTROL);
		121	if (connected)
		122	tmp &= ~DC_HPDx_INT_POLARITY;
		123	else
		124	tmp \|= DC_HPDx_INT_POLARITY;
		125	WREG32(DC_HPD5_INT_CONTROL, tmp);
		126	break;
		127	case RADEON_HPD_6:
		128	tmp = RREG32(DC_HPD6_INT_CONTROL);
		129	if (connected)
		130	tmp &= ~DC_HPDx_INT_POLARITY;
		131	else
		132	tmp \|= DC_HPDx_INT_POLARITY;
		133	WREG32(DC_HPD6_INT_CONTROL, tmp);
		134	break;
		135	default:
		136	break;
		137	}
1430	serge	138	}
		139
1963	serge	140	#if 0
1430	serge	141	void evergreen_hpd_init(struct radeon_device *rdev)
		142	{
1963	serge	143	struct drm_device *dev = rdev->ddev;
		144	struct drm_connector *connector;
		145	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \|
		146	DC_HPDx_RX_INT_TIMER(0xfa) \| DC_HPDx_EN;
		147
		148	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		149	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		150	switch (radeon_connector->hpd.hpd) {
		151	case RADEON_HPD_1:
		152	WREG32(DC_HPD1_CONTROL, tmp);
		153	rdev->irq.hpd[0] = true;
		154	break;
		155	case RADEON_HPD_2:
		156	WREG32(DC_HPD2_CONTROL, tmp);
		157	rdev->irq.hpd[1] = true;
		158	break;
		159	case RADEON_HPD_3:
		160	WREG32(DC_HPD3_CONTROL, tmp);
		161	rdev->irq.hpd[2] = true;
		162	break;
		163	case RADEON_HPD_4:
		164	WREG32(DC_HPD4_CONTROL, tmp);
		165	rdev->irq.hpd[3] = true;
		166	break;
		167	case RADEON_HPD_5:
		168	WREG32(DC_HPD5_CONTROL, tmp);
		169	rdev->irq.hpd[4] = true;
		170	break;
		171	case RADEON_HPD_6:
		172	WREG32(DC_HPD6_CONTROL, tmp);
		173	rdev->irq.hpd[5] = true;
		174	break;
		175	default:
		176	break;
		177	}
		178	}
		179	if (rdev->irq.installed)
		180	evergreen_irq_set(rdev);
1430	serge	181	}
		182
1963	serge	183	void evergreen_hpd_fini(struct radeon_device *rdev)
		184	{
		185	struct drm_device *dev = rdev->ddev;
		186	struct drm_connector *connector;
1430	serge	187
1963	serge	188	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		189	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		190	switch (radeon_connector->hpd.hpd) {
		191	case RADEON_HPD_1:
		192	WREG32(DC_HPD1_CONTROL, 0);
		193	rdev->irq.hpd[0] = false;
		194	break;
		195	case RADEON_HPD_2:
		196	WREG32(DC_HPD2_CONTROL, 0);
		197	rdev->irq.hpd[1] = false;
		198	break;
		199	case RADEON_HPD_3:
		200	WREG32(DC_HPD3_CONTROL, 0);
		201	rdev->irq.hpd[2] = false;
		202	break;
		203	case RADEON_HPD_4:
		204	WREG32(DC_HPD4_CONTROL, 0);
		205	rdev->irq.hpd[3] = false;
		206	break;
		207	case RADEON_HPD_5:
		208	WREG32(DC_HPD5_CONTROL, 0);
		209	rdev->irq.hpd[4] = false;
		210	break;
		211	case RADEON_HPD_6:
		212	WREG32(DC_HPD6_CONTROL, 0);
		213	rdev->irq.hpd[5] = false;
		214	break;
		215	default:
		216	break;
		217	}
		218	}
1430	serge	219	}
		220
1963	serge	221	#endif
1986	serge	222	/* watermark setup */
1963	serge	223
1986	serge	224	static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
		225	struct radeon_crtc *radeon_crtc,
		226	struct drm_display_mode *mode,
		227	struct drm_display_mode *other_mode)
		228	{
		229	u32 tmp;
		230	/*
		231	* Line Buffer Setup
		232	* There are 3 line buffers, each one shared by 2 display controllers.
		233	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
		234	* the display controllers. The paritioning is done via one of four
		235	* preset allocations specified in bits 2:0:
		236	* first display controller
		237	* 0 - first half of lb (3840 * 2)
		238	* 1 - first 3/4 of lb (5760 * 2)
		239	* 2 - whole lb (7680 * 2), other crtc must be disabled
		240	* 3 - first 1/4 of lb (1920 * 2)
		241	* second display controller
		242	* 4 - second half of lb (3840 * 2)
		243	* 5 - second 3/4 of lb (5760 * 2)
		244	* 6 - whole lb (7680 * 2), other crtc must be disabled
		245	* 7 - last 1/4 of lb (1920 * 2)
		246	*/
		247	/* this can get tricky if we have two large displays on a paired group
		248	* of crtcs. Ideally for multiple large displays we'd assign them to
		249	* non-linked crtcs for maximum line buffer allocation.
		250	*/
		251	if (radeon_crtc->base.enabled && mode) {
		252	if (other_mode)
		253	tmp = 0; /* 1/2 */
		254	else
		255	tmp = 2; /* whole */
		256	} else
		257	tmp = 0;
1963	serge	258
1986	serge	259	/* second controller of the pair uses second half of the lb */
		260	if (radeon_crtc->crtc_id % 2)
		261	tmp += 4;
		262	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
		263
		264	if (radeon_crtc->base.enabled && mode) {
		265	switch (tmp) {
		266	case 0:
		267	case 4:
		268	default:
		269	if (ASIC_IS_DCE5(rdev))
		270	return 4096 * 2;
		271	else
		272	return 3840 * 2;
		273	case 1:
		274	case 5:
		275	if (ASIC_IS_DCE5(rdev))
		276	return 6144 * 2;
		277	else
		278	return 5760 * 2;
		279	case 2:
		280	case 6:
		281	if (ASIC_IS_DCE5(rdev))
		282	return 8192 * 2;
		283	else
		284	return 7680 * 2;
		285	case 3:
		286	case 7:
		287	if (ASIC_IS_DCE5(rdev))
		288	return 2048 * 2;
		289	else
		290	return 1920 * 2;
		291	}
		292	}
		293
		294	/* controller not enabled, so no lb used */
		295	return 0;
		296	}
		297
		298	static u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
		299	{
		300	u32 tmp = RREG32(MC_SHARED_CHMAP);
		301
		302	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		303	case 0:
		304	default:
		305	return 1;
		306	case 1:
		307	return 2;
		308	case 2:
		309	return 4;
		310	case 3:
		311	return 8;
		312	}
		313	}
		314
		315	struct evergreen_wm_params {
		316	u32 dram_channels; /* number of dram channels */
		317	u32 yclk; /* bandwidth per dram data pin in kHz */
		318	u32 sclk; /* engine clock in kHz */
		319	u32 disp_clk; /* display clock in kHz */
		320	u32 src_width; /* viewport width */
		321	u32 active_time; /* active display time in ns */
		322	u32 blank_time; /* blank time in ns */
		323	bool interlaced; /* mode is interlaced */
		324	fixed20_12 vsc; /* vertical scale ratio */
		325	u32 num_heads; /* number of active crtcs */
		326	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
		327	u32 lb_size; /* line buffer allocated to pipe */
		328	u32 vtaps; /* vertical scaler taps */
		329	};
		330
		331	static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
		332	{
		333	/* Calculate DRAM Bandwidth and the part allocated to display. */
		334	fixed20_12 dram_efficiency; /* 0.7 */
		335	fixed20_12 yclk, dram_channels, bandwidth;
		336	fixed20_12 a;
		337
		338	a.full = dfixed_const(1000);
		339	yclk.full = dfixed_const(wm->yclk);
		340	yclk.full = dfixed_div(yclk, a);
		341	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		342	a.full = dfixed_const(10);
		343	dram_efficiency.full = dfixed_const(7);
		344	dram_efficiency.full = dfixed_div(dram_efficiency, a);
		345	bandwidth.full = dfixed_mul(dram_channels, yclk);
		346	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
		347
		348	return dfixed_trunc(bandwidth);
		349	}
		350
		351	static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		352	{
		353	/* Calculate DRAM Bandwidth and the part allocated to display. */
		354	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
		355	fixed20_12 yclk, dram_channels, bandwidth;
		356	fixed20_12 a;
		357
		358	a.full = dfixed_const(1000);
		359	yclk.full = dfixed_const(wm->yclk);
		360	yclk.full = dfixed_div(yclk, a);
		361	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		362	a.full = dfixed_const(10);
		363	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
		364	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
		365	bandwidth.full = dfixed_mul(dram_channels, yclk);
		366	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
		367
		368	return dfixed_trunc(bandwidth);
		369	}
		370
		371	static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
		372	{
		373	/* Calculate the display Data return Bandwidth */
		374	fixed20_12 return_efficiency; /* 0.8 */
		375	fixed20_12 sclk, bandwidth;
		376	fixed20_12 a;
		377
		378	a.full = dfixed_const(1000);
		379	sclk.full = dfixed_const(wm->sclk);
		380	sclk.full = dfixed_div(sclk, a);
		381	a.full = dfixed_const(10);
		382	return_efficiency.full = dfixed_const(8);
		383	return_efficiency.full = dfixed_div(return_efficiency, a);
		384	a.full = dfixed_const(32);
		385	bandwidth.full = dfixed_mul(a, sclk);
		386	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
		387
		388	return dfixed_trunc(bandwidth);
		389	}
		390
		391	static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
		392	{
		393	/* Calculate the DMIF Request Bandwidth */
		394	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
		395	fixed20_12 disp_clk, bandwidth;
		396	fixed20_12 a;
		397
		398	a.full = dfixed_const(1000);
		399	disp_clk.full = dfixed_const(wm->disp_clk);
		400	disp_clk.full = dfixed_div(disp_clk, a);
		401	a.full = dfixed_const(10);
		402	disp_clk_request_efficiency.full = dfixed_const(8);
		403	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
		404	a.full = dfixed_const(32);
		405	bandwidth.full = dfixed_mul(a, disp_clk);
		406	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
		407
		408	return dfixed_trunc(bandwidth);
		409	}
		410
		411	static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
		412	{
		413	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
		414	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
		415	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
		416	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
		417
		418	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
		419	}
		420
		421	static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
		422	{
		423	/* Calculate the display mode Average Bandwidth
		424	* DisplayMode should contain the source and destination dimensions,
		425	* timing, etc.
		426	*/
		427	fixed20_12 bpp;
		428	fixed20_12 line_time;
		429	fixed20_12 src_width;
		430	fixed20_12 bandwidth;
		431	fixed20_12 a;
		432
		433	a.full = dfixed_const(1000);
		434	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
		435	line_time.full = dfixed_div(line_time, a);
		436	bpp.full = dfixed_const(wm->bytes_per_pixel);
		437	src_width.full = dfixed_const(wm->src_width);
		438	bandwidth.full = dfixed_mul(src_width, bpp);
		439	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
		440	bandwidth.full = dfixed_div(bandwidth, line_time);
		441
		442	return dfixed_trunc(bandwidth);
		443	}
		444
		445	static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
		446	{
		447	/* First calcualte the latency in ns */
		448	u32 mc_latency = 2000; /* 2000 ns. */
		449	u32 available_bandwidth = evergreen_available_bandwidth(wm);
		450	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
		451	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
		452	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
		453	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
		454	(wm->num_heads * cursor_line_pair_return_time);
		455	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
		456	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
		457	fixed20_12 a, b, c;
		458
		459	if (wm->num_heads == 0)
		460	return 0;
		461
		462	a.full = dfixed_const(2);
		463	b.full = dfixed_const(1);
		464	if ((wm->vsc.full > a.full) \|\|
		465	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) \|\|
		466	(wm->vtaps >= 5) \|\|
		467	((wm->vsc.full >= a.full) && wm->interlaced))
		468	max_src_lines_per_dst_line = 4;
		469	else
		470	max_src_lines_per_dst_line = 2;
		471
		472	a.full = dfixed_const(available_bandwidth);
		473	b.full = dfixed_const(wm->num_heads);
		474	a.full = dfixed_div(a, b);
		475
		476	b.full = dfixed_const(1000);
		477	c.full = dfixed_const(wm->disp_clk);
		478	b.full = dfixed_div(c, b);
		479	c.full = dfixed_const(wm->bytes_per_pixel);
		480	b.full = dfixed_mul(b, c);
		481
		482	lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
		483
		484	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
		485	b.full = dfixed_const(1000);
		486	c.full = dfixed_const(lb_fill_bw);
		487	b.full = dfixed_div(c, b);
		488	a.full = dfixed_div(a, b);
		489	line_fill_time = dfixed_trunc(a);
		490
		491	if (line_fill_time < wm->active_time)
		492	return latency;
		493	else
		494	return latency + (line_fill_time - wm->active_time);
		495
		496	}
		497
		498	static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		499	{
		500	if (evergreen_average_bandwidth(wm) <=
		501	(evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
		502	return true;
		503	else
		504	return false;
		505	};
		506
		507	static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
		508	{
		509	if (evergreen_average_bandwidth(wm) <=
		510	(evergreen_available_bandwidth(wm) / wm->num_heads))
		511	return true;
		512	else
		513	return false;
		514	};
		515
		516	static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
		517	{
		518	u32 lb_partitions = wm->lb_size / wm->src_width;
		519	u32 line_time = wm->active_time + wm->blank_time;
		520	u32 latency_tolerant_lines;
		521	u32 latency_hiding;
		522	fixed20_12 a;
		523
		524	a.full = dfixed_const(1);
		525	if (wm->vsc.full > a.full)
		526	latency_tolerant_lines = 1;
		527	else {
		528	if (lb_partitions <= (wm->vtaps + 1))
		529	latency_tolerant_lines = 1;
		530	else
		531	latency_tolerant_lines = 2;
		532	}
		533
		534	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
		535
		536	if (evergreen_latency_watermark(wm) <= latency_hiding)
		537	return true;
		538	else
		539	return false;
		540	}
		541
		542	static void evergreen_program_watermarks(struct radeon_device *rdev,
		543	struct radeon_crtc *radeon_crtc,
		544	u32 lb_size, u32 num_heads)
		545	{
		546	struct drm_display_mode *mode = &radeon_crtc->base.mode;
		547	struct evergreen_wm_params wm;
		548	u32 pixel_period;
		549	u32 line_time = 0;
		550	u32 latency_watermark_a = 0, latency_watermark_b = 0;
		551	u32 priority_a_mark = 0, priority_b_mark = 0;
		552	u32 priority_a_cnt = PRIORITY_OFF;
		553	u32 priority_b_cnt = PRIORITY_OFF;
		554	u32 pipe_offset = radeon_crtc->crtc_id * 16;
		555	u32 tmp, arb_control3;
		556	fixed20_12 a, b, c;
		557
		558	if (radeon_crtc->base.enabled && num_heads && mode) {
		559	pixel_period = 1000000 / (u32)mode->clock;
		560	line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
		561	priority_a_cnt = 0;
		562	priority_b_cnt = 0;
		563
		564	wm.yclk = rdev->pm.current_mclk * 10;
		565	wm.sclk = rdev->pm.current_sclk * 10;
		566	wm.disp_clk = mode->clock;
		567	wm.src_width = mode->crtc_hdisplay;
		568	wm.active_time = mode->crtc_hdisplay * pixel_period;
		569	wm.blank_time = line_time - wm.active_time;
		570	wm.interlaced = false;
		571	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		572	wm.interlaced = true;
		573	wm.vsc = radeon_crtc->vsc;
		574	wm.vtaps = 1;
		575	if (radeon_crtc->rmx_type != RMX_OFF)
		576	wm.vtaps = 2;
		577	wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
		578	wm.lb_size = lb_size;
		579	wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
		580	wm.num_heads = num_heads;
		581
		582	/* set for high clocks */
		583	latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
		584	/* set for low clocks */
		585	/* wm.yclk = low clk; wm.sclk = low clk */
		586	latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
		587
		588	/* possibly force display priority to high */
		589	/* should really do this at mode validation time... */
		590	if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) \|\|
		591	!evergreen_average_bandwidth_vs_available_bandwidth(&wm) \|\|
		592	!evergreen_check_latency_hiding(&wm) \|\|
		593	(rdev->disp_priority == 2)) {
		594	DRM_INFO("force priority to high\n");
		595	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		596	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		597	}
		598
		599	a.full = dfixed_const(1000);
		600	b.full = dfixed_const(mode->clock);
		601	b.full = dfixed_div(b, a);
		602	c.full = dfixed_const(latency_watermark_a);
		603	c.full = dfixed_mul(c, b);
		604	c.full = dfixed_mul(c, radeon_crtc->hsc);
		605	c.full = dfixed_div(c, a);
		606	a.full = dfixed_const(16);
		607	c.full = dfixed_div(c, a);
		608	priority_a_mark = dfixed_trunc(c);
		609	priority_a_cnt \|= priority_a_mark & PRIORITY_MARK_MASK;
		610
		611	a.full = dfixed_const(1000);
		612	b.full = dfixed_const(mode->clock);
		613	b.full = dfixed_div(b, a);
		614	c.full = dfixed_const(latency_watermark_b);
		615	c.full = dfixed_mul(c, b);
		616	c.full = dfixed_mul(c, radeon_crtc->hsc);
		617	c.full = dfixed_div(c, a);
		618	a.full = dfixed_const(16);
		619	c.full = dfixed_div(c, a);
		620	priority_b_mark = dfixed_trunc(c);
		621	priority_b_cnt \|= priority_b_mark & PRIORITY_MARK_MASK;
		622	}
		623
		624	/* select wm A */
		625	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		626	tmp = arb_control3;
		627	tmp &= ~LATENCY_WATERMARK_MASK(3);
		628	tmp \|= LATENCY_WATERMARK_MASK(1);
		629	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		630	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		631	(LATENCY_LOW_WATERMARK(latency_watermark_a) \|
		632	LATENCY_HIGH_WATERMARK(line_time)));
		633	/* select wm B */
		634	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		635	tmp &= ~LATENCY_WATERMARK_MASK(3);
		636	tmp \|= LATENCY_WATERMARK_MASK(2);
		637	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		638	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		639	(LATENCY_LOW_WATERMARK(latency_watermark_b) \|
		640	LATENCY_HIGH_WATERMARK(line_time)));
		641	/* restore original selection */
		642	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
		643
		644	/* write the priority marks */
		645	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
		646	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
		647
		648	}
		649
1963	serge	650	void evergreen_bandwidth_update(struct radeon_device *rdev)
1430	serge	651	{
1986	serge	652	struct drm_display_mode *mode0 = NULL;
		653	struct drm_display_mode *mode1 = NULL;
		654	u32 num_heads = 0, lb_size;
		655	int i;
		656
		657	radeon_update_display_priority(rdev);
		658
		659	for (i = 0; i < rdev->num_crtc; i++) {
		660	if (rdev->mode_info.crtcs[i]->base.enabled)
		661	num_heads++;
		662	}
		663	for (i = 0; i < rdev->num_crtc; i += 2) {
		664	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
		665	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
		666	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
		667	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
		668	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
		669	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
		670	}
1430	serge	671	}
		672
1963	serge	673	int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1430	serge	674	{
		675	unsigned i;
		676	u32 tmp;
		677
		678	for (i = 0; i < rdev->usec_timeout; i++) {
		679	/* read MC_STATUS */
		680	tmp = RREG32(SRBM_STATUS) & 0x1F00;
		681	if (!tmp)
		682	return 0;
		683	udelay(1);
		684	}
		685	return -1;
		686	}
		687
		688	/*
		689	* GART
		690	*/
1963	serge	691	void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
		692	{
		693	unsigned i;
		694	u32 tmp;
		695
		696	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		697
		698	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		699	for (i = 0; i < rdev->usec_timeout; i++) {
		700	/* read MC_STATUS */
		701	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		702	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		703	if (tmp == 2) {
		704	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		705	return;
		706	}
		707	if (tmp) {
		708	return;
		709	}
		710	udelay(1);
		711	}
		712	}
		713
1430	serge	714	int evergreen_pcie_gart_enable(struct radeon_device *rdev)
		715	{
		716	u32 tmp;
1963	serge	717	int r;
1430	serge	718
		719	if (rdev->gart.table.vram.robj == NULL) {
		720	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		721	return -EINVAL;
		722	}
		723	r = radeon_gart_table_vram_pin(rdev);
		724	if (r)
		725	return r;
		726	radeon_gart_restore(rdev);
		727	/* Setup L2 cache */
		728	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		729	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		730	EFFECTIVE_L2_QUEUE_SIZE(7));
		731	WREG32(VM_L2_CNTL2, 0);
		732	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		733	/* Setup TLB control */
		734	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		735	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		736	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		737	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
1963	serge	738	if (rdev->flags & RADEON_IS_IGP) {
		739	WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
		740	WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
		741	WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
		742	} else {
1430	serge	743	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		744	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		745	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1963	serge	746	}
1430	serge	747	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		748	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		749	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		750	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
		751	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		752	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		753	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		754	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		755	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		756	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		757	(u32)(rdev->dummy_page.addr >> 12));
1963	serge	758	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	759
1963	serge	760	evergreen_pcie_gart_tlb_flush(rdev);
1430	serge	761	rdev->gart.ready = true;
		762	return 0;
		763	}
		764
		765	void evergreen_pcie_gart_disable(struct radeon_device *rdev)
		766	{
		767	u32 tmp;
1963	serge	768	int r;
1430	serge	769
		770	/* Disable all tables */
1963	serge	771	WREG32(VM_CONTEXT0_CNTL, 0);
		772	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	773
		774	/* Setup L2 cache */
		775	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		776	EFFECTIVE_L2_QUEUE_SIZE(7));
		777	WREG32(VM_L2_CNTL2, 0);
		778	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		779	/* Setup TLB control */
		780	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		781	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		782	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		783	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		784	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		785	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		786	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		787	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
		788	if (rdev->gart.table.vram.robj) {
		789	r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
		790	if (likely(r == 0)) {
		791	radeon_bo_kunmap(rdev->gart.table.vram.robj);
		792	radeon_bo_unpin(rdev->gart.table.vram.robj);
		793	radeon_bo_unreserve(rdev->gart.table.vram.robj);
		794	}
		795	}
		796	}
		797
		798	void evergreen_pcie_gart_fini(struct radeon_device *rdev)
		799	{
		800	evergreen_pcie_gart_disable(rdev);
		801	radeon_gart_table_vram_free(rdev);
		802	radeon_gart_fini(rdev);
		803	}
		804
		805
		806	void evergreen_agp_enable(struct radeon_device *rdev)
		807	{
		808	u32 tmp;
		809
		810	/* Setup L2 cache */
		811	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		812	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		813	EFFECTIVE_L2_QUEUE_SIZE(7));
		814	WREG32(VM_L2_CNTL2, 0);
		815	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		816	/* Setup TLB control */
		817	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		818	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		819	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		820	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		821	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		822	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		823	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		824	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		825	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		826	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		827	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1963	serge	828	WREG32(VM_CONTEXT0_CNTL, 0);
		829	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	830	}
		831
1963	serge	832	void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	833	{
		834	save->vga_control[0] = RREG32(D1VGA_CONTROL);
		835	save->vga_control[1] = RREG32(D2VGA_CONTROL);
		836	save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL);
		837	save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL);
		838	save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL);
		839	save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL);
		840	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
		841	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
		842	save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
		843	save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
1963	serge	844	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	845	save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
		846	save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
		847	save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
		848	save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
1963	serge	849	}
1430	serge	850
		851	/* Stop all video */
		852	WREG32(VGA_RENDER_CONTROL, 0);
		853	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
		854	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1963	serge	855	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	856	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
		857	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
		858	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
		859	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1963	serge	860	}
1430	serge	861	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		862	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1963	serge	863	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	864	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		865	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		866	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		867	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	868	}
1430	serge	869	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		870	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1963	serge	871	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	872	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		873	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		874	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		875	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	876	}
1430	serge	877
		878	WREG32(D1VGA_CONTROL, 0);
		879	WREG32(D2VGA_CONTROL, 0);
		880	WREG32(EVERGREEN_D3VGA_CONTROL, 0);
		881	WREG32(EVERGREEN_D4VGA_CONTROL, 0);
		882	WREG32(EVERGREEN_D5VGA_CONTROL, 0);
		883	WREG32(EVERGREEN_D6VGA_CONTROL, 0);
		884	}
		885
1963	serge	886	void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	887	{
		888	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
		889	upper_32_bits(rdev->mc.vram_start));
		890	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
		891	upper_32_bits(rdev->mc.vram_start));
		892	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
		893	(u32)rdev->mc.vram_start);
		894	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
		895	(u32)rdev->mc.vram_start);
		896
		897	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
		898	upper_32_bits(rdev->mc.vram_start));
		899	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
		900	upper_32_bits(rdev->mc.vram_start));
		901	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
		902	(u32)rdev->mc.vram_start);
		903	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
		904	(u32)rdev->mc.vram_start);
		905
1963	serge	906	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	907	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
		908	upper_32_bits(rdev->mc.vram_start));
		909	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
		910	upper_32_bits(rdev->mc.vram_start));
		911	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
		912	(u32)rdev->mc.vram_start);
		913	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
		914	(u32)rdev->mc.vram_start);
		915
		916	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
		917	upper_32_bits(rdev->mc.vram_start));
		918	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
		919	upper_32_bits(rdev->mc.vram_start));
		920	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
		921	(u32)rdev->mc.vram_start);
		922	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
		923	(u32)rdev->mc.vram_start);
		924
		925	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
		926	upper_32_bits(rdev->mc.vram_start));
		927	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
		928	upper_32_bits(rdev->mc.vram_start));
		929	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
		930	(u32)rdev->mc.vram_start);
		931	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
		932	(u32)rdev->mc.vram_start);
		933
		934	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
		935	upper_32_bits(rdev->mc.vram_start));
		936	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
		937	upper_32_bits(rdev->mc.vram_start));
		938	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
		939	(u32)rdev->mc.vram_start);
		940	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
		941	(u32)rdev->mc.vram_start);
1963	serge	942	}
1430	serge	943
		944	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
		945	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
		946	/* Unlock host access */
		947	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
		948	mdelay(1);
		949	/* Restore video state */
		950	WREG32(D1VGA_CONTROL, save->vga_control[0]);
		951	WREG32(D2VGA_CONTROL, save->vga_control[1]);
		952	WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]);
		953	WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]);
		954	WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]);
		955	WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
		956	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
		957	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
1963	serge	958	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	959	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
		960	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
		961	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
		962	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1963	serge	963	}
1430	serge	964	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]);
		965	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]);
1963	serge	966	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	967	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
		968	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
		969	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
		970	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
1963	serge	971	}
1430	serge	972	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		973	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
1963	serge	974	if (!(rdev->flags & RADEON_IS_IGP)) {
1430	serge	975	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		976	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		977	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		978	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	979	}
1430	serge	980	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
		981	}
		982
1963	serge	983	void evergreen_mc_program(struct radeon_device *rdev)
1430	serge	984	{
		985	struct evergreen_mc_save save;
		986	u32 tmp;
		987	int i, j;
		988
		989	/* Initialize HDP */
		990	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		991	WREG32((0x2c14 + j), 0x00000000);
		992	WREG32((0x2c18 + j), 0x00000000);
		993	WREG32((0x2c1c + j), 0x00000000);
		994	WREG32((0x2c20 + j), 0x00000000);
		995	WREG32((0x2c24 + j), 0x00000000);
		996	}
		997	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		998
		999	evergreen_mc_stop(rdev, &save);
		1000	if (evergreen_mc_wait_for_idle(rdev)) {
		1001	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1002	}
		1003	/* Lockout access through VGA aperture*/
		1004	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		1005	/* Update configuration */
		1006	if (rdev->flags & RADEON_IS_AGP) {
		1007	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		1008	/* VRAM before AGP */
		1009	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1010	rdev->mc.vram_start >> 12);
		1011	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1012	rdev->mc.gtt_end >> 12);
		1013	} else {
		1014	/* VRAM after AGP */
		1015	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1016	rdev->mc.gtt_start >> 12);
		1017	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1018	rdev->mc.vram_end >> 12);
		1019	}
		1020	} else {
		1021	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		1022	rdev->mc.vram_start >> 12);
		1023	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		1024	rdev->mc.vram_end >> 12);
		1025	}
		1026	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
1963	serge	1027	if (rdev->flags & RADEON_IS_IGP) {
		1028	tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
		1029	tmp \|= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
		1030	tmp \|= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
		1031	WREG32(MC_FUS_VM_FB_OFFSET, tmp);
		1032	}
1430	serge	1033	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		1034	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		1035	WREG32(MC_VM_FB_LOCATION, tmp);
		1036	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1963	serge	1037	WREG32(HDP_NONSURFACE_INFO, (2 << 7) \| (1 << 30));
		1038	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1430	serge	1039	if (rdev->flags & RADEON_IS_AGP) {
		1040	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
		1041	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
		1042	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		1043	} else {
		1044	WREG32(MC_VM_AGP_BASE, 0);
		1045	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		1046	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		1047	}
		1048	if (evergreen_mc_wait_for_idle(rdev)) {
		1049	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1050	}
		1051	evergreen_mc_resume(rdev, &save);
		1052	/* we need to own VRAM, so turn off the VGA renderer here
		1053	* to stop it overwriting our objects */
		1054	rv515_vga_render_disable(rdev);
		1055	}
		1056
		1057	/*
		1058	* CP.
		1059	*/
1986	serge	1060	void evergreen_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		1061	{
		1062	/* set to DX10/11 mode */
		1063	radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
		1064	radeon_ring_write(rdev, 1);
		1065	/* FIXME: implement */
		1066	radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		1067	radeon_ring_write(rdev,
		1068	#ifdef __BIG_ENDIAN
		1069	(2 << 0) \|
		1070	#endif
		1071	(ib->gpu_addr & 0xFFFFFFFC));
		1072	radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
		1073	radeon_ring_write(rdev, ib->length_dw);
		1074	}
1963	serge	1075
1986	serge	1076
1963	serge	1077	static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1430	serge	1078	{
1963	serge	1079	const __be32 *fw_data;
		1080	int i;
		1081
		1082	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1083	return -EINVAL;
		1084
		1085	r700_cp_stop(rdev);
		1086	WREG32(CP_RB_CNTL,
		1087	#ifdef __BIG_ENDIAN
		1088	BUF_SWAP_32BIT \|
		1089	#endif
		1090	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
		1091
		1092	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1093	WREG32(CP_PFP_UCODE_ADDR, 0);
		1094	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
		1095	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		1096	WREG32(CP_PFP_UCODE_ADDR, 0);
		1097
		1098	fw_data = (const __be32 *)rdev->me_fw->data;
		1099	WREG32(CP_ME_RAM_WADDR, 0);
		1100	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
		1101	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		1102
		1103	WREG32(CP_PFP_UCODE_ADDR, 0);
		1104	WREG32(CP_ME_RAM_WADDR, 0);
		1105	WREG32(CP_ME_RAM_RADDR, 0);
		1106	return 0;
1430	serge	1107	}
		1108
1963	serge	1109	static int evergreen_cp_start(struct radeon_device *rdev)
		1110	{
		1111	int r, i;
		1112	uint32_t cp_me;
1430	serge	1113
1963	serge	1114	r = radeon_ring_lock(rdev, 7);
		1115	if (r) {
		1116	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1117	return r;
		1118	}
		1119	radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1120	radeon_ring_write(rdev, 0x1);
		1121	radeon_ring_write(rdev, 0x0);
		1122	radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
		1123	radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1124	radeon_ring_write(rdev, 0);
		1125	radeon_ring_write(rdev, 0);
		1126	radeon_ring_unlock_commit(rdev);
		1127
		1128	cp_me = 0xff;
		1129	WREG32(CP_ME_CNTL, cp_me);
		1130
		1131	r = radeon_ring_lock(rdev, evergreen_default_size + 19);
		1132	if (r) {
		1133	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1134	return r;
		1135	}
		1136
		1137	/* setup clear context state */
		1138	radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1139	radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
		1140
		1141	for (i = 0; i < evergreen_default_size; i++)
		1142	radeon_ring_write(rdev, evergreen_default_state[i]);
		1143
		1144	radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1145	radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
		1146
		1147	/* set clear context state */
		1148	radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
		1149	radeon_ring_write(rdev, 0);
		1150
		1151	/* SQ_VTX_BASE_VTX_LOC */
		1152	radeon_ring_write(rdev, 0xc0026f00);
		1153	radeon_ring_write(rdev, 0x00000000);
		1154	radeon_ring_write(rdev, 0x00000000);
		1155	radeon_ring_write(rdev, 0x00000000);
		1156
		1157	/* Clear consts */
		1158	radeon_ring_write(rdev, 0xc0036f00);
		1159	radeon_ring_write(rdev, 0x00000bc4);
		1160	radeon_ring_write(rdev, 0xffffffff);
		1161	radeon_ring_write(rdev, 0xffffffff);
		1162	radeon_ring_write(rdev, 0xffffffff);
		1163
		1164	radeon_ring_write(rdev, 0xc0026900);
		1165	radeon_ring_write(rdev, 0x00000316);
		1166	radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		1167	radeon_ring_write(rdev, 0x00000010); /* */
		1168
		1169	radeon_ring_unlock_commit(rdev);
		1170
		1171	return 0;
		1172	}
		1173
		1174	int evergreen_cp_resume(struct radeon_device *rdev)
1430	serge	1175	{
1963	serge	1176	u32 tmp;
		1177	u32 rb_bufsz;
		1178	int r;
1430	serge	1179
1963	serge	1180	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
		1181	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP \|
		1182	SOFT_RESET_PA \|
		1183	SOFT_RESET_SH \|
		1184	SOFT_RESET_VGT \|
		1185	SOFT_RESET_SX));
		1186	RREG32(GRBM_SOFT_RESET);
		1187	mdelay(15);
		1188	WREG32(GRBM_SOFT_RESET, 0);
		1189	RREG32(GRBM_SOFT_RESET);
		1190
		1191	/* Set ring buffer size */
		1192	rb_bufsz = drm_order(rdev->cp.ring_size / 8);
		1193	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
		1194	#ifdef __BIG_ENDIAN
		1195	tmp \|= BUF_SWAP_32BIT;
		1196	#endif
		1197	WREG32(CP_RB_CNTL, tmp);
		1198	WREG32(CP_SEM_WAIT_TIMER, 0x4);
		1199
		1200	/* Set the write pointer delay */
		1201	WREG32(CP_RB_WPTR_DELAY, 0);
		1202
		1203	/* Initialize the ring buffer's read and write pointers */
		1204	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1205	WREG32(CP_RB_RPTR_WR, 0);
		1206	WREG32(CP_RB_WPTR, 0);
		1207
		1208	/* set the wb address wether it's enabled or not */
		1209	WREG32(CP_RB_RPTR_ADDR,
		1210	#ifdef __BIG_ENDIAN
		1211	RB_RPTR_SWAP(2) \|
		1212	#endif
		1213	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1214	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1215	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1216
		1217	if (rdev->wb.enabled)
		1218	WREG32(SCRATCH_UMSK, 0xff);
		1219	else {
		1220	tmp \|= RB_NO_UPDATE;
		1221	WREG32(SCRATCH_UMSK, 0);
		1222	}
		1223
		1224	mdelay(1);
		1225	WREG32(CP_RB_CNTL, tmp);
		1226
		1227	WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
		1228	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1229
		1230	rdev->cp.rptr = RREG32(CP_RB_RPTR);
		1231	rdev->cp.wptr = RREG32(CP_RB_WPTR);
		1232
		1233	evergreen_cp_start(rdev);
		1234	rdev->cp.ready = true;
		1235	r = radeon_ring_test(rdev);
		1236	if (r) {
		1237	rdev->cp.ready = false;
		1238	return r;
		1239	}
1430	serge	1240	return 0;
		1241	}
		1242
		1243	/*
		1244	* Core functions
		1245	*/
1963	serge	1246	static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
		1247	u32 num_tile_pipes,
1430	serge	1248	u32 num_backends,
		1249	u32 backend_disable_mask)
		1250	{
		1251	u32 backend_map = 0;
1963	serge	1252	u32 enabled_backends_mask = 0;
		1253	u32 enabled_backends_count = 0;
		1254	u32 cur_pipe;
		1255	u32 swizzle_pipe[EVERGREEN_MAX_PIPES];
		1256	u32 cur_backend = 0;
		1257	u32 i;
		1258	bool force_no_swizzle;
1430	serge	1259
1963	serge	1260	if (num_tile_pipes > EVERGREEN_MAX_PIPES)
		1261	num_tile_pipes = EVERGREEN_MAX_PIPES;
		1262	if (num_tile_pipes < 1)
		1263	num_tile_pipes = 1;
		1264	if (num_backends > EVERGREEN_MAX_BACKENDS)
		1265	num_backends = EVERGREEN_MAX_BACKENDS;
		1266	if (num_backends < 1)
		1267	num_backends = 1;
		1268
		1269	for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
		1270	if (((backend_disable_mask >> i) & 1) == 0) {
		1271	enabled_backends_mask \|= (1 << i);
		1272	++enabled_backends_count;
		1273	}
		1274	if (enabled_backends_count == num_backends)
		1275	break;
		1276	}
		1277
		1278	if (enabled_backends_count == 0) {
		1279	enabled_backends_mask = 1;
		1280	enabled_backends_count = 1;
		1281	}
		1282
		1283	if (enabled_backends_count != num_backends)
		1284	num_backends = enabled_backends_count;
		1285
		1286	memset((uint8_t )&swizzle_pipe[0], 0, sizeof(u32) EVERGREEN_MAX_PIPES);
		1287	switch (rdev->family) {
		1288	case CHIP_CEDAR:
		1289	case CHIP_REDWOOD:
		1290	case CHIP_PALM:
		1291	case CHIP_SUMO:
		1292	case CHIP_SUMO2:
		1293	case CHIP_TURKS:
		1294	case CHIP_CAICOS:
		1295	force_no_swizzle = false;
		1296	break;
		1297	case CHIP_CYPRESS:
		1298	case CHIP_HEMLOCK:
		1299	case CHIP_JUNIPER:
		1300	case CHIP_BARTS:
		1301	default:
		1302	force_no_swizzle = true;
		1303	break;
		1304	}
		1305	if (force_no_swizzle) {
		1306	bool last_backend_enabled = false;
		1307
		1308	force_no_swizzle = false;
		1309	for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
		1310	if (((enabled_backends_mask >> i) & 1) == 1) {
		1311	if (last_backend_enabled)
		1312	force_no_swizzle = true;
		1313	last_backend_enabled = true;
		1314	} else
		1315	last_backend_enabled = false;
		1316	}
		1317	}
		1318
		1319	switch (num_tile_pipes) {
		1320	case 1:
		1321	case 3:
		1322	case 5:
		1323	case 7:
		1324	DRM_ERROR("odd number of pipes!\n");
		1325	break;
		1326	case 2:
		1327	swizzle_pipe[0] = 0;
		1328	swizzle_pipe[1] = 1;
		1329	break;
		1330	case 4:
		1331	if (force_no_swizzle) {
		1332	swizzle_pipe[0] = 0;
		1333	swizzle_pipe[1] = 1;
		1334	swizzle_pipe[2] = 2;
		1335	swizzle_pipe[3] = 3;
		1336	} else {
		1337	swizzle_pipe[0] = 0;
		1338	swizzle_pipe[1] = 2;
		1339	swizzle_pipe[2] = 1;
		1340	swizzle_pipe[3] = 3;
		1341	}
		1342	break;
		1343	case 6:
		1344	if (force_no_swizzle) {
		1345	swizzle_pipe[0] = 0;
		1346	swizzle_pipe[1] = 1;
		1347	swizzle_pipe[2] = 2;
		1348	swizzle_pipe[3] = 3;
		1349	swizzle_pipe[4] = 4;
		1350	swizzle_pipe[5] = 5;
		1351	} else {
		1352	swizzle_pipe[0] = 0;
		1353	swizzle_pipe[1] = 2;
		1354	swizzle_pipe[2] = 4;
		1355	swizzle_pipe[3] = 1;
		1356	swizzle_pipe[4] = 3;
		1357	swizzle_pipe[5] = 5;
		1358	}
		1359	break;
		1360	case 8:
		1361	if (force_no_swizzle) {
		1362	swizzle_pipe[0] = 0;
		1363	swizzle_pipe[1] = 1;
		1364	swizzle_pipe[2] = 2;
		1365	swizzle_pipe[3] = 3;
		1366	swizzle_pipe[4] = 4;
		1367	swizzle_pipe[5] = 5;
		1368	swizzle_pipe[6] = 6;
		1369	swizzle_pipe[7] = 7;
		1370	} else {
		1371	swizzle_pipe[0] = 0;
		1372	swizzle_pipe[1] = 2;
		1373	swizzle_pipe[2] = 4;
		1374	swizzle_pipe[3] = 6;
		1375	swizzle_pipe[4] = 1;
		1376	swizzle_pipe[5] = 3;
		1377	swizzle_pipe[6] = 5;
		1378	swizzle_pipe[7] = 7;
		1379	}
		1380	break;
		1381	}
		1382
		1383	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
		1384	while (((1 << cur_backend) & enabled_backends_mask) == 0)
		1385	cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
		1386
		1387	backend_map \|= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4)));
		1388
		1389	cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
		1390	}
		1391
1430	serge	1392	return backend_map;
		1393	}
		1394
1986	serge	1395	static void evergreen_program_channel_remap(struct radeon_device *rdev)
		1396	{
		1397	u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp;
		1398
		1399	tmp = RREG32(MC_SHARED_CHMAP);
		1400	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		1401	case 0:
		1402	case 1:
		1403	case 2:
		1404	case 3:
		1405	default:
		1406	/* default mapping */
		1407	mc_shared_chremap = 0x00fac688;
		1408	break;
		1409	}
		1410
		1411	switch (rdev->family) {
		1412	case CHIP_HEMLOCK:
		1413	case CHIP_CYPRESS:
		1414	case CHIP_BARTS:
		1415	tcp_chan_steer_lo = 0x54763210;
		1416	tcp_chan_steer_hi = 0x0000ba98;
		1417	break;
		1418	case CHIP_JUNIPER:
		1419	case CHIP_REDWOOD:
		1420	case CHIP_CEDAR:
		1421	case CHIP_PALM:
		1422	case CHIP_SUMO:
		1423	case CHIP_SUMO2:
		1424	case CHIP_TURKS:
		1425	case CHIP_CAICOS:
		1426	default:
		1427	tcp_chan_steer_lo = 0x76543210;
		1428	tcp_chan_steer_hi = 0x0000ba98;
		1429	break;
		1430	}
		1431
		1432	WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo);
		1433	WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi);
		1434	WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
		1435	}
		1436
1430	serge	1437	static void evergreen_gpu_init(struct radeon_device *rdev)
		1438	{
1963	serge	1439	u32 cc_rb_backend_disable = 0;
		1440	u32 cc_gc_shader_pipe_config;
		1441	u32 gb_addr_config = 0;
		1442	u32 mc_shared_chmap, mc_arb_ramcfg;
		1443	u32 gb_backend_map;
		1444	u32 grbm_gfx_index;
		1445	u32 sx_debug_1;
		1446	u32 smx_dc_ctl0;
		1447	u32 sq_config;
		1448	u32 sq_lds_resource_mgmt;
		1449	u32 sq_gpr_resource_mgmt_1;
		1450	u32 sq_gpr_resource_mgmt_2;
		1451	u32 sq_gpr_resource_mgmt_3;
		1452	u32 sq_thread_resource_mgmt;
		1453	u32 sq_thread_resource_mgmt_2;
		1454	u32 sq_stack_resource_mgmt_1;
		1455	u32 sq_stack_resource_mgmt_2;
		1456	u32 sq_stack_resource_mgmt_3;
		1457	u32 vgt_cache_invalidation;
		1458	u32 hdp_host_path_cntl, tmp;
		1459	int i, j, num_shader_engines, ps_thread_count;
		1460
		1461	switch (rdev->family) {
		1462	case CHIP_CYPRESS:
		1463	case CHIP_HEMLOCK:
		1464	rdev->config.evergreen.num_ses = 2;
		1465	rdev->config.evergreen.max_pipes = 4;
		1466	rdev->config.evergreen.max_tile_pipes = 8;
		1467	rdev->config.evergreen.max_simds = 10;
		1468	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1469	rdev->config.evergreen.max_gprs = 256;
		1470	rdev->config.evergreen.max_threads = 248;
		1471	rdev->config.evergreen.max_gs_threads = 32;
		1472	rdev->config.evergreen.max_stack_entries = 512;
		1473	rdev->config.evergreen.sx_num_of_sets = 4;
		1474	rdev->config.evergreen.sx_max_export_size = 256;
		1475	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1476	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1477	rdev->config.evergreen.max_hw_contexts = 8;
		1478	rdev->config.evergreen.sq_num_cf_insts = 2;
		1479
		1480	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1481	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1482	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1483	break;
		1484	case CHIP_JUNIPER:
		1485	rdev->config.evergreen.num_ses = 1;
		1486	rdev->config.evergreen.max_pipes = 4;
		1487	rdev->config.evergreen.max_tile_pipes = 4;
		1488	rdev->config.evergreen.max_simds = 10;
		1489	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1490	rdev->config.evergreen.max_gprs = 256;
		1491	rdev->config.evergreen.max_threads = 248;
		1492	rdev->config.evergreen.max_gs_threads = 32;
		1493	rdev->config.evergreen.max_stack_entries = 512;
		1494	rdev->config.evergreen.sx_num_of_sets = 4;
		1495	rdev->config.evergreen.sx_max_export_size = 256;
		1496	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1497	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1498	rdev->config.evergreen.max_hw_contexts = 8;
		1499	rdev->config.evergreen.sq_num_cf_insts = 2;
		1500
		1501	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1502	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1503	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1504	break;
		1505	case CHIP_REDWOOD:
		1506	rdev->config.evergreen.num_ses = 1;
		1507	rdev->config.evergreen.max_pipes = 4;
		1508	rdev->config.evergreen.max_tile_pipes = 4;
		1509	rdev->config.evergreen.max_simds = 5;
		1510	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1511	rdev->config.evergreen.max_gprs = 256;
		1512	rdev->config.evergreen.max_threads = 248;
		1513	rdev->config.evergreen.max_gs_threads = 32;
		1514	rdev->config.evergreen.max_stack_entries = 256;
		1515	rdev->config.evergreen.sx_num_of_sets = 4;
		1516	rdev->config.evergreen.sx_max_export_size = 256;
		1517	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1518	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1519	rdev->config.evergreen.max_hw_contexts = 8;
		1520	rdev->config.evergreen.sq_num_cf_insts = 2;
		1521
		1522	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1523	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1524	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1525	break;
		1526	case CHIP_CEDAR:
		1527	default:
		1528	rdev->config.evergreen.num_ses = 1;
		1529	rdev->config.evergreen.max_pipes = 2;
		1530	rdev->config.evergreen.max_tile_pipes = 2;
		1531	rdev->config.evergreen.max_simds = 2;
		1532	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1533	rdev->config.evergreen.max_gprs = 256;
		1534	rdev->config.evergreen.max_threads = 192;
		1535	rdev->config.evergreen.max_gs_threads = 16;
		1536	rdev->config.evergreen.max_stack_entries = 256;
		1537	rdev->config.evergreen.sx_num_of_sets = 4;
		1538	rdev->config.evergreen.sx_max_export_size = 128;
		1539	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1540	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1541	rdev->config.evergreen.max_hw_contexts = 4;
		1542	rdev->config.evergreen.sq_num_cf_insts = 1;
		1543
		1544	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1545	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1546	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1547	break;
		1548	case CHIP_PALM:
		1549	rdev->config.evergreen.num_ses = 1;
		1550	rdev->config.evergreen.max_pipes = 2;
		1551	rdev->config.evergreen.max_tile_pipes = 2;
		1552	rdev->config.evergreen.max_simds = 2;
		1553	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1554	rdev->config.evergreen.max_gprs = 256;
		1555	rdev->config.evergreen.max_threads = 192;
		1556	rdev->config.evergreen.max_gs_threads = 16;
		1557	rdev->config.evergreen.max_stack_entries = 256;
		1558	rdev->config.evergreen.sx_num_of_sets = 4;
		1559	rdev->config.evergreen.sx_max_export_size = 128;
		1560	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1561	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1562	rdev->config.evergreen.max_hw_contexts = 4;
		1563	rdev->config.evergreen.sq_num_cf_insts = 1;
		1564
		1565	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1566	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1567	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1568	break;
		1569	case CHIP_SUMO:
		1570	rdev->config.evergreen.num_ses = 1;
		1571	rdev->config.evergreen.max_pipes = 4;
		1572	rdev->config.evergreen.max_tile_pipes = 2;
		1573	if (rdev->pdev->device == 0x9648)
		1574	rdev->config.evergreen.max_simds = 3;
		1575	else if ((rdev->pdev->device == 0x9647) \|\|
		1576	(rdev->pdev->device == 0x964a))
		1577	rdev->config.evergreen.max_simds = 4;
		1578	else
		1579	rdev->config.evergreen.max_simds = 5;
		1580	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1581	rdev->config.evergreen.max_gprs = 256;
		1582	rdev->config.evergreen.max_threads = 248;
		1583	rdev->config.evergreen.max_gs_threads = 32;
		1584	rdev->config.evergreen.max_stack_entries = 256;
		1585	rdev->config.evergreen.sx_num_of_sets = 4;
		1586	rdev->config.evergreen.sx_max_export_size = 256;
		1587	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1588	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1589	rdev->config.evergreen.max_hw_contexts = 8;
		1590	rdev->config.evergreen.sq_num_cf_insts = 2;
		1591
		1592	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1593	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1594	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1595	break;
		1596	case CHIP_SUMO2:
		1597	rdev->config.evergreen.num_ses = 1;
		1598	rdev->config.evergreen.max_pipes = 4;
		1599	rdev->config.evergreen.max_tile_pipes = 4;
		1600	rdev->config.evergreen.max_simds = 2;
		1601	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1602	rdev->config.evergreen.max_gprs = 256;
		1603	rdev->config.evergreen.max_threads = 248;
		1604	rdev->config.evergreen.max_gs_threads = 32;
		1605	rdev->config.evergreen.max_stack_entries = 512;
		1606	rdev->config.evergreen.sx_num_of_sets = 4;
		1607	rdev->config.evergreen.sx_max_export_size = 256;
		1608	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1609	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1610	rdev->config.evergreen.max_hw_contexts = 8;
		1611	rdev->config.evergreen.sq_num_cf_insts = 2;
		1612
		1613	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1614	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1615	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1616	break;
		1617	case CHIP_BARTS:
		1618	rdev->config.evergreen.num_ses = 2;
		1619	rdev->config.evergreen.max_pipes = 4;
		1620	rdev->config.evergreen.max_tile_pipes = 8;
		1621	rdev->config.evergreen.max_simds = 7;
		1622	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		1623	rdev->config.evergreen.max_gprs = 256;
		1624	rdev->config.evergreen.max_threads = 248;
		1625	rdev->config.evergreen.max_gs_threads = 32;
		1626	rdev->config.evergreen.max_stack_entries = 512;
		1627	rdev->config.evergreen.sx_num_of_sets = 4;
		1628	rdev->config.evergreen.sx_max_export_size = 256;
		1629	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1630	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1631	rdev->config.evergreen.max_hw_contexts = 8;
		1632	rdev->config.evergreen.sq_num_cf_insts = 2;
		1633
		1634	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1635	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1636	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1637	break;
		1638	case CHIP_TURKS:
		1639	rdev->config.evergreen.num_ses = 1;
		1640	rdev->config.evergreen.max_pipes = 4;
		1641	rdev->config.evergreen.max_tile_pipes = 4;
		1642	rdev->config.evergreen.max_simds = 6;
		1643	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		1644	rdev->config.evergreen.max_gprs = 256;
		1645	rdev->config.evergreen.max_threads = 248;
		1646	rdev->config.evergreen.max_gs_threads = 32;
		1647	rdev->config.evergreen.max_stack_entries = 256;
		1648	rdev->config.evergreen.sx_num_of_sets = 4;
		1649	rdev->config.evergreen.sx_max_export_size = 256;
		1650	rdev->config.evergreen.sx_max_export_pos_size = 64;
		1651	rdev->config.evergreen.sx_max_export_smx_size = 192;
		1652	rdev->config.evergreen.max_hw_contexts = 8;
		1653	rdev->config.evergreen.sq_num_cf_insts = 2;
		1654
		1655	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		1656	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1657	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1658	break;
		1659	case CHIP_CAICOS:
		1660	rdev->config.evergreen.num_ses = 1;
		1661	rdev->config.evergreen.max_pipes = 4;
		1662	rdev->config.evergreen.max_tile_pipes = 2;
		1663	rdev->config.evergreen.max_simds = 2;
		1664	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		1665	rdev->config.evergreen.max_gprs = 256;
		1666	rdev->config.evergreen.max_threads = 192;
		1667	rdev->config.evergreen.max_gs_threads = 16;
		1668	rdev->config.evergreen.max_stack_entries = 256;
		1669	rdev->config.evergreen.sx_num_of_sets = 4;
		1670	rdev->config.evergreen.sx_max_export_size = 128;
		1671	rdev->config.evergreen.sx_max_export_pos_size = 32;
		1672	rdev->config.evergreen.sx_max_export_smx_size = 96;
		1673	rdev->config.evergreen.max_hw_contexts = 4;
		1674	rdev->config.evergreen.sq_num_cf_insts = 1;
		1675
		1676	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		1677	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		1678	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
		1679	break;
		1680	}
		1681
		1682	/* Initialize HDP */
		1683	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1684	WREG32((0x2c14 + j), 0x00000000);
		1685	WREG32((0x2c18 + j), 0x00000000);
		1686	WREG32((0x2c1c + j), 0x00000000);
		1687	WREG32((0x2c20 + j), 0x00000000);
		1688	WREG32((0x2c24 + j), 0x00000000);
		1689	}
		1690
		1691	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1692
		1693	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & ~2;
		1694
		1695	cc_gc_shader_pipe_config \|=
		1696	INACTIVE_QD_PIPES((EVERGREEN_MAX_PIPES_MASK << rdev->config.evergreen.max_pipes)
		1697	& EVERGREEN_MAX_PIPES_MASK);
		1698	cc_gc_shader_pipe_config \|=
		1699	INACTIVE_SIMDS((EVERGREEN_MAX_SIMDS_MASK << rdev->config.evergreen.max_simds)
		1700	& EVERGREEN_MAX_SIMDS_MASK);
		1701
		1702	cc_rb_backend_disable =
		1703	BACKEND_DISABLE((EVERGREEN_MAX_BACKENDS_MASK << rdev->config.evergreen.max_backends)
		1704	& EVERGREEN_MAX_BACKENDS_MASK);
		1705
		1706
		1707	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
		1708	if (rdev->flags & RADEON_IS_IGP)
		1709	mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
		1710	else
		1711	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
		1712
		1713	switch (rdev->config.evergreen.max_tile_pipes) {
		1714	case 1:
		1715	default:
		1716	gb_addr_config \|= NUM_PIPES(0);
		1717	break;
		1718	case 2:
		1719	gb_addr_config \|= NUM_PIPES(1);
		1720	break;
		1721	case 4:
		1722	gb_addr_config \|= NUM_PIPES(2);
		1723	break;
		1724	case 8:
		1725	gb_addr_config \|= NUM_PIPES(3);
		1726	break;
		1727	}
		1728
		1729	gb_addr_config \|= PIPE_INTERLEAVE_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
		1730	gb_addr_config \|= BANK_INTERLEAVE_SIZE(0);
		1731	gb_addr_config \|= NUM_SHADER_ENGINES(rdev->config.evergreen.num_ses - 1);
		1732	gb_addr_config \|= SHADER_ENGINE_TILE_SIZE(1);
		1733	gb_addr_config \|= NUM_GPUS(0); /* Hemlock? */
		1734	gb_addr_config \|= MULTI_GPU_TILE_SIZE(2);
		1735
		1736	if (((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) > 2)
		1737	gb_addr_config \|= ROW_SIZE(2);
		1738	else
		1739	gb_addr_config \|= ROW_SIZE((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT);
		1740
		1741	if (rdev->ddev->pdev->device == 0x689e) {
		1742	u32 efuse_straps_4;
		1743	u32 efuse_straps_3;
		1744	u8 efuse_box_bit_131_124;
		1745
		1746	WREG32(RCU_IND_INDEX, 0x204);
		1747	efuse_straps_4 = RREG32(RCU_IND_DATA);
		1748	WREG32(RCU_IND_INDEX, 0x203);
		1749	efuse_straps_3 = RREG32(RCU_IND_DATA);
		1750	efuse_box_bit_131_124 = (u8)(((efuse_straps_4 & 0xf) << 4) \| ((efuse_straps_3 & 0xf0000000) >> 28));
		1751
		1752	switch(efuse_box_bit_131_124) {
		1753	case 0x00:
		1754	gb_backend_map = 0x76543210;
		1755	break;
		1756	case 0x55:
		1757	gb_backend_map = 0x77553311;
		1758	break;
		1759	case 0x56:
		1760	gb_backend_map = 0x77553300;
		1761	break;
		1762	case 0x59:
		1763	gb_backend_map = 0x77552211;
		1764	break;
		1765	case 0x66:
		1766	gb_backend_map = 0x77443300;
		1767	break;
		1768	case 0x99:
		1769	gb_backend_map = 0x66552211;
		1770	break;
		1771	case 0x5a:
		1772	gb_backend_map = 0x77552200;
		1773	break;
		1774	case 0xaa:
		1775	gb_backend_map = 0x66442200;
		1776	break;
		1777	case 0x95:
		1778	gb_backend_map = 0x66553311;
		1779	break;
		1780	default:
		1781	DRM_ERROR("bad backend map, using default\n");
		1782	gb_backend_map =
		1783	evergreen_get_tile_pipe_to_backend_map(rdev,
		1784	rdev->config.evergreen.max_tile_pipes,
		1785	rdev->config.evergreen.max_backends,
		1786	((EVERGREEN_MAX_BACKENDS_MASK <<
		1787	rdev->config.evergreen.max_backends) &
		1788	EVERGREEN_MAX_BACKENDS_MASK));
		1789	break;
		1790	}
		1791	} else if (rdev->ddev->pdev->device == 0x68b9) {
		1792	u32 efuse_straps_3;
		1793	u8 efuse_box_bit_127_124;
		1794
		1795	WREG32(RCU_IND_INDEX, 0x203);
		1796	efuse_straps_3 = RREG32(RCU_IND_DATA);
		1797	efuse_box_bit_127_124 = (u8)((efuse_straps_3 & 0xF0000000) >> 28);
		1798
		1799	switch(efuse_box_bit_127_124) {
		1800	case 0x0:
		1801	gb_backend_map = 0x00003210;
		1802	break;
		1803	case 0x5:
		1804	case 0x6:
		1805	case 0x9:
		1806	case 0xa:
		1807	gb_backend_map = 0x00003311;
		1808	break;
		1809	default:
		1810	DRM_ERROR("bad backend map, using default\n");
		1811	gb_backend_map =
		1812	evergreen_get_tile_pipe_to_backend_map(rdev,
		1813	rdev->config.evergreen.max_tile_pipes,
		1814	rdev->config.evergreen.max_backends,
		1815	((EVERGREEN_MAX_BACKENDS_MASK <<
		1816	rdev->config.evergreen.max_backends) &
		1817	EVERGREEN_MAX_BACKENDS_MASK));
		1818	break;
		1819	}
		1820	} else {
		1821	switch (rdev->family) {
		1822	case CHIP_CYPRESS:
		1823	case CHIP_HEMLOCK:
		1824	case CHIP_BARTS:
		1825	gb_backend_map = 0x66442200;
		1826	break;
		1827	case CHIP_JUNIPER:
		1828	gb_backend_map = 0x00006420;
		1829	break;
		1830	default:
		1831	gb_backend_map =
		1832	evergreen_get_tile_pipe_to_backend_map(rdev,
		1833	rdev->config.evergreen.max_tile_pipes,
		1834	rdev->config.evergreen.max_backends,
		1835	((EVERGREEN_MAX_BACKENDS_MASK <<
		1836	rdev->config.evergreen.max_backends) &
		1837	EVERGREEN_MAX_BACKENDS_MASK));
		1838	}
		1839	}
		1840
		1841	/* setup tiling info dword. gb_addr_config is not adequate since it does
		1842	* not have bank info, so create a custom tiling dword.
		1843	* bits 3:0 num_pipes
		1844	* bits 7:4 num_banks
		1845	* bits 11:8 group_size
		1846	* bits 15:12 row_size
		1847	*/
		1848	rdev->config.evergreen.tile_config = 0;
		1849	switch (rdev->config.evergreen.max_tile_pipes) {
		1850	case 1:
		1851	default:
		1852	rdev->config.evergreen.tile_config \|= (0 << 0);
		1853	break;
		1854	case 2:
		1855	rdev->config.evergreen.tile_config \|= (1 << 0);
		1856	break;
		1857	case 4:
		1858	rdev->config.evergreen.tile_config \|= (2 << 0);
		1859	break;
		1860	case 8:
		1861	rdev->config.evergreen.tile_config \|= (3 << 0);
		1862	break;
		1863	}
1986	serge	1864	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1963	serge	1865	if (rdev->flags & RADEON_IS_IGP)
1986	serge	1866	rdev->config.evergreen.tile_config \|= 1 << 4;
1963	serge	1867	else
		1868	rdev->config.evergreen.tile_config \|=
		1869	((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
		1870	rdev->config.evergreen.tile_config \|=
		1871	((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT) << 8;
		1872	rdev->config.evergreen.tile_config \|=
		1873	((gb_addr_config & 0x30000000) >> 28) << 12;
		1874
		1875	WREG32(GB_BACKEND_MAP, gb_backend_map);
		1876	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		1877	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
		1878	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
		1879
		1880	evergreen_program_channel_remap(rdev);
		1881
		1882	num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
		1883	grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
		1884
		1885	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
		1886	u32 rb = cc_rb_backend_disable \| (0xf0 << 16);
		1887	u32 sp = cc_gc_shader_pipe_config;
		1888	u32 gfx = grbm_gfx_index \| SE_INDEX(i);
		1889
		1890	if (i == num_shader_engines) {
		1891	rb \|= BACKEND_DISABLE(EVERGREEN_MAX_BACKENDS_MASK);
		1892	sp \|= INACTIVE_SIMDS(EVERGREEN_MAX_SIMDS_MASK);
		1893	}
		1894
		1895	WREG32(GRBM_GFX_INDEX, gfx);
		1896	WREG32(RLC_GFX_INDEX, gfx);
		1897
		1898	WREG32(CC_RB_BACKEND_DISABLE, rb);
		1899	WREG32(CC_SYS_RB_BACKEND_DISABLE, rb);
		1900	WREG32(GC_USER_RB_BACKEND_DISABLE, rb);
		1901	WREG32(CC_GC_SHADER_PIPE_CONFIG, sp);
		1902	}
		1903
		1904	grbm_gfx_index \|= SE_BROADCAST_WRITES;
		1905	WREG32(GRBM_GFX_INDEX, grbm_gfx_index);
		1906	WREG32(RLC_GFX_INDEX, grbm_gfx_index);
		1907
		1908	WREG32(CGTS_SYS_TCC_DISABLE, 0);
		1909	WREG32(CGTS_TCC_DISABLE, 0);
		1910	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
		1911	WREG32(CGTS_USER_TCC_DISABLE, 0);
		1912
		1913	/* set HW defaults for 3D engine */
		1914	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \|
		1915	ROQ_IB2_START(0x2b)));
		1916
		1917	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
		1918
		1919	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \|
		1920	SYNC_GRADIENT \|
		1921	SYNC_WALKER \|
		1922	SYNC_ALIGNER));
		1923
		1924	sx_debug_1 = RREG32(SX_DEBUG_1);
		1925	sx_debug_1 \|= ENABLE_NEW_SMX_ADDRESS;
		1926	WREG32(SX_DEBUG_1, sx_debug_1);
		1927
		1928
		1929	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
		1930	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
		1931	smx_dc_ctl0 \|= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
		1932	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
		1933
		1934	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) \|
		1935	POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) \|
		1936	SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
		1937
		1938	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) \|
		1939	SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) \|
		1940	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
		1941
		1942	WREG32(VGT_NUM_INSTANCES, 1);
		1943	WREG32(SPI_CONFIG_CNTL, 0);
		1944	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
		1945	WREG32(CP_PERFMON_CNTL, 0);
		1946
		1947	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) \|
		1948	FETCH_FIFO_HIWATER(0x4) \|
		1949	DONE_FIFO_HIWATER(0xe0) \|
		1950	ALU_UPDATE_FIFO_HIWATER(0x8)));
		1951
		1952	sq_config = RREG32(SQ_CONFIG);
		1953	sq_config &= ~(PS_PRIO(3) \|
		1954	VS_PRIO(3) \|
		1955	GS_PRIO(3) \|
		1956	ES_PRIO(3));
		1957	sq_config \|= (VC_ENABLE \|
		1958	EXPORT_SRC_C \|
		1959	PS_PRIO(0) \|
		1960	VS_PRIO(1) \|
		1961	GS_PRIO(2) \|
		1962	ES_PRIO(3));
		1963
		1964	switch (rdev->family) {
		1965	case CHIP_CEDAR:
		1966	case CHIP_PALM:
		1967	case CHIP_SUMO:
		1968	case CHIP_SUMO2:
		1969	case CHIP_CAICOS:
		1970	/* no vertex cache */
		1971	sq_config &= ~VC_ENABLE;
		1972	break;
		1973	default:
		1974	break;
		1975	}
		1976
		1977	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
		1978
		1979	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
		1980	sq_gpr_resource_mgmt_1 \|= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
		1981	sq_gpr_resource_mgmt_1 \|= NUM_CLAUSE_TEMP_GPRS(4);
		1982	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		1983	sq_gpr_resource_mgmt_2 \|= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		1984	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		1985	sq_gpr_resource_mgmt_3 \|= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		1986
		1987	switch (rdev->family) {
		1988	case CHIP_CEDAR:
		1989	case CHIP_PALM:
		1990	case CHIP_SUMO:
		1991	case CHIP_SUMO2:
		1992	ps_thread_count = 96;
		1993	break;
		1994	default:
		1995	ps_thread_count = 128;
		1996	break;
		1997	}
		1998
		1999	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
		2000	sq_thread_resource_mgmt \|= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2001	sq_thread_resource_mgmt \|= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2002	sq_thread_resource_mgmt \|= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2003	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2004	sq_thread_resource_mgmt_2 \|= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		2005
		2006	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2007	sq_stack_resource_mgmt_1 \|= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2008	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2009	sq_stack_resource_mgmt_2 \|= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2010	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2011	sq_stack_resource_mgmt_3 \|= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		2012
		2013	WREG32(SQ_CONFIG, sq_config);
		2014	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		2015	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		2016	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
		2017	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		2018	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
		2019	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		2020	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		2021	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
		2022	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
		2023	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
		2024
		2025	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		2026	FORCE_EOV_MAX_REZ_CNT(255)));
		2027
		2028	switch (rdev->family) {
		2029	case CHIP_CEDAR:
		2030	case CHIP_PALM:
		2031	case CHIP_SUMO:
		2032	case CHIP_SUMO2:
		2033	case CHIP_CAICOS:
		2034	vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
		2035	break;
		2036	default:
		2037	vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
		2038	break;
		2039	}
		2040	vgt_cache_invalidation \|= AUTO_INVLD_EN(ES_AND_GS_AUTO);
		2041	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
		2042
		2043	WREG32(VGT_GS_VERTEX_REUSE, 16);
		2044	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
		2045	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		2046
		2047	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
		2048	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
		2049
		2050	WREG32(CB_PERF_CTR0_SEL_0, 0);
		2051	WREG32(CB_PERF_CTR0_SEL_1, 0);
		2052	WREG32(CB_PERF_CTR1_SEL_0, 0);
		2053	WREG32(CB_PERF_CTR1_SEL_1, 0);
		2054	WREG32(CB_PERF_CTR2_SEL_0, 0);
		2055	WREG32(CB_PERF_CTR2_SEL_1, 0);
		2056	WREG32(CB_PERF_CTR3_SEL_0, 0);
		2057	WREG32(CB_PERF_CTR3_SEL_1, 0);
		2058
		2059	/* clear render buffer base addresses */
		2060	WREG32(CB_COLOR0_BASE, 0);
		2061	WREG32(CB_COLOR1_BASE, 0);
		2062	WREG32(CB_COLOR2_BASE, 0);
		2063	WREG32(CB_COLOR3_BASE, 0);
		2064	WREG32(CB_COLOR4_BASE, 0);
		2065	WREG32(CB_COLOR5_BASE, 0);
		2066	WREG32(CB_COLOR6_BASE, 0);
		2067	WREG32(CB_COLOR7_BASE, 0);
		2068	WREG32(CB_COLOR8_BASE, 0);
		2069	WREG32(CB_COLOR9_BASE, 0);
		2070	WREG32(CB_COLOR10_BASE, 0);
		2071	WREG32(CB_COLOR11_BASE, 0);
		2072
		2073	/* set the shader const cache sizes to 0 */
		2074	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
		2075	WREG32(i, 0);
		2076	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
		2077	WREG32(i, 0);
		2078
		2079	tmp = RREG32(HDP_MISC_CNTL);
		2080	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		2081	WREG32(HDP_MISC_CNTL, tmp);
		2082
		2083	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		2084	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		2085
		2086	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		2087
		2088	udelay(50);
		2089
1430	serge	2090	}
		2091
		2092	int evergreen_mc_init(struct radeon_device *rdev)
		2093	{
		2094	u32 tmp;
		2095	int chansize, numchan;
		2096
		2097	/* Get VRAM informations */
		2098	rdev->mc.vram_is_ddr = true;
		2099	tmp = RREG32(MC_ARB_RAMCFG);
		2100	if (tmp & CHANSIZE_OVERRIDE) {
		2101	chansize = 16;
		2102	} else if (tmp & CHANSIZE_MASK) {
		2103	chansize = 64;
		2104	} else {
		2105	chansize = 32;
		2106	}
		2107	tmp = RREG32(MC_SHARED_CHMAP);
		2108	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		2109	case 0:
		2110	default:
		2111	numchan = 1;
		2112	break;
		2113	case 1:
		2114	numchan = 2;
		2115	break;
		2116	case 2:
		2117	numchan = 4;
		2118	break;
		2119	case 3:
		2120	numchan = 8;
		2121	break;
		2122	}
		2123	rdev->mc.vram_width = numchan * chansize;
		2124	/* Could aper size report 0 ? */
1963	serge	2125	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		2126	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1430	serge	2127	/* Setup GPU memory space */
1963	serge	2128	if (rdev->flags & RADEON_IS_IGP) {
		2129	/* size in bytes on fusion */
		2130	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		2131	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
		2132	} else {
1430	serge	2133	/* size in MB on evergreen */
		2134	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
		2135	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
1963	serge	2136	}
1430	serge	2137	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1963	serge	2138	r700_vram_gtt_location(rdev, &rdev->mc);
		2139	radeon_update_bandwidth_info(rdev);
		2140
1430	serge	2141	return 0;
		2142	}
		2143
1963	serge	2144	bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
1430	serge	2145	{
1986	serge	2146	u32 srbm_status;
		2147	u32 grbm_status;
		2148	u32 grbm_status_se0, grbm_status_se1;
		2149	struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
		2150	int r;
		2151
		2152	srbm_status = RREG32(SRBM_STATUS);
		2153	grbm_status = RREG32(GRBM_STATUS);
		2154	grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
		2155	grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
		2156	if (!(grbm_status & GUI_ACTIVE)) {
		2157	r100_gpu_lockup_update(lockup, &rdev->cp);
1963	serge	2158	return false;
1986	serge	2159	}
		2160	/* force CP activities */
		2161	r = radeon_ring_lock(rdev, 2);
		2162	if (!r) {
		2163	/* PACKET2 NOP */
		2164	radeon_ring_write(rdev, 0x80000000);
		2165	radeon_ring_write(rdev, 0x80000000);
		2166	radeon_ring_unlock_commit(rdev);
		2167	}
		2168	rdev->cp.rptr = RREG32(CP_RB_RPTR);
		2169	return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
1963	serge	2170	}
		2171
		2172	static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
		2173	{
		2174	struct evergreen_mc_save save;
		2175	u32 grbm_reset = 0;
		2176
		2177	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
		2178	return 0;
		2179
		2180	dev_info(rdev->dev, "GPU softreset \n");
		2181	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2182	RREG32(GRBM_STATUS));
		2183	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2184	RREG32(GRBM_STATUS_SE0));
		2185	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2186	RREG32(GRBM_STATUS_SE1));
		2187	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2188	RREG32(SRBM_STATUS));
		2189	evergreen_mc_stop(rdev, &save);
		2190	if (evergreen_mc_wait_for_idle(rdev)) {
		2191	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		2192	}
		2193	/* Disable CP parsing/prefetching */
		2194	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT);
		2195
		2196	/* reset all the gfx blocks */
		2197	grbm_reset = (SOFT_RESET_CP \|
		2198	SOFT_RESET_CB \|
		2199	SOFT_RESET_DB \|
		2200	SOFT_RESET_PA \|
		2201	SOFT_RESET_SC \|
		2202	SOFT_RESET_SPI \|
		2203	SOFT_RESET_SH \|
		2204	SOFT_RESET_SX \|
		2205	SOFT_RESET_TC \|
		2206	SOFT_RESET_TA \|
		2207	SOFT_RESET_VC \|
		2208	SOFT_RESET_VGT);
		2209
		2210	dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
		2211	WREG32(GRBM_SOFT_RESET, grbm_reset);
		2212	(void)RREG32(GRBM_SOFT_RESET);
		2213	udelay(50);
		2214	WREG32(GRBM_SOFT_RESET, 0);
		2215	(void)RREG32(GRBM_SOFT_RESET);
		2216	/* Wait a little for things to settle down */
		2217	udelay(50);
		2218	dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
		2219	RREG32(GRBM_STATUS));
		2220	dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
		2221	RREG32(GRBM_STATUS_SE0));
		2222	dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
		2223	RREG32(GRBM_STATUS_SE1));
		2224	dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
		2225	RREG32(SRBM_STATUS));
		2226	evergreen_mc_resume(rdev, &save);
1430	serge	2227	return 0;
		2228	}
		2229
1963	serge	2230	int evergreen_asic_reset(struct radeon_device *rdev)
		2231	{
		2232	return evergreen_gpu_soft_reset(rdev);
		2233	}
		2234
		2235	/* Interrupts */
		2236
		2237	u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
		2238	{
		2239	switch (crtc) {
		2240	case 0:
		2241	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC0_REGISTER_OFFSET);
		2242	case 1:
		2243	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC1_REGISTER_OFFSET);
		2244	case 2:
		2245	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC2_REGISTER_OFFSET);
		2246	case 3:
		2247	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC3_REGISTER_OFFSET);
		2248	case 4:
		2249	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC4_REGISTER_OFFSET);
		2250	case 5:
		2251	return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC5_REGISTER_OFFSET);
		2252	default:
		2253	return 0;
		2254	}
		2255	}
		2256
		2257	void evergreen_disable_interrupt_state(struct radeon_device *rdev)
		2258	{
		2259	u32 tmp;
		2260
		2261	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		2262	WREG32(GRBM_INT_CNTL, 0);
		2263	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2264	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
		2265	if (!(rdev->flags & RADEON_IS_IGP)) {
		2266	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2267	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		2268	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2269	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		2270	}
		2271
		2272	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		2273	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
		2274	if (!(rdev->flags & RADEON_IS_IGP)) {
		2275	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		2276	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		2277	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		2278	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		2279	}
		2280
		2281	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		2282	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2283
		2284	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2285	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2286	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2287	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2288	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2289	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2290	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2291	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2292	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2293	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2294	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2295	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2296
		2297	}
1430	serge	2298	static int evergreen_startup(struct radeon_device *rdev)
		2299	{
		2300	int r;
		2301
		2302	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		2303	r = r600_init_microcode(rdev);
		2304	if (r) {
		2305	DRM_ERROR("Failed to load firmware!\n");
		2306	return r;
		2307	}
		2308	}
1963	serge	2309
1430	serge	2310	evergreen_mc_program(rdev);
		2311	if (rdev->flags & RADEON_IS_AGP) {
1963	serge	2312	evergreen_agp_enable(rdev);
1430	serge	2313	} else {
		2314	r = evergreen_pcie_gart_enable(rdev);
		2315	if (r)
		2316	return r;
		2317	}
		2318	evergreen_gpu_init(rdev);
		2319	#if 0
1963	serge	2320	r = evergreen_blit_init(rdev);
		2321	if (r) {
		2322	evergreen_blit_fini(rdev);
		2323	rdev->asic->copy = NULL;
		2324	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		2325	}
1430	serge	2326
1963	serge	2327	/* allocate wb buffer */
		2328	r = radeon_wb_init(rdev);
		2329	if (r)
		2330	return r;
		2331	#endif
		2332
		2333	/* Enable IRQ */
		2334
1430	serge	2335	r = radeon_ring_init(rdev, rdev->cp.ring_size);
		2336	if (r)
		2337	return r;
		2338	r = evergreen_cp_load_microcode(rdev);
		2339	if (r)
		2340	return r;
1963	serge	2341	r = evergreen_cp_resume(rdev);
1430	serge	2342	if (r)
		2343	return r;
1963	serge	2344
1430	serge	2345	return 0;
		2346	}
		2347
		2348
		2349
		2350
		2351
		2352
		2353
		2354	/* Plan is to move initialization in that function and use
		2355	* helper function so that radeon_device_init pretty much
		2356	* do nothing more than calling asic specific function. This
		2357	* should also allow to remove a bunch of callback function
		2358	* like vram_info.
		2359	*/
		2360	int evergreen_init(struct radeon_device *rdev)
		2361	{
		2362	int r;
		2363
		2364	/* This don't do much */
		2365	r = radeon_gem_init(rdev);
		2366	if (r)
		2367	return r;
		2368	/* Read BIOS */
		2369	if (!radeon_get_bios(rdev)) {
		2370	if (ASIC_IS_AVIVO(rdev))
		2371	return -EINVAL;
		2372	}
		2373	/* Must be an ATOMBIOS */
		2374	if (!rdev->is_atom_bios) {
1986	serge	2375	dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
1430	serge	2376	return -EINVAL;
		2377	}
		2378	r = radeon_atombios_init(rdev);
		2379	if (r)
		2380	return r;
1986	serge	2381	/* reset the asic, the gfx blocks are often in a bad state
		2382	* after the driver is unloaded or after a resume
		2383	*/
		2384	if (radeon_asic_reset(rdev))
		2385	dev_warn(rdev->dev, "GPU reset failed !\n");
1430	serge	2386	/* Post card if necessary */
1986	serge	2387	if (!radeon_card_posted(rdev)) {
1430	serge	2388	if (!rdev->bios) {
		2389	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		2390	return -EINVAL;
		2391	}
		2392	DRM_INFO("GPU not posted. posting now...\n");
		2393	atom_asic_init(rdev->mode_info.atom_context);
		2394	}
		2395	/* Initialize scratch registers */
		2396	r600_scratch_init(rdev);
		2397	/* Initialize surface registers */
		2398	radeon_surface_init(rdev);
		2399	/* Initialize clocks */
		2400	radeon_get_clock_info(rdev->ddev);
		2401	/* Fence driver */
		2402	/* initialize AGP */
		2403	if (rdev->flags & RADEON_IS_AGP) {
		2404	r = radeon_agp_init(rdev);
		2405	if (r)
		2406	radeon_agp_disable(rdev);
		2407	}
		2408	/* initialize memory controller */
		2409	r = evergreen_mc_init(rdev);
		2410	if (r)
		2411	return r;
		2412	/* Memory manager */
		2413	r = radeon_bo_init(rdev);
		2414	if (r)
		2415	return r;
1963	serge	2416
1430	serge	2417
		2418	rdev->cp.ring_obj = NULL;
		2419	r600_ring_init(rdev, 1024 * 1024);
		2420
1963	serge	2421	// rdev->ih.ring_obj = NULL;
		2422	// r600_ih_ring_init(rdev, 64 * 1024);
1430	serge	2423
		2424	r = r600_pcie_gart_init(rdev);
		2425	if (r)
		2426	return r;
1963	serge	2427
		2428	rdev->accel_working = true;
1430	serge	2429	r = evergreen_startup(rdev);
		2430	if (r) {
1963	serge	2431	dev_err(rdev->dev, "disabling GPU acceleration\n");
1430	serge	2432	rdev->accel_working = false;
		2433	}
		2434	if (rdev->accel_working) {
		2435	}
		2436	return 0;
		2437	}
		2438
1986	serge	2439
		2440	static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
1430	serge	2441	{
1986	serge	2442	u32 link_width_cntl, speed_cntl;
		2443
		2444	if (radeon_pcie_gen2 == 0)
		2445	return;
		2446
		2447	if (rdev->flags & RADEON_IS_IGP)
		2448	return;
		2449
		2450	if (!(rdev->flags & RADEON_IS_PCIE))
		2451	return;
		2452
		2453	/* x2 cards have a special sequence */
		2454	if (ASIC_IS_X2(rdev))
		2455	return;
		2456
		2457	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2458	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) \|\|
		2459	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		2460
		2461	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		2462	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		2463	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		2464
		2465	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2466	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		2467	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2468
		2469	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2470	speed_cntl \|= LC_CLR_FAILED_SPD_CHANGE_CNT;
		2471	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2472
		2473	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2474	speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
		2475	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2476
		2477	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		2478	speed_cntl \|= LC_GEN2_EN_STRAP;
		2479	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		2480
		2481	} else {
		2482	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		2483	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		2484	if (1)
		2485	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		2486	else
		2487	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		2488	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		2489	}
1430	serge	2490	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/evergreen.c @ 5179 – Rev 1986