WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/evergreen.c

Rev	Author	Line No.	Line
1430	serge	1	/*
		2	* Copyright 2010 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
1963	serge	25	#include
2997	Serge	26	#include
1430	serge	27	#include "radeon.h"
1963	serge	28	#include "radeon_asic.h"
2997	Serge	29	#include
1963	serge	30	#include "evergreend.h"
1430	serge	31	#include "atom.h"
		32	#include "avivod.h"
		33	#include "evergreen_reg.h"
1986	serge	34	#include "evergreen_blit_shaders.h"
5078	serge	35	#include "radeon_ucode.h"
1430	serge	36
2997	Serge	37	static const u32 crtc_offsets[6] =
		38	{
		39	EVERGREEN_CRTC0_REGISTER_OFFSET,
		40	EVERGREEN_CRTC1_REGISTER_OFFSET,
		41	EVERGREEN_CRTC2_REGISTER_OFFSET,
		42	EVERGREEN_CRTC3_REGISTER_OFFSET,
		43	EVERGREEN_CRTC4_REGISTER_OFFSET,
		44	EVERGREEN_CRTC5_REGISTER_OFFSET
		45	};
		46
5078	serge	47	#include "clearstate_evergreen.h"
		48
		49	static const u32 sumo_rlc_save_restore_register_list[] =
		50	{
		51	0x98fc,
		52	0x9830,
		53	0x9834,
		54	0x9838,
		55	0x9870,
		56	0x9874,
		57	0x8a14,
		58	0x8b24,
		59	0x8bcc,
		60	0x8b10,
		61	0x8d00,
		62	0x8d04,
		63	0x8c00,
		64	0x8c04,
		65	0x8c08,
		66	0x8c0c,
		67	0x8d8c,
		68	0x8c20,
		69	0x8c24,
		70	0x8c28,
		71	0x8c18,
		72	0x8c1c,
		73	0x8cf0,
		74	0x8e2c,
		75	0x8e38,
		76	0x8c30,
		77	0x9508,
		78	0x9688,
		79	0x9608,
		80	0x960c,
		81	0x9610,
		82	0x9614,
		83	0x88c4,
		84	0x88d4,
		85	0xa008,
		86	0x900c,
		87	0x9100,
		88	0x913c,
		89	0x98f8,
		90	0x98f4,
		91	0x9b7c,
		92	0x3f8c,
		93	0x8950,
		94	0x8954,
		95	0x8a18,
		96	0x8b28,
		97	0x9144,
		98	0x9148,
		99	0x914c,
		100	0x3f90,
		101	0x3f94,
		102	0x915c,
		103	0x9160,
		104	0x9178,
		105	0x917c,
		106	0x9180,
		107	0x918c,
		108	0x9190,
		109	0x9194,
		110	0x9198,
		111	0x919c,
		112	0x91a8,
		113	0x91ac,
		114	0x91b0,
		115	0x91b4,
		116	0x91b8,
		117	0x91c4,
		118	0x91c8,
		119	0x91cc,
		120	0x91d0,
		121	0x91d4,
		122	0x91e0,
		123	0x91e4,
		124	0x91ec,
		125	0x91f0,
		126	0x91f4,
		127	0x9200,
		128	0x9204,
		129	0x929c,
		130	0x9150,
		131	0x802c,
		132	};
		133
1430	serge	134	static void evergreen_gpu_init(struct radeon_device *rdev);
		135	void evergreen_fini(struct radeon_device *rdev);
2997	Serge	136	void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
5078	serge	137	void evergreen_program_aspm(struct radeon_device *rdev);
2997	Serge	138	extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
		139	int ring, u32 cp_int_cntl);
5078	serge	140	extern void cayman_vm_decode_fault(struct radeon_device *rdev,
		141	u32 status, u32 addr);
		142	void cik_init_cp_pg_table(struct radeon_device *rdev);
1430	serge	143
5078	serge	144	extern u32 si_get_csb_size(struct radeon_device *rdev);
		145	extern void si_get_csb_buffer(struct radeon_device rdev, volatile u32 buffer);
		146	extern u32 cik_get_csb_size(struct radeon_device *rdev);
		147	extern void cik_get_csb_buffer(struct radeon_device rdev, volatile u32 buffer);
		148	extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
		149
3764	Serge	150	static const u32 evergreen_golden_registers[] =
		151	{
		152	0x3f90, 0xffff0000, 0xff000000,
		153	0x9148, 0xffff0000, 0xff000000,
		154	0x3f94, 0xffff0000, 0xff000000,
		155	0x914c, 0xffff0000, 0xff000000,
		156	0x9b7c, 0xffffffff, 0x00000000,
		157	0x8a14, 0xffffffff, 0x00000007,
		158	0x8b10, 0xffffffff, 0x00000000,
		159	0x960c, 0xffffffff, 0x54763210,
		160	0x88c4, 0xffffffff, 0x000000c2,
		161	0x88d4, 0xffffffff, 0x00000010,
		162	0x8974, 0xffffffff, 0x00000000,
		163	0xc78, 0x00000080, 0x00000080,
		164	0x5eb4, 0xffffffff, 0x00000002,
		165	0x5e78, 0xffffffff, 0x001000f0,
		166	0x6104, 0x01000300, 0x00000000,
		167	0x5bc0, 0x00300000, 0x00000000,
		168	0x7030, 0xffffffff, 0x00000011,
		169	0x7c30, 0xffffffff, 0x00000011,
		170	0x10830, 0xffffffff, 0x00000011,
		171	0x11430, 0xffffffff, 0x00000011,
		172	0x12030, 0xffffffff, 0x00000011,
		173	0x12c30, 0xffffffff, 0x00000011,
		174	0xd02c, 0xffffffff, 0x08421000,
		175	0x240c, 0xffffffff, 0x00000380,
		176	0x8b24, 0xffffffff, 0x00ff0fff,
		177	0x28a4c, 0x06000000, 0x06000000,
		178	0x10c, 0x00000001, 0x00000001,
		179	0x8d00, 0xffffffff, 0x100e4848,
		180	0x8d04, 0xffffffff, 0x00164745,
		181	0x8c00, 0xffffffff, 0xe4000003,
		182	0x8c04, 0xffffffff, 0x40600060,
		183	0x8c08, 0xffffffff, 0x001c001c,
		184	0x8cf0, 0xffffffff, 0x08e00620,
		185	0x8c20, 0xffffffff, 0x00800080,
		186	0x8c24, 0xffffffff, 0x00800080,
		187	0x8c18, 0xffffffff, 0x20202078,
		188	0x8c1c, 0xffffffff, 0x00001010,
		189	0x28350, 0xffffffff, 0x00000000,
		190	0xa008, 0xffffffff, 0x00010000,
5078	serge	191	0x5c4, 0xffffffff, 0x00000001,
3764	Serge	192	0x9508, 0xffffffff, 0x00000002,
		193	0x913c, 0x0000000f, 0x0000000a
		194	};
		195
		196	static const u32 evergreen_golden_registers2[] =
		197	{
		198	0x2f4c, 0xffffffff, 0x00000000,
		199	0x54f4, 0xffffffff, 0x00000000,
		200	0x54f0, 0xffffffff, 0x00000000,
		201	0x5498, 0xffffffff, 0x00000000,
		202	0x549c, 0xffffffff, 0x00000000,
		203	0x5494, 0xffffffff, 0x00000000,
		204	0x53cc, 0xffffffff, 0x00000000,
		205	0x53c8, 0xffffffff, 0x00000000,
		206	0x53c4, 0xffffffff, 0x00000000,
		207	0x53c0, 0xffffffff, 0x00000000,
		208	0x53bc, 0xffffffff, 0x00000000,
		209	0x53b8, 0xffffffff, 0x00000000,
		210	0x53b4, 0xffffffff, 0x00000000,
		211	0x53b0, 0xffffffff, 0x00000000
		212	};
		213
		214	static const u32 cypress_mgcg_init[] =
		215	{
		216	0x802c, 0xffffffff, 0xc0000000,
		217	0x5448, 0xffffffff, 0x00000100,
		218	0x55e4, 0xffffffff, 0x00000100,
		219	0x160c, 0xffffffff, 0x00000100,
		220	0x5644, 0xffffffff, 0x00000100,
		221	0xc164, 0xffffffff, 0x00000100,
		222	0x8a18, 0xffffffff, 0x00000100,
		223	0x897c, 0xffffffff, 0x06000100,
		224	0x8b28, 0xffffffff, 0x00000100,
		225	0x9144, 0xffffffff, 0x00000100,
		226	0x9a60, 0xffffffff, 0x00000100,
		227	0x9868, 0xffffffff, 0x00000100,
		228	0x8d58, 0xffffffff, 0x00000100,
		229	0x9510, 0xffffffff, 0x00000100,
		230	0x949c, 0xffffffff, 0x00000100,
		231	0x9654, 0xffffffff, 0x00000100,
		232	0x9030, 0xffffffff, 0x00000100,
		233	0x9034, 0xffffffff, 0x00000100,
		234	0x9038, 0xffffffff, 0x00000100,
		235	0x903c, 0xffffffff, 0x00000100,
		236	0x9040, 0xffffffff, 0x00000100,
		237	0xa200, 0xffffffff, 0x00000100,
		238	0xa204, 0xffffffff, 0x00000100,
		239	0xa208, 0xffffffff, 0x00000100,
		240	0xa20c, 0xffffffff, 0x00000100,
		241	0x971c, 0xffffffff, 0x00000100,
		242	0x977c, 0xffffffff, 0x00000100,
		243	0x3f80, 0xffffffff, 0x00000100,
		244	0xa210, 0xffffffff, 0x00000100,
		245	0xa214, 0xffffffff, 0x00000100,
		246	0x4d8, 0xffffffff, 0x00000100,
		247	0x9784, 0xffffffff, 0x00000100,
		248	0x9698, 0xffffffff, 0x00000100,
		249	0x4d4, 0xffffffff, 0x00000200,
		250	0x30cc, 0xffffffff, 0x00000100,
		251	0xd0c0, 0xffffffff, 0xff000100,
		252	0x802c, 0xffffffff, 0x40000000,
		253	0x915c, 0xffffffff, 0x00010000,
		254	0x9160, 0xffffffff, 0x00030002,
		255	0x9178, 0xffffffff, 0x00070000,
		256	0x917c, 0xffffffff, 0x00030002,
		257	0x9180, 0xffffffff, 0x00050004,
		258	0x918c, 0xffffffff, 0x00010006,
		259	0x9190, 0xffffffff, 0x00090008,
		260	0x9194, 0xffffffff, 0x00070000,
		261	0x9198, 0xffffffff, 0x00030002,
		262	0x919c, 0xffffffff, 0x00050004,
		263	0x91a8, 0xffffffff, 0x00010006,
		264	0x91ac, 0xffffffff, 0x00090008,
		265	0x91b0, 0xffffffff, 0x00070000,
		266	0x91b4, 0xffffffff, 0x00030002,
		267	0x91b8, 0xffffffff, 0x00050004,
		268	0x91c4, 0xffffffff, 0x00010006,
		269	0x91c8, 0xffffffff, 0x00090008,
		270	0x91cc, 0xffffffff, 0x00070000,
		271	0x91d0, 0xffffffff, 0x00030002,
		272	0x91d4, 0xffffffff, 0x00050004,
		273	0x91e0, 0xffffffff, 0x00010006,
		274	0x91e4, 0xffffffff, 0x00090008,
		275	0x91e8, 0xffffffff, 0x00000000,
		276	0x91ec, 0xffffffff, 0x00070000,
		277	0x91f0, 0xffffffff, 0x00030002,
		278	0x91f4, 0xffffffff, 0x00050004,
		279	0x9200, 0xffffffff, 0x00010006,
		280	0x9204, 0xffffffff, 0x00090008,
		281	0x9208, 0xffffffff, 0x00070000,
		282	0x920c, 0xffffffff, 0x00030002,
		283	0x9210, 0xffffffff, 0x00050004,
		284	0x921c, 0xffffffff, 0x00010006,
		285	0x9220, 0xffffffff, 0x00090008,
		286	0x9224, 0xffffffff, 0x00070000,
		287	0x9228, 0xffffffff, 0x00030002,
		288	0x922c, 0xffffffff, 0x00050004,
		289	0x9238, 0xffffffff, 0x00010006,
		290	0x923c, 0xffffffff, 0x00090008,
		291	0x9240, 0xffffffff, 0x00070000,
		292	0x9244, 0xffffffff, 0x00030002,
		293	0x9248, 0xffffffff, 0x00050004,
		294	0x9254, 0xffffffff, 0x00010006,
		295	0x9258, 0xffffffff, 0x00090008,
		296	0x925c, 0xffffffff, 0x00070000,
		297	0x9260, 0xffffffff, 0x00030002,
		298	0x9264, 0xffffffff, 0x00050004,
		299	0x9270, 0xffffffff, 0x00010006,
		300	0x9274, 0xffffffff, 0x00090008,
		301	0x9278, 0xffffffff, 0x00070000,
		302	0x927c, 0xffffffff, 0x00030002,
		303	0x9280, 0xffffffff, 0x00050004,
		304	0x928c, 0xffffffff, 0x00010006,
		305	0x9290, 0xffffffff, 0x00090008,
		306	0x9294, 0xffffffff, 0x00000000,
		307	0x929c, 0xffffffff, 0x00000001,
		308	0x802c, 0xffffffff, 0x40010000,
		309	0x915c, 0xffffffff, 0x00010000,
		310	0x9160, 0xffffffff, 0x00030002,
		311	0x9178, 0xffffffff, 0x00070000,
		312	0x917c, 0xffffffff, 0x00030002,
		313	0x9180, 0xffffffff, 0x00050004,
		314	0x918c, 0xffffffff, 0x00010006,
		315	0x9190, 0xffffffff, 0x00090008,
		316	0x9194, 0xffffffff, 0x00070000,
		317	0x9198, 0xffffffff, 0x00030002,
		318	0x919c, 0xffffffff, 0x00050004,
		319	0x91a8, 0xffffffff, 0x00010006,
		320	0x91ac, 0xffffffff, 0x00090008,
		321	0x91b0, 0xffffffff, 0x00070000,
		322	0x91b4, 0xffffffff, 0x00030002,
		323	0x91b8, 0xffffffff, 0x00050004,
		324	0x91c4, 0xffffffff, 0x00010006,
		325	0x91c8, 0xffffffff, 0x00090008,
		326	0x91cc, 0xffffffff, 0x00070000,
		327	0x91d0, 0xffffffff, 0x00030002,
		328	0x91d4, 0xffffffff, 0x00050004,
		329	0x91e0, 0xffffffff, 0x00010006,
		330	0x91e4, 0xffffffff, 0x00090008,
		331	0x91e8, 0xffffffff, 0x00000000,
		332	0x91ec, 0xffffffff, 0x00070000,
		333	0x91f0, 0xffffffff, 0x00030002,
		334	0x91f4, 0xffffffff, 0x00050004,
		335	0x9200, 0xffffffff, 0x00010006,
		336	0x9204, 0xffffffff, 0x00090008,
		337	0x9208, 0xffffffff, 0x00070000,
		338	0x920c, 0xffffffff, 0x00030002,
		339	0x9210, 0xffffffff, 0x00050004,
		340	0x921c, 0xffffffff, 0x00010006,
		341	0x9220, 0xffffffff, 0x00090008,
		342	0x9224, 0xffffffff, 0x00070000,
		343	0x9228, 0xffffffff, 0x00030002,
		344	0x922c, 0xffffffff, 0x00050004,
		345	0x9238, 0xffffffff, 0x00010006,
		346	0x923c, 0xffffffff, 0x00090008,
		347	0x9240, 0xffffffff, 0x00070000,
		348	0x9244, 0xffffffff, 0x00030002,
		349	0x9248, 0xffffffff, 0x00050004,
		350	0x9254, 0xffffffff, 0x00010006,
		351	0x9258, 0xffffffff, 0x00090008,
		352	0x925c, 0xffffffff, 0x00070000,
		353	0x9260, 0xffffffff, 0x00030002,
		354	0x9264, 0xffffffff, 0x00050004,
		355	0x9270, 0xffffffff, 0x00010006,
		356	0x9274, 0xffffffff, 0x00090008,
		357	0x9278, 0xffffffff, 0x00070000,
		358	0x927c, 0xffffffff, 0x00030002,
		359	0x9280, 0xffffffff, 0x00050004,
		360	0x928c, 0xffffffff, 0x00010006,
		361	0x9290, 0xffffffff, 0x00090008,
		362	0x9294, 0xffffffff, 0x00000000,
		363	0x929c, 0xffffffff, 0x00000001,
		364	0x802c, 0xffffffff, 0xc0000000
		365	};
		366
		367	static const u32 redwood_mgcg_init[] =
		368	{
		369	0x802c, 0xffffffff, 0xc0000000,
		370	0x5448, 0xffffffff, 0x00000100,
		371	0x55e4, 0xffffffff, 0x00000100,
		372	0x160c, 0xffffffff, 0x00000100,
		373	0x5644, 0xffffffff, 0x00000100,
		374	0xc164, 0xffffffff, 0x00000100,
		375	0x8a18, 0xffffffff, 0x00000100,
		376	0x897c, 0xffffffff, 0x06000100,
		377	0x8b28, 0xffffffff, 0x00000100,
		378	0x9144, 0xffffffff, 0x00000100,
		379	0x9a60, 0xffffffff, 0x00000100,
		380	0x9868, 0xffffffff, 0x00000100,
		381	0x8d58, 0xffffffff, 0x00000100,
		382	0x9510, 0xffffffff, 0x00000100,
		383	0x949c, 0xffffffff, 0x00000100,
		384	0x9654, 0xffffffff, 0x00000100,
		385	0x9030, 0xffffffff, 0x00000100,
		386	0x9034, 0xffffffff, 0x00000100,
		387	0x9038, 0xffffffff, 0x00000100,
		388	0x903c, 0xffffffff, 0x00000100,
		389	0x9040, 0xffffffff, 0x00000100,
		390	0xa200, 0xffffffff, 0x00000100,
		391	0xa204, 0xffffffff, 0x00000100,
		392	0xa208, 0xffffffff, 0x00000100,
		393	0xa20c, 0xffffffff, 0x00000100,
		394	0x971c, 0xffffffff, 0x00000100,
		395	0x977c, 0xffffffff, 0x00000100,
		396	0x3f80, 0xffffffff, 0x00000100,
		397	0xa210, 0xffffffff, 0x00000100,
		398	0xa214, 0xffffffff, 0x00000100,
		399	0x4d8, 0xffffffff, 0x00000100,
		400	0x9784, 0xffffffff, 0x00000100,
		401	0x9698, 0xffffffff, 0x00000100,
		402	0x4d4, 0xffffffff, 0x00000200,
		403	0x30cc, 0xffffffff, 0x00000100,
		404	0xd0c0, 0xffffffff, 0xff000100,
		405	0x802c, 0xffffffff, 0x40000000,
		406	0x915c, 0xffffffff, 0x00010000,
		407	0x9160, 0xffffffff, 0x00030002,
		408	0x9178, 0xffffffff, 0x00070000,
		409	0x917c, 0xffffffff, 0x00030002,
		410	0x9180, 0xffffffff, 0x00050004,
		411	0x918c, 0xffffffff, 0x00010006,
		412	0x9190, 0xffffffff, 0x00090008,
		413	0x9194, 0xffffffff, 0x00070000,
		414	0x9198, 0xffffffff, 0x00030002,
		415	0x919c, 0xffffffff, 0x00050004,
		416	0x91a8, 0xffffffff, 0x00010006,
		417	0x91ac, 0xffffffff, 0x00090008,
		418	0x91b0, 0xffffffff, 0x00070000,
		419	0x91b4, 0xffffffff, 0x00030002,
		420	0x91b8, 0xffffffff, 0x00050004,
		421	0x91c4, 0xffffffff, 0x00010006,
		422	0x91c8, 0xffffffff, 0x00090008,
		423	0x91cc, 0xffffffff, 0x00070000,
		424	0x91d0, 0xffffffff, 0x00030002,
		425	0x91d4, 0xffffffff, 0x00050004,
		426	0x91e0, 0xffffffff, 0x00010006,
		427	0x91e4, 0xffffffff, 0x00090008,
		428	0x91e8, 0xffffffff, 0x00000000,
		429	0x91ec, 0xffffffff, 0x00070000,
		430	0x91f0, 0xffffffff, 0x00030002,
		431	0x91f4, 0xffffffff, 0x00050004,
		432	0x9200, 0xffffffff, 0x00010006,
		433	0x9204, 0xffffffff, 0x00090008,
		434	0x9294, 0xffffffff, 0x00000000,
		435	0x929c, 0xffffffff, 0x00000001,
		436	0x802c, 0xffffffff, 0xc0000000
		437	};
		438
		439	static const u32 cedar_golden_registers[] =
		440	{
		441	0x3f90, 0xffff0000, 0xff000000,
		442	0x9148, 0xffff0000, 0xff000000,
		443	0x3f94, 0xffff0000, 0xff000000,
		444	0x914c, 0xffff0000, 0xff000000,
		445	0x9b7c, 0xffffffff, 0x00000000,
		446	0x8a14, 0xffffffff, 0x00000007,
		447	0x8b10, 0xffffffff, 0x00000000,
		448	0x960c, 0xffffffff, 0x54763210,
		449	0x88c4, 0xffffffff, 0x000000c2,
		450	0x88d4, 0xffffffff, 0x00000000,
		451	0x8974, 0xffffffff, 0x00000000,
		452	0xc78, 0x00000080, 0x00000080,
		453	0x5eb4, 0xffffffff, 0x00000002,
		454	0x5e78, 0xffffffff, 0x001000f0,
		455	0x6104, 0x01000300, 0x00000000,
		456	0x5bc0, 0x00300000, 0x00000000,
		457	0x7030, 0xffffffff, 0x00000011,
		458	0x7c30, 0xffffffff, 0x00000011,
		459	0x10830, 0xffffffff, 0x00000011,
		460	0x11430, 0xffffffff, 0x00000011,
		461	0xd02c, 0xffffffff, 0x08421000,
		462	0x240c, 0xffffffff, 0x00000380,
		463	0x8b24, 0xffffffff, 0x00ff0fff,
		464	0x28a4c, 0x06000000, 0x06000000,
		465	0x10c, 0x00000001, 0x00000001,
		466	0x8d00, 0xffffffff, 0x100e4848,
		467	0x8d04, 0xffffffff, 0x00164745,
		468	0x8c00, 0xffffffff, 0xe4000003,
		469	0x8c04, 0xffffffff, 0x40600060,
		470	0x8c08, 0xffffffff, 0x001c001c,
		471	0x8cf0, 0xffffffff, 0x08e00410,
		472	0x8c20, 0xffffffff, 0x00800080,
		473	0x8c24, 0xffffffff, 0x00800080,
		474	0x8c18, 0xffffffff, 0x20202078,
		475	0x8c1c, 0xffffffff, 0x00001010,
		476	0x28350, 0xffffffff, 0x00000000,
		477	0xa008, 0xffffffff, 0x00010000,
5078	serge	478	0x5c4, 0xffffffff, 0x00000001,
3764	Serge	479	0x9508, 0xffffffff, 0x00000002
		480	};
		481
		482	static const u32 cedar_mgcg_init[] =
		483	{
		484	0x802c, 0xffffffff, 0xc0000000,
		485	0x5448, 0xffffffff, 0x00000100,
		486	0x55e4, 0xffffffff, 0x00000100,
		487	0x160c, 0xffffffff, 0x00000100,
		488	0x5644, 0xffffffff, 0x00000100,
		489	0xc164, 0xffffffff, 0x00000100,
		490	0x8a18, 0xffffffff, 0x00000100,
		491	0x897c, 0xffffffff, 0x06000100,
		492	0x8b28, 0xffffffff, 0x00000100,
		493	0x9144, 0xffffffff, 0x00000100,
		494	0x9a60, 0xffffffff, 0x00000100,
		495	0x9868, 0xffffffff, 0x00000100,
		496	0x8d58, 0xffffffff, 0x00000100,
		497	0x9510, 0xffffffff, 0x00000100,
		498	0x949c, 0xffffffff, 0x00000100,
		499	0x9654, 0xffffffff, 0x00000100,
		500	0x9030, 0xffffffff, 0x00000100,
		501	0x9034, 0xffffffff, 0x00000100,
		502	0x9038, 0xffffffff, 0x00000100,
		503	0x903c, 0xffffffff, 0x00000100,
		504	0x9040, 0xffffffff, 0x00000100,
		505	0xa200, 0xffffffff, 0x00000100,
		506	0xa204, 0xffffffff, 0x00000100,
		507	0xa208, 0xffffffff, 0x00000100,
		508	0xa20c, 0xffffffff, 0x00000100,
		509	0x971c, 0xffffffff, 0x00000100,
		510	0x977c, 0xffffffff, 0x00000100,
		511	0x3f80, 0xffffffff, 0x00000100,
		512	0xa210, 0xffffffff, 0x00000100,
		513	0xa214, 0xffffffff, 0x00000100,
		514	0x4d8, 0xffffffff, 0x00000100,
		515	0x9784, 0xffffffff, 0x00000100,
		516	0x9698, 0xffffffff, 0x00000100,
		517	0x4d4, 0xffffffff, 0x00000200,
		518	0x30cc, 0xffffffff, 0x00000100,
		519	0xd0c0, 0xffffffff, 0xff000100,
		520	0x802c, 0xffffffff, 0x40000000,
		521	0x915c, 0xffffffff, 0x00010000,
		522	0x9178, 0xffffffff, 0x00050000,
		523	0x917c, 0xffffffff, 0x00030002,
		524	0x918c, 0xffffffff, 0x00010004,
		525	0x9190, 0xffffffff, 0x00070006,
		526	0x9194, 0xffffffff, 0x00050000,
		527	0x9198, 0xffffffff, 0x00030002,
		528	0x91a8, 0xffffffff, 0x00010004,
		529	0x91ac, 0xffffffff, 0x00070006,
		530	0x91e8, 0xffffffff, 0x00000000,
		531	0x9294, 0xffffffff, 0x00000000,
		532	0x929c, 0xffffffff, 0x00000001,
		533	0x802c, 0xffffffff, 0xc0000000
		534	};
		535
		536	static const u32 juniper_mgcg_init[] =
		537	{
		538	0x802c, 0xffffffff, 0xc0000000,
		539	0x5448, 0xffffffff, 0x00000100,
		540	0x55e4, 0xffffffff, 0x00000100,
		541	0x160c, 0xffffffff, 0x00000100,
		542	0x5644, 0xffffffff, 0x00000100,
		543	0xc164, 0xffffffff, 0x00000100,
		544	0x8a18, 0xffffffff, 0x00000100,
		545	0x897c, 0xffffffff, 0x06000100,
		546	0x8b28, 0xffffffff, 0x00000100,
		547	0x9144, 0xffffffff, 0x00000100,
		548	0x9a60, 0xffffffff, 0x00000100,
		549	0x9868, 0xffffffff, 0x00000100,
		550	0x8d58, 0xffffffff, 0x00000100,
		551	0x9510, 0xffffffff, 0x00000100,
		552	0x949c, 0xffffffff, 0x00000100,
		553	0x9654, 0xffffffff, 0x00000100,
		554	0x9030, 0xffffffff, 0x00000100,
		555	0x9034, 0xffffffff, 0x00000100,
		556	0x9038, 0xffffffff, 0x00000100,
		557	0x903c, 0xffffffff, 0x00000100,
		558	0x9040, 0xffffffff, 0x00000100,
		559	0xa200, 0xffffffff, 0x00000100,
		560	0xa204, 0xffffffff, 0x00000100,
		561	0xa208, 0xffffffff, 0x00000100,
		562	0xa20c, 0xffffffff, 0x00000100,
		563	0x971c, 0xffffffff, 0x00000100,
		564	0xd0c0, 0xffffffff, 0xff000100,
		565	0x802c, 0xffffffff, 0x40000000,
		566	0x915c, 0xffffffff, 0x00010000,
		567	0x9160, 0xffffffff, 0x00030002,
		568	0x9178, 0xffffffff, 0x00070000,
		569	0x917c, 0xffffffff, 0x00030002,
		570	0x9180, 0xffffffff, 0x00050004,
		571	0x918c, 0xffffffff, 0x00010006,
		572	0x9190, 0xffffffff, 0x00090008,
		573	0x9194, 0xffffffff, 0x00070000,
		574	0x9198, 0xffffffff, 0x00030002,
		575	0x919c, 0xffffffff, 0x00050004,
		576	0x91a8, 0xffffffff, 0x00010006,
		577	0x91ac, 0xffffffff, 0x00090008,
		578	0x91b0, 0xffffffff, 0x00070000,
		579	0x91b4, 0xffffffff, 0x00030002,
		580	0x91b8, 0xffffffff, 0x00050004,
		581	0x91c4, 0xffffffff, 0x00010006,
		582	0x91c8, 0xffffffff, 0x00090008,
		583	0x91cc, 0xffffffff, 0x00070000,
		584	0x91d0, 0xffffffff, 0x00030002,
		585	0x91d4, 0xffffffff, 0x00050004,
		586	0x91e0, 0xffffffff, 0x00010006,
		587	0x91e4, 0xffffffff, 0x00090008,
		588	0x91e8, 0xffffffff, 0x00000000,
		589	0x91ec, 0xffffffff, 0x00070000,
		590	0x91f0, 0xffffffff, 0x00030002,
		591	0x91f4, 0xffffffff, 0x00050004,
		592	0x9200, 0xffffffff, 0x00010006,
		593	0x9204, 0xffffffff, 0x00090008,
		594	0x9208, 0xffffffff, 0x00070000,
		595	0x920c, 0xffffffff, 0x00030002,
		596	0x9210, 0xffffffff, 0x00050004,
		597	0x921c, 0xffffffff, 0x00010006,
		598	0x9220, 0xffffffff, 0x00090008,
		599	0x9224, 0xffffffff, 0x00070000,
		600	0x9228, 0xffffffff, 0x00030002,
		601	0x922c, 0xffffffff, 0x00050004,
		602	0x9238, 0xffffffff, 0x00010006,
		603	0x923c, 0xffffffff, 0x00090008,
		604	0x9240, 0xffffffff, 0x00070000,
		605	0x9244, 0xffffffff, 0x00030002,
		606	0x9248, 0xffffffff, 0x00050004,
		607	0x9254, 0xffffffff, 0x00010006,
		608	0x9258, 0xffffffff, 0x00090008,
		609	0x925c, 0xffffffff, 0x00070000,
		610	0x9260, 0xffffffff, 0x00030002,
		611	0x9264, 0xffffffff, 0x00050004,
		612	0x9270, 0xffffffff, 0x00010006,
		613	0x9274, 0xffffffff, 0x00090008,
		614	0x9278, 0xffffffff, 0x00070000,
		615	0x927c, 0xffffffff, 0x00030002,
		616	0x9280, 0xffffffff, 0x00050004,
		617	0x928c, 0xffffffff, 0x00010006,
		618	0x9290, 0xffffffff, 0x00090008,
		619	0x9294, 0xffffffff, 0x00000000,
		620	0x929c, 0xffffffff, 0x00000001,
		621	0x802c, 0xffffffff, 0xc0000000,
		622	0x977c, 0xffffffff, 0x00000100,
		623	0x3f80, 0xffffffff, 0x00000100,
		624	0xa210, 0xffffffff, 0x00000100,
		625	0xa214, 0xffffffff, 0x00000100,
		626	0x4d8, 0xffffffff, 0x00000100,
		627	0x9784, 0xffffffff, 0x00000100,
		628	0x9698, 0xffffffff, 0x00000100,
		629	0x4d4, 0xffffffff, 0x00000200,
		630	0x30cc, 0xffffffff, 0x00000100,
		631	0x802c, 0xffffffff, 0xc0000000
		632	};
		633
		634	static const u32 supersumo_golden_registers[] =
		635	{
		636	0x5eb4, 0xffffffff, 0x00000002,
5078	serge	637	0x5c4, 0xffffffff, 0x00000001,
3764	Serge	638	0x7030, 0xffffffff, 0x00000011,
		639	0x7c30, 0xffffffff, 0x00000011,
		640	0x6104, 0x01000300, 0x00000000,
		641	0x5bc0, 0x00300000, 0x00000000,
		642	0x8c04, 0xffffffff, 0x40600060,
		643	0x8c08, 0xffffffff, 0x001c001c,
		644	0x8c20, 0xffffffff, 0x00800080,
		645	0x8c24, 0xffffffff, 0x00800080,
		646	0x8c18, 0xffffffff, 0x20202078,
		647	0x8c1c, 0xffffffff, 0x00001010,
		648	0x918c, 0xffffffff, 0x00010006,
		649	0x91a8, 0xffffffff, 0x00010006,
		650	0x91c4, 0xffffffff, 0x00010006,
		651	0x91e0, 0xffffffff, 0x00010006,
		652	0x9200, 0xffffffff, 0x00010006,
		653	0x9150, 0xffffffff, 0x6e944040,
		654	0x917c, 0xffffffff, 0x00030002,
		655	0x9180, 0xffffffff, 0x00050004,
		656	0x9198, 0xffffffff, 0x00030002,
		657	0x919c, 0xffffffff, 0x00050004,
		658	0x91b4, 0xffffffff, 0x00030002,
		659	0x91b8, 0xffffffff, 0x00050004,
		660	0x91d0, 0xffffffff, 0x00030002,
		661	0x91d4, 0xffffffff, 0x00050004,
		662	0x91f0, 0xffffffff, 0x00030002,
		663	0x91f4, 0xffffffff, 0x00050004,
		664	0x915c, 0xffffffff, 0x00010000,
		665	0x9160, 0xffffffff, 0x00030002,
		666	0x3f90, 0xffff0000, 0xff000000,
		667	0x9178, 0xffffffff, 0x00070000,
		668	0x9194, 0xffffffff, 0x00070000,
		669	0x91b0, 0xffffffff, 0x00070000,
		670	0x91cc, 0xffffffff, 0x00070000,
		671	0x91ec, 0xffffffff, 0x00070000,
		672	0x9148, 0xffff0000, 0xff000000,
		673	0x9190, 0xffffffff, 0x00090008,
		674	0x91ac, 0xffffffff, 0x00090008,
		675	0x91c8, 0xffffffff, 0x00090008,
		676	0x91e4, 0xffffffff, 0x00090008,
		677	0x9204, 0xffffffff, 0x00090008,
		678	0x3f94, 0xffff0000, 0xff000000,
		679	0x914c, 0xffff0000, 0xff000000,
		680	0x929c, 0xffffffff, 0x00000001,
		681	0x8a18, 0xffffffff, 0x00000100,
		682	0x8b28, 0xffffffff, 0x00000100,
		683	0x9144, 0xffffffff, 0x00000100,
		684	0x5644, 0xffffffff, 0x00000100,
		685	0x9b7c, 0xffffffff, 0x00000000,
		686	0x8030, 0xffffffff, 0x0000100a,
		687	0x8a14, 0xffffffff, 0x00000007,
		688	0x8b24, 0xffffffff, 0x00ff0fff,
		689	0x8b10, 0xffffffff, 0x00000000,
		690	0x28a4c, 0x06000000, 0x06000000,
		691	0x4d8, 0xffffffff, 0x00000100,
		692	0x913c, 0xffff000f, 0x0100000a,
		693	0x960c, 0xffffffff, 0x54763210,
		694	0x88c4, 0xffffffff, 0x000000c2,
		695	0x88d4, 0xffffffff, 0x00000010,
		696	0x8974, 0xffffffff, 0x00000000,
		697	0xc78, 0x00000080, 0x00000080,
		698	0x5e78, 0xffffffff, 0x001000f0,
		699	0xd02c, 0xffffffff, 0x08421000,
		700	0xa008, 0xffffffff, 0x00010000,
		701	0x8d00, 0xffffffff, 0x100e4848,
		702	0x8d04, 0xffffffff, 0x00164745,
		703	0x8c00, 0xffffffff, 0xe4000003,
		704	0x8cf0, 0x1fffffff, 0x08e00620,
		705	0x28350, 0xffffffff, 0x00000000,
		706	0x9508, 0xffffffff, 0x00000002
		707	};
		708
		709	static const u32 sumo_golden_registers[] =
		710	{
		711	0x900c, 0x00ffffff, 0x0017071f,
		712	0x8c18, 0xffffffff, 0x10101060,
		713	0x8c1c, 0xffffffff, 0x00001010,
		714	0x8c30, 0x0000000f, 0x00000005,
		715	0x9688, 0x0000000f, 0x00000007
		716	};
		717
		718	static const u32 wrestler_golden_registers[] =
		719	{
		720	0x5eb4, 0xffffffff, 0x00000002,
5078	serge	721	0x5c4, 0xffffffff, 0x00000001,
3764	Serge	722	0x7030, 0xffffffff, 0x00000011,
		723	0x7c30, 0xffffffff, 0x00000011,
		724	0x6104, 0x01000300, 0x00000000,
		725	0x5bc0, 0x00300000, 0x00000000,
		726	0x918c, 0xffffffff, 0x00010006,
		727	0x91a8, 0xffffffff, 0x00010006,
		728	0x9150, 0xffffffff, 0x6e944040,
		729	0x917c, 0xffffffff, 0x00030002,
		730	0x9198, 0xffffffff, 0x00030002,
		731	0x915c, 0xffffffff, 0x00010000,
		732	0x3f90, 0xffff0000, 0xff000000,
		733	0x9178, 0xffffffff, 0x00070000,
		734	0x9194, 0xffffffff, 0x00070000,
		735	0x9148, 0xffff0000, 0xff000000,
		736	0x9190, 0xffffffff, 0x00090008,
		737	0x91ac, 0xffffffff, 0x00090008,
		738	0x3f94, 0xffff0000, 0xff000000,
		739	0x914c, 0xffff0000, 0xff000000,
		740	0x929c, 0xffffffff, 0x00000001,
		741	0x8a18, 0xffffffff, 0x00000100,
		742	0x8b28, 0xffffffff, 0x00000100,
		743	0x9144, 0xffffffff, 0x00000100,
		744	0x9b7c, 0xffffffff, 0x00000000,
		745	0x8030, 0xffffffff, 0x0000100a,
		746	0x8a14, 0xffffffff, 0x00000001,
		747	0x8b24, 0xffffffff, 0x00ff0fff,
		748	0x8b10, 0xffffffff, 0x00000000,
		749	0x28a4c, 0x06000000, 0x06000000,
		750	0x4d8, 0xffffffff, 0x00000100,
		751	0x913c, 0xffff000f, 0x0100000a,
		752	0x960c, 0xffffffff, 0x54763210,
		753	0x88c4, 0xffffffff, 0x000000c2,
		754	0x88d4, 0xffffffff, 0x00000010,
		755	0x8974, 0xffffffff, 0x00000000,
		756	0xc78, 0x00000080, 0x00000080,
		757	0x5e78, 0xffffffff, 0x001000f0,
		758	0xd02c, 0xffffffff, 0x08421000,
		759	0xa008, 0xffffffff, 0x00010000,
		760	0x8d00, 0xffffffff, 0x100e4848,
		761	0x8d04, 0xffffffff, 0x00164745,
		762	0x8c00, 0xffffffff, 0xe4000003,
		763	0x8cf0, 0x1fffffff, 0x08e00410,
		764	0x28350, 0xffffffff, 0x00000000,
		765	0x9508, 0xffffffff, 0x00000002,
		766	0x900c, 0xffffffff, 0x0017071f,
		767	0x8c18, 0xffffffff, 0x10101060,
		768	0x8c1c, 0xffffffff, 0x00001010
		769	};
		770
		771	static const u32 barts_golden_registers[] =
		772	{
		773	0x5eb4, 0xffffffff, 0x00000002,
		774	0x5e78, 0x8f311ff1, 0x001000f0,
		775	0x3f90, 0xffff0000, 0xff000000,
		776	0x9148, 0xffff0000, 0xff000000,
		777	0x3f94, 0xffff0000, 0xff000000,
		778	0x914c, 0xffff0000, 0xff000000,
		779	0xc78, 0x00000080, 0x00000080,
		780	0xbd4, 0x70073777, 0x00010001,
		781	0xd02c, 0xbfffff1f, 0x08421000,
		782	0xd0b8, 0x03773777, 0x02011003,
		783	0x5bc0, 0x00200000, 0x50100000,
		784	0x98f8, 0x33773777, 0x02011003,
		785	0x98fc, 0xffffffff, 0x76543210,
		786	0x7030, 0x31000311, 0x00000011,
		787	0x2f48, 0x00000007, 0x02011003,
		788	0x6b28, 0x00000010, 0x00000012,
		789	0x7728, 0x00000010, 0x00000012,
		790	0x10328, 0x00000010, 0x00000012,
		791	0x10f28, 0x00000010, 0x00000012,
		792	0x11b28, 0x00000010, 0x00000012,
		793	0x12728, 0x00000010, 0x00000012,
		794	0x240c, 0x000007ff, 0x00000380,
		795	0x8a14, 0xf000001f, 0x00000007,
		796	0x8b24, 0x3fff3fff, 0x00ff0fff,
		797	0x8b10, 0x0000ff0f, 0x00000000,
		798	0x28a4c, 0x07ffffff, 0x06000000,
		799	0x10c, 0x00000001, 0x00010003,
		800	0xa02c, 0xffffffff, 0x0000009b,
		801	0x913c, 0x0000000f, 0x0100000a,
		802	0x8d00, 0xffff7f7f, 0x100e4848,
		803	0x8d04, 0x00ffffff, 0x00164745,
		804	0x8c00, 0xfffc0003, 0xe4000003,
		805	0x8c04, 0xf8ff00ff, 0x40600060,
		806	0x8c08, 0x00ff00ff, 0x001c001c,
		807	0x8cf0, 0x1fff1fff, 0x08e00620,
		808	0x8c20, 0x0fff0fff, 0x00800080,
		809	0x8c24, 0x0fff0fff, 0x00800080,
		810	0x8c18, 0xffffffff, 0x20202078,
		811	0x8c1c, 0x0000ffff, 0x00001010,
		812	0x28350, 0x00000f01, 0x00000000,
		813	0x9508, 0x3700001f, 0x00000002,
		814	0x960c, 0xffffffff, 0x54763210,
		815	0x88c4, 0x001f3ae3, 0x000000c2,
		816	0x88d4, 0x0000001f, 0x00000010,
		817	0x8974, 0xffffffff, 0x00000000
		818	};
		819
		820	static const u32 turks_golden_registers[] =
		821	{
		822	0x5eb4, 0xffffffff, 0x00000002,
		823	0x5e78, 0x8f311ff1, 0x001000f0,
		824	0x8c8, 0x00003000, 0x00001070,
		825	0x8cc, 0x000fffff, 0x00040035,
		826	0x3f90, 0xffff0000, 0xfff00000,
		827	0x9148, 0xffff0000, 0xfff00000,
		828	0x3f94, 0xffff0000, 0xfff00000,
		829	0x914c, 0xffff0000, 0xfff00000,
		830	0xc78, 0x00000080, 0x00000080,
		831	0xbd4, 0x00073007, 0x00010002,
		832	0xd02c, 0xbfffff1f, 0x08421000,
		833	0xd0b8, 0x03773777, 0x02010002,
		834	0x5bc0, 0x00200000, 0x50100000,
		835	0x98f8, 0x33773777, 0x00010002,
		836	0x98fc, 0xffffffff, 0x33221100,
		837	0x7030, 0x31000311, 0x00000011,
		838	0x2f48, 0x33773777, 0x00010002,
		839	0x6b28, 0x00000010, 0x00000012,
		840	0x7728, 0x00000010, 0x00000012,
		841	0x10328, 0x00000010, 0x00000012,
		842	0x10f28, 0x00000010, 0x00000012,
		843	0x11b28, 0x00000010, 0x00000012,
		844	0x12728, 0x00000010, 0x00000012,
		845	0x240c, 0x000007ff, 0x00000380,
		846	0x8a14, 0xf000001f, 0x00000007,
		847	0x8b24, 0x3fff3fff, 0x00ff0fff,
		848	0x8b10, 0x0000ff0f, 0x00000000,
		849	0x28a4c, 0x07ffffff, 0x06000000,
		850	0x10c, 0x00000001, 0x00010003,
		851	0xa02c, 0xffffffff, 0x0000009b,
		852	0x913c, 0x0000000f, 0x0100000a,
		853	0x8d00, 0xffff7f7f, 0x100e4848,
		854	0x8d04, 0x00ffffff, 0x00164745,
		855	0x8c00, 0xfffc0003, 0xe4000003,
		856	0x8c04, 0xf8ff00ff, 0x40600060,
		857	0x8c08, 0x00ff00ff, 0x001c001c,
		858	0x8cf0, 0x1fff1fff, 0x08e00410,
		859	0x8c20, 0x0fff0fff, 0x00800080,
		860	0x8c24, 0x0fff0fff, 0x00800080,
		861	0x8c18, 0xffffffff, 0x20202078,
		862	0x8c1c, 0x0000ffff, 0x00001010,
		863	0x28350, 0x00000f01, 0x00000000,
		864	0x9508, 0x3700001f, 0x00000002,
		865	0x960c, 0xffffffff, 0x54763210,
		866	0x88c4, 0x001f3ae3, 0x000000c2,
		867	0x88d4, 0x0000001f, 0x00000010,
		868	0x8974, 0xffffffff, 0x00000000
		869	};
		870
		871	static const u32 caicos_golden_registers[] =
		872	{
		873	0x5eb4, 0xffffffff, 0x00000002,
		874	0x5e78, 0x8f311ff1, 0x001000f0,
		875	0x8c8, 0x00003420, 0x00001450,
		876	0x8cc, 0x000fffff, 0x00040035,
		877	0x3f90, 0xffff0000, 0xfffc0000,
		878	0x9148, 0xffff0000, 0xfffc0000,
		879	0x3f94, 0xffff0000, 0xfffc0000,
		880	0x914c, 0xffff0000, 0xfffc0000,
		881	0xc78, 0x00000080, 0x00000080,
		882	0xbd4, 0x00073007, 0x00010001,
		883	0xd02c, 0xbfffff1f, 0x08421000,
		884	0xd0b8, 0x03773777, 0x02010001,
		885	0x5bc0, 0x00200000, 0x50100000,
		886	0x98f8, 0x33773777, 0x02010001,
		887	0x98fc, 0xffffffff, 0x33221100,
		888	0x7030, 0x31000311, 0x00000011,
		889	0x2f48, 0x33773777, 0x02010001,
		890	0x6b28, 0x00000010, 0x00000012,
		891	0x7728, 0x00000010, 0x00000012,
		892	0x10328, 0x00000010, 0x00000012,
		893	0x10f28, 0x00000010, 0x00000012,
		894	0x11b28, 0x00000010, 0x00000012,
		895	0x12728, 0x00000010, 0x00000012,
		896	0x240c, 0x000007ff, 0x00000380,
		897	0x8a14, 0xf000001f, 0x00000001,
		898	0x8b24, 0x3fff3fff, 0x00ff0fff,
		899	0x8b10, 0x0000ff0f, 0x00000000,
		900	0x28a4c, 0x07ffffff, 0x06000000,
		901	0x10c, 0x00000001, 0x00010003,
		902	0xa02c, 0xffffffff, 0x0000009b,
		903	0x913c, 0x0000000f, 0x0100000a,
		904	0x8d00, 0xffff7f7f, 0x100e4848,
		905	0x8d04, 0x00ffffff, 0x00164745,
		906	0x8c00, 0xfffc0003, 0xe4000003,
		907	0x8c04, 0xf8ff00ff, 0x40600060,
		908	0x8c08, 0x00ff00ff, 0x001c001c,
		909	0x8cf0, 0x1fff1fff, 0x08e00410,
		910	0x8c20, 0x0fff0fff, 0x00800080,
		911	0x8c24, 0x0fff0fff, 0x00800080,
		912	0x8c18, 0xffffffff, 0x20202078,
		913	0x8c1c, 0x0000ffff, 0x00001010,
		914	0x28350, 0x00000f01, 0x00000000,
		915	0x9508, 0x3700001f, 0x00000002,
		916	0x960c, 0xffffffff, 0x54763210,
		917	0x88c4, 0x001f3ae3, 0x000000c2,
		918	0x88d4, 0x0000001f, 0x00000010,
		919	0x8974, 0xffffffff, 0x00000000
		920	};
		921
		922	static void evergreen_init_golden_registers(struct radeon_device *rdev)
		923	{
		924	switch (rdev->family) {
		925	case CHIP_CYPRESS:
		926	case CHIP_HEMLOCK:
		927	radeon_program_register_sequence(rdev,
		928	evergreen_golden_registers,
		929	(const u32)ARRAY_SIZE(evergreen_golden_registers));
		930	radeon_program_register_sequence(rdev,
		931	evergreen_golden_registers2,
		932	(const u32)ARRAY_SIZE(evergreen_golden_registers2));
		933	radeon_program_register_sequence(rdev,
		934	cypress_mgcg_init,
		935	(const u32)ARRAY_SIZE(cypress_mgcg_init));
		936	break;
		937	case CHIP_JUNIPER:
		938	radeon_program_register_sequence(rdev,
		939	evergreen_golden_registers,
		940	(const u32)ARRAY_SIZE(evergreen_golden_registers));
		941	radeon_program_register_sequence(rdev,
		942	evergreen_golden_registers2,
		943	(const u32)ARRAY_SIZE(evergreen_golden_registers2));
		944	radeon_program_register_sequence(rdev,
		945	juniper_mgcg_init,
		946	(const u32)ARRAY_SIZE(juniper_mgcg_init));
		947	break;
		948	case CHIP_REDWOOD:
		949	radeon_program_register_sequence(rdev,
		950	evergreen_golden_registers,
		951	(const u32)ARRAY_SIZE(evergreen_golden_registers));
		952	radeon_program_register_sequence(rdev,
		953	evergreen_golden_registers2,
		954	(const u32)ARRAY_SIZE(evergreen_golden_registers2));
		955	radeon_program_register_sequence(rdev,
		956	redwood_mgcg_init,
		957	(const u32)ARRAY_SIZE(redwood_mgcg_init));
		958	break;
		959	case CHIP_CEDAR:
		960	radeon_program_register_sequence(rdev,
		961	cedar_golden_registers,
		962	(const u32)ARRAY_SIZE(cedar_golden_registers));
		963	radeon_program_register_sequence(rdev,
		964	evergreen_golden_registers2,
		965	(const u32)ARRAY_SIZE(evergreen_golden_registers2));
		966	radeon_program_register_sequence(rdev,
		967	cedar_mgcg_init,
		968	(const u32)ARRAY_SIZE(cedar_mgcg_init));
		969	break;
		970	case CHIP_PALM:
		971	radeon_program_register_sequence(rdev,
		972	wrestler_golden_registers,
		973	(const u32)ARRAY_SIZE(wrestler_golden_registers));
		974	break;
		975	case CHIP_SUMO:
		976	radeon_program_register_sequence(rdev,
		977	supersumo_golden_registers,
		978	(const u32)ARRAY_SIZE(supersumo_golden_registers));
		979	break;
		980	case CHIP_SUMO2:
		981	radeon_program_register_sequence(rdev,
		982	supersumo_golden_registers,
		983	(const u32)ARRAY_SIZE(supersumo_golden_registers));
		984	radeon_program_register_sequence(rdev,
		985	sumo_golden_registers,
		986	(const u32)ARRAY_SIZE(sumo_golden_registers));
		987	break;
		988	case CHIP_BARTS:
		989	radeon_program_register_sequence(rdev,
		990	barts_golden_registers,
		991	(const u32)ARRAY_SIZE(barts_golden_registers));
		992	break;
		993	case CHIP_TURKS:
		994	radeon_program_register_sequence(rdev,
		995	turks_golden_registers,
		996	(const u32)ARRAY_SIZE(turks_golden_registers));
		997	break;
		998	case CHIP_CAICOS:
		999	radeon_program_register_sequence(rdev,
		1000	caicos_golden_registers,
		1001	(const u32)ARRAY_SIZE(caicos_golden_registers));
		1002	break;
		1003	default:
		1004	break;
		1005	}
		1006	}
		1007
2997	Serge	1008	void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
		1009	unsigned bankh, unsigned mtaspect,
		1010	unsigned *tile_split)
		1011	{
		1012	*bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
		1013	*bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
		1014	*mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
		1015	*tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
		1016	switch (*bankw) {
		1017	default:
		1018	case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
		1019	case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
		1020	case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
		1021	case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
		1022	}
		1023	switch (*bankh) {
		1024	default:
		1025	case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
		1026	case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
		1027	case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
		1028	case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
		1029	}
		1030	switch (*mtaspect) {
		1031	default:
		1032	case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
		1033	case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
		1034	case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
		1035	case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
		1036	}
		1037	}
1990	serge	1038
3764	Serge	1039	static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
		1040	u32 cntl_reg, u32 status_reg)
		1041	{
		1042	int r, i;
		1043	struct atom_clock_dividers dividers;
		1044
		1045	r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
		1046	clock, false, ÷rs);
		1047	if (r)
		1048	return r;
		1049
		1050	WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN\|DCLK_DIVIDER_MASK));
		1051
		1052	for (i = 0; i < 100; i++) {
		1053	if (RREG32(status_reg) & DCLK_STATUS)
		1054	break;
		1055	mdelay(10);
		1056	}
		1057	if (i == 100)
		1058	return -ETIMEDOUT;
		1059
		1060	return 0;
		1061	}
		1062
		1063	int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
		1064	{
		1065	int r = 0;
		1066	u32 cg_scratch = RREG32(CG_SCRATCH1);
		1067
		1068	r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
		1069	if (r)
		1070	goto done;
		1071	cg_scratch &= 0xffff0000;
		1072	cg_scratch \|= vclk / 100; /* Mhz */
		1073
		1074	r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
		1075	if (r)
		1076	goto done;
		1077	cg_scratch &= 0x0000ffff;
		1078	cg_scratch \|= (dclk / 100) << 16; /* Mhz */
		1079
		1080	done:
		1081	WREG32(CG_SCRATCH1, cg_scratch);
		1082
		1083	return r;
		1084	}
		1085
		1086	int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
		1087	{
		1088	/* start off with something large */
		1089	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
		1090	int r;
		1091
		1092	/* bypass vclk and dclk with bclk */
		1093	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		1094	VCLK_SRC_SEL(1) \| DCLK_SRC_SEL(1),
		1095	~(VCLK_SRC_SEL_MASK \| DCLK_SRC_SEL_MASK));
		1096
		1097	/* put PLL in bypass mode */
		1098	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
		1099
		1100	if (!vclk \|\| !dclk) {
		1101	/* keep the Bypass mode, put PLL to sleep */
		1102	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
		1103	return 0;
		1104	}
		1105
		1106	// r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
		1107	// 16384, 0x03FFFFFF, 0, 128, 5,
		1108	// &fb_div, &vclk_div, &dclk_div);
		1109	if (r)
		1110	return r;
		1111
		1112	/* set VCO_MODE to 1 */
		1113	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
		1114
		1115	/* toggle UPLL_SLEEP to 1 then back to 0 */
		1116	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
		1117	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
		1118
		1119	/* deassert UPLL_RESET */
		1120	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
		1121
		1122	mdelay(1);
		1123
		1124	// r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
		1125	// if (r)
		1126	// return r;
		1127
		1128	/* assert UPLL_RESET again */
		1129	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
		1130
		1131	/* disable spread spectrum. */
		1132	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
		1133
		1134	/* set feedback divider */
		1135	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
		1136
		1137	/* set ref divider to 0 */
		1138	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
		1139
		1140	if (fb_div < 307200)
		1141	WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
		1142	else
		1143	WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
		1144
		1145	/* set PDIV_A and PDIV_B */
		1146	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		1147	UPLL_PDIV_A(vclk_div) \| UPLL_PDIV_B(dclk_div),
		1148	~(UPLL_PDIV_A_MASK \| UPLL_PDIV_B_MASK));
		1149
		1150	/* give the PLL some time to settle */
		1151	mdelay(15);
		1152
		1153	/* deassert PLL_RESET */
		1154	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
		1155
		1156	mdelay(15);
		1157
		1158	/* switch from bypass mode to normal mode */
		1159	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
		1160
		1161	// r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
		1162	// if (r)
		1163	// return r;
		1164
		1165	/* switch VCLK and DCLK selection */
		1166	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		1167	VCLK_SRC_SEL(2) \| DCLK_SRC_SEL(2),
		1168	~(VCLK_SRC_SEL_MASK \| DCLK_SRC_SEL_MASK));
		1169
		1170	mdelay(100);
		1171
		1172	return 0;
		1173	}
		1174
2997	Serge	1175	void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
		1176	{
5078	serge	1177	int readrq;
		1178	u16 v;
1990	serge	1179
5078	serge	1180	readrq = pcie_get_readrq(rdev->pdev);
		1181	v = ffs(readrq) - 8;
2997	Serge	1182	/* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
		1183	* to avoid hangs or perfomance issues
		1184	*/
5078	serge	1185	if ((v == 0) \|\| (v == 6) \|\| (v == 7))
		1186	pcie_set_readrq(rdev->pdev, 512);
		1187	}
		1188
		1189	void dce4_program_fmt(struct drm_encoder *encoder)
		1190	{
		1191	struct drm_device *dev = encoder->dev;
		1192	struct radeon_device *rdev = dev->dev_private;
		1193	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
		1194	struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
		1195	struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
		1196	int bpc = 0;
		1197	u32 tmp = 0;
		1198	enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
		1199
		1200	if (connector) {
		1201	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		1202	bpc = radeon_get_monitor_bpc(connector);
		1203	dither = radeon_connector->dither;
2997	Serge	1204	}
5078	serge	1205
		1206	/* LVDS/eDP FMT is set up by atom */
		1207	if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
		1208	return;
		1209
		1210	/* not needed for analog */
		1211	if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) \|\|
		1212	(radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
		1213	return;
		1214
		1215	if (bpc == 0)
		1216	return;
		1217
		1218	switch (bpc) {
		1219	case 6:
		1220	if (dither == RADEON_FMT_DITHER_ENABLE)
		1221	/* XXX sort out optimal dither settings */
		1222	tmp \|= (FMT_FRAME_RANDOM_ENABLE \| FMT_HIGHPASS_RANDOM_ENABLE \|
		1223	FMT_SPATIAL_DITHER_EN);
		1224	else
		1225	tmp \|= FMT_TRUNCATE_EN;
		1226	break;
		1227	case 8:
		1228	if (dither == RADEON_FMT_DITHER_ENABLE)
		1229	/* XXX sort out optimal dither settings */
		1230	tmp \|= (FMT_FRAME_RANDOM_ENABLE \| FMT_HIGHPASS_RANDOM_ENABLE \|
		1231	FMT_RGB_RANDOM_ENABLE \|
		1232	FMT_SPATIAL_DITHER_EN \| FMT_SPATIAL_DITHER_DEPTH);
		1233	else
		1234	tmp \|= (FMT_TRUNCATE_EN \| FMT_TRUNCATE_DEPTH);
		1235	break;
		1236	case 10:
		1237	default:
		1238	/* not needed */
		1239	break;
		1240	}
		1241
		1242	WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
2997	Serge	1243	}
		1244
3764	Serge	1245	static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
		1246	{
		1247	if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
		1248	return true;
		1249	else
		1250	return false;
		1251	}
		1252
		1253	static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
		1254	{
		1255	u32 pos1, pos2;
		1256
		1257	pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
		1258	pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
		1259
		1260	if (pos1 != pos2)
		1261	return true;
		1262	else
		1263	return false;
		1264	}
		1265
2997	Serge	1266	/**
		1267	* dce4_wait_for_vblank - vblank wait asic callback.
		1268	*
		1269	* @rdev: radeon_device pointer
		1270	* @crtc: crtc to wait for vblank on
		1271	*
		1272	* Wait for vblank on the requested crtc (evergreen+).
		1273	*/
		1274	void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
		1275	{
3764	Serge	1276	unsigned i = 0;
2997	Serge	1277
		1278	if (crtc >= rdev->num_crtc)
		1279	return;
		1280
3764	Serge	1281	if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
		1282	return;
		1283
		1284	/* depending on when we hit vblank, we may be close to active; if so,
		1285	* wait for another frame.
		1286	*/
		1287	while (dce4_is_in_vblank(rdev, crtc)) {
		1288	if (i++ % 100 == 0) {
		1289	if (!dce4_is_counter_moving(rdev, crtc))
2997	Serge	1290	break;
		1291	}
3764	Serge	1292	}
		1293
		1294	while (!dce4_is_in_vblank(rdev, crtc)) {
		1295	if (i++ % 100 == 0) {
		1296	if (!dce4_is_counter_moving(rdev, crtc))
2997	Serge	1297	break;
		1298	}
		1299	}
		1300	}
		1301
		1302	/**
		1303	* evergreen_page_flip - pageflip callback.
		1304	*
		1305	* @rdev: radeon_device pointer
		1306	* @crtc_id: crtc to cleanup pageflip on
		1307	* @crtc_base: new address of the crtc (GPU MC address)
		1308	*
		1309	* Does the actual pageflip (evergreen+).
		1310	* During vblank we take the crtc lock and wait for the update_pending
		1311	* bit to go high, when it does, we release the lock, and allow the
		1312	* double buffered update to take place.
		1313	* Returns the current update pending status.
		1314	*/
5078	serge	1315	void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1990	serge	1316	{
		1317	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
		1318	u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
2997	Serge	1319	int i;
1990	serge	1320
		1321	/* Lock the graphics update lock */
		1322	tmp \|= EVERGREEN_GRPH_UPDATE_LOCK;
		1323	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
		1324
		1325	/* update the scanout addresses */
		1326	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		1327	upper_32_bits(crtc_base));
		1328	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		1329	(u32)crtc_base);
		1330
		1331	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
		1332	upper_32_bits(crtc_base));
		1333	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
		1334	(u32)crtc_base);
		1335
		1336	/* Wait for update_pending to go high. */
2997	Serge	1337	for (i = 0; i < rdev->usec_timeout; i++) {
		1338	if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
		1339	break;
		1340	udelay(1);
		1341	}
1990	serge	1342	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
		1343
		1344	/* Unlock the lock, so double-buffering can take place inside vblank */
		1345	tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
		1346	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
5078	serge	1347	}
1990	serge	1348
5078	serge	1349	/**
		1350	* evergreen_page_flip_pending - check if page flip is still pending
		1351	*
		1352	* @rdev: radeon_device pointer
		1353	* @crtc_id: crtc to check
		1354	*
		1355	* Returns the current update pending status.
		1356	*/
		1357	bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
		1358	{
		1359	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
		1360
1990	serge	1361	/* Return current update_pending status: */
5078	serge	1362	return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
		1363	EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1990	serge	1364	}
		1365
		1366	/* get temperature in millidegrees */
		1367	int evergreen_get_temp(struct radeon_device *rdev)
		1368	{
		1369	u32 temp, toffset;
		1370	int actual_temp = 0;
		1371
		1372	if (rdev->family == CHIP_JUNIPER) {
		1373	toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
		1374	TOFFSET_SHIFT;
		1375	temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
		1376	TS0_ADC_DOUT_SHIFT;
		1377
		1378	if (toffset & 0x100)
		1379	actual_temp = temp / 2 - (0x200 - toffset);
		1380	else
		1381	actual_temp = temp / 2 + toffset;
		1382
		1383	actual_temp = actual_temp * 1000;
		1384
		1385	} else {
		1386	temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
		1387	ASIC_T_SHIFT;
		1388
		1389	if (temp & 0x400)
		1390	actual_temp = -256;
		1391	else if (temp & 0x200)
		1392	actual_temp = 255;
		1393	else if (temp & 0x100) {
		1394	actual_temp = temp & 0x1ff;
		1395	actual_temp \|= ~0x1ff;
		1396	} else
		1397	actual_temp = temp & 0xff;
		1398
		1399	actual_temp = (actual_temp * 1000) / 2;
		1400	}
		1401
		1402	return actual_temp;
		1403	}
		1404
		1405	int sumo_get_temp(struct radeon_device *rdev)
		1406	{
		1407	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
		1408	int actual_temp = temp - 49;
		1409
		1410	return actual_temp * 1000;
		1411	}
		1412
2997	Serge	1413	/**
		1414	* sumo_pm_init_profile - Initialize power profiles callback.
		1415	*
		1416	* @rdev: radeon_device pointer
		1417	*
		1418	* Initialize the power states used in profile mode
		1419	* (sumo, trinity, SI).
		1420	* Used for profile mode only.
		1421	*/
		1422	void sumo_pm_init_profile(struct radeon_device *rdev)
		1423	{
		1424	int idx;
		1425
		1426	/* default */
		1427	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		1428	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		1429	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		1430	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
		1431
		1432	/* low,mid sh/mh */
		1433	if (rdev->flags & RADEON_IS_MOBILITY)
		1434	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
		1435	else
		1436	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		1437
		1438	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
		1439	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
		1440	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		1441	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		1442
		1443	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
		1444	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
		1445	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		1446	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		1447
		1448	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
		1449	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
		1450	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		1451	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
		1452
		1453	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
		1454	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
		1455	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		1456	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
		1457
		1458	/* high sh/mh */
		1459	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		1460	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
		1461	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
		1462	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		1463	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
		1464	rdev->pm.power_state[idx].num_clock_modes - 1;
		1465
		1466	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
		1467	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
		1468	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		1469	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
		1470	rdev->pm.power_state[idx].num_clock_modes - 1;
		1471	}
		1472
		1473	/**
3764	Serge	1474	* btc_pm_init_profile - Initialize power profiles callback.
		1475	*
		1476	* @rdev: radeon_device pointer
		1477	*
		1478	* Initialize the power states used in profile mode
		1479	* (BTC, cayman).
		1480	* Used for profile mode only.
		1481	*/
		1482	void btc_pm_init_profile(struct radeon_device *rdev)
		1483	{
		1484	int idx;
		1485
		1486	/* default */
		1487	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
		1488	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
		1489	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
		1490	rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
		1491	/* starting with BTC, there is one state that is used for both
		1492	* MH and SH. Difference is that we always use the high clock index for
		1493	* mclk.
		1494	*/
		1495	if (rdev->flags & RADEON_IS_MOBILITY)
		1496	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
		1497	else
		1498	idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
		1499	/* low sh */
		1500	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
		1501	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
		1502	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
		1503	rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
		1504	/* mid sh */
		1505	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
		1506	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
		1507	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
		1508	rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
		1509	/* high sh */
		1510	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
		1511	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
		1512	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
		1513	rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
		1514	/* low mh */
		1515	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
		1516	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
		1517	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
		1518	rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
		1519	/* mid mh */
		1520	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
		1521	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
		1522	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
		1523	rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
		1524	/* high mh */
		1525	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
		1526	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
		1527	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
		1528	rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
		1529	}
		1530
		1531	/**
2997	Serge	1532	* evergreen_pm_misc - set additional pm hw parameters callback.
		1533	*
		1534	* @rdev: radeon_device pointer
		1535	*
		1536	* Set non-clock parameters associated with a power state
		1537	* (voltage, etc.) (evergreen+).
		1538	*/
1990	serge	1539	void evergreen_pm_misc(struct radeon_device *rdev)
		1540	{
		1541	int req_ps_idx = rdev->pm.requested_power_state_index;
		1542	int req_cm_idx = rdev->pm.requested_clock_mode_index;
		1543	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
		1544	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
		1545
		1546	if (voltage->type == VOLTAGE_SW) {
5078	serge	1547	/* 0xff0x are flags rather then an actual voltage */
		1548	if ((voltage->voltage & 0xff00) == 0xff00)
1990	serge	1549	return;
		1550	if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
		1551	radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
		1552	rdev->pm.current_vddc = voltage->voltage;
		1553	DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
		1554	}
3764	Serge	1555
		1556	/* starting with BTC, there is one state that is used for both
		1557	* MH and SH. Difference is that we always use the high clock index for
		1558	* mclk and vddci.
		1559	*/
		1560	if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
		1561	(rdev->family >= CHIP_BARTS) &&
		1562	rdev->pm.active_crtc_count &&
		1563	((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) \|\|
		1564	(rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
		1565	voltage = &rdev->pm.power_state[req_ps_idx].
		1566	clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
		1567
5078	serge	1568	/* 0xff0x are flags rather then an actual voltage */
		1569	if ((voltage->vddci & 0xff00) == 0xff00)
1990	serge	1570	return;
		1571	if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
		1572	radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
		1573	rdev->pm.current_vddci = voltage->vddci;
		1574	DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
		1575	}
		1576	}
		1577	}
		1578
2997	Serge	1579	/**
		1580	* evergreen_pm_prepare - pre-power state change callback.
		1581	*
		1582	* @rdev: radeon_device pointer
		1583	*
		1584	* Prepare for a power state change (evergreen+).
		1585	*/
1990	serge	1586	void evergreen_pm_prepare(struct radeon_device *rdev)
		1587	{
		1588	struct drm_device *ddev = rdev->ddev;
		1589	struct drm_crtc *crtc;
		1590	struct radeon_crtc *radeon_crtc;
		1591	u32 tmp;
		1592
		1593	/* disable any active CRTCs */
		1594	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		1595	radeon_crtc = to_radeon_crtc(crtc);
		1596	if (radeon_crtc->enabled) {
		1597	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		1598	tmp \|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		1599	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		1600	}
		1601	}
		1602	}
		1603
2997	Serge	1604	/**
		1605	* evergreen_pm_finish - post-power state change callback.
		1606	*
		1607	* @rdev: radeon_device pointer
		1608	*
		1609	* Clean up after a power state change (evergreen+).
		1610	*/
1990	serge	1611	void evergreen_pm_finish(struct radeon_device *rdev)
		1612	{
		1613	struct drm_device *ddev = rdev->ddev;
		1614	struct drm_crtc *crtc;
		1615	struct radeon_crtc *radeon_crtc;
		1616	u32 tmp;
		1617
		1618	/* enable any active CRTCs */
		1619	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
		1620	radeon_crtc = to_radeon_crtc(crtc);
		1621	if (radeon_crtc->enabled) {
		1622	tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
		1623	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		1624	WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
		1625	}
		1626	}
		1627	}
		1628
2997	Serge	1629	/**
		1630	* evergreen_hpd_sense - hpd sense callback.
		1631	*
		1632	* @rdev: radeon_device pointer
		1633	* @hpd: hpd (hotplug detect) pin
		1634	*
		1635	* Checks if a digital monitor is connected (evergreen+).
		1636	* Returns true if connected, false if not connected.
		1637	*/
1430	serge	1638	bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		1639	{
		1640	bool connected = false;
1963	serge	1641
		1642	switch (hpd) {
		1643	case RADEON_HPD_1:
		1644	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		1645	connected = true;
		1646	break;
		1647	case RADEON_HPD_2:
		1648	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		1649	connected = true;
		1650	break;
		1651	case RADEON_HPD_3:
		1652	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		1653	connected = true;
		1654	break;
		1655	case RADEON_HPD_4:
		1656	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		1657	connected = true;
		1658	break;
		1659	case RADEON_HPD_5:
		1660	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		1661	connected = true;
		1662	break;
		1663	case RADEON_HPD_6:
		1664	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		1665	connected = true;
		1666	break;
		1667	default:
		1668	break;
		1669	}
		1670
1430	serge	1671	return connected;
		1672	}
		1673
2997	Serge	1674	/**
		1675	* evergreen_hpd_set_polarity - hpd set polarity callback.
		1676	*
		1677	* @rdev: radeon_device pointer
		1678	* @hpd: hpd (hotplug detect) pin
		1679	*
		1680	* Set the polarity of the hpd pin (evergreen+).
		1681	*/
1430	serge	1682	void evergreen_hpd_set_polarity(struct radeon_device *rdev,
		1683	enum radeon_hpd_id hpd)
		1684	{
1963	serge	1685	u32 tmp;
		1686	bool connected = evergreen_hpd_sense(rdev, hpd);
		1687
		1688	switch (hpd) {
		1689	case RADEON_HPD_1:
		1690	tmp = RREG32(DC_HPD1_INT_CONTROL);
		1691	if (connected)
		1692	tmp &= ~DC_HPDx_INT_POLARITY;
		1693	else
		1694	tmp \|= DC_HPDx_INT_POLARITY;
		1695	WREG32(DC_HPD1_INT_CONTROL, tmp);
		1696	break;
		1697	case RADEON_HPD_2:
		1698	tmp = RREG32(DC_HPD2_INT_CONTROL);
		1699	if (connected)
		1700	tmp &= ~DC_HPDx_INT_POLARITY;
		1701	else
		1702	tmp \|= DC_HPDx_INT_POLARITY;
		1703	WREG32(DC_HPD2_INT_CONTROL, tmp);
		1704	break;
		1705	case RADEON_HPD_3:
		1706	tmp = RREG32(DC_HPD3_INT_CONTROL);
		1707	if (connected)
		1708	tmp &= ~DC_HPDx_INT_POLARITY;
		1709	else
		1710	tmp \|= DC_HPDx_INT_POLARITY;
		1711	WREG32(DC_HPD3_INT_CONTROL, tmp);
		1712	break;
		1713	case RADEON_HPD_4:
		1714	tmp = RREG32(DC_HPD4_INT_CONTROL);
		1715	if (connected)
		1716	tmp &= ~DC_HPDx_INT_POLARITY;
		1717	else
		1718	tmp \|= DC_HPDx_INT_POLARITY;
		1719	WREG32(DC_HPD4_INT_CONTROL, tmp);
		1720	break;
		1721	case RADEON_HPD_5:
		1722	tmp = RREG32(DC_HPD5_INT_CONTROL);
		1723	if (connected)
		1724	tmp &= ~DC_HPDx_INT_POLARITY;
		1725	else
		1726	tmp \|= DC_HPDx_INT_POLARITY;
		1727	WREG32(DC_HPD5_INT_CONTROL, tmp);
		1728	break;
		1729	case RADEON_HPD_6:
		1730	tmp = RREG32(DC_HPD6_INT_CONTROL);
		1731	if (connected)
		1732	tmp &= ~DC_HPDx_INT_POLARITY;
		1733	else
		1734	tmp \|= DC_HPDx_INT_POLARITY;
		1735	WREG32(DC_HPD6_INT_CONTROL, tmp);
		1736	break;
		1737	default:
		1738	break;
		1739	}
1430	serge	1740	}
		1741
2997	Serge	1742	/**
		1743	* evergreen_hpd_init - hpd setup callback.
		1744	*
		1745	* @rdev: radeon_device pointer
		1746	*
		1747	* Setup the hpd pins used by the card (evergreen+).
		1748	* Enable the pin, set the polarity, and enable the hpd interrupts.
		1749	*/
1430	serge	1750	void evergreen_hpd_init(struct radeon_device *rdev)
		1751	{
1963	serge	1752	struct drm_device *dev = rdev->ddev;
		1753	struct drm_connector *connector;
2997	Serge	1754	unsigned enabled = 0;
1963	serge	1755	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \|
		1756	DC_HPDx_RX_INT_TIMER(0xfa) \| DC_HPDx_EN;
		1757
		1758	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		1759	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
3764	Serge	1760
		1761	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP \|\|
		1762	connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
		1763	/* don't try to enable hpd on eDP or LVDS avoid breaking the
		1764	* aux dp channel on imac and help (but not completely fix)
		1765	* https://bugzilla.redhat.com/show_bug.cgi?id=726143
		1766	* also avoid interrupt storms during dpms.
		1767	*/
		1768	continue;
		1769	}
1963	serge	1770	switch (radeon_connector->hpd.hpd) {
		1771	case RADEON_HPD_1:
		1772	WREG32(DC_HPD1_CONTROL, tmp);
		1773	break;
		1774	case RADEON_HPD_2:
		1775	WREG32(DC_HPD2_CONTROL, tmp);
		1776	break;
		1777	case RADEON_HPD_3:
		1778	WREG32(DC_HPD3_CONTROL, tmp);
		1779	break;
		1780	case RADEON_HPD_4:
		1781	WREG32(DC_HPD4_CONTROL, tmp);
		1782	break;
		1783	case RADEON_HPD_5:
		1784	WREG32(DC_HPD5_CONTROL, tmp);
		1785	break;
		1786	case RADEON_HPD_6:
		1787	WREG32(DC_HPD6_CONTROL, tmp);
		1788	break;
		1789	default:
		1790	break;
		1791	}
2997	Serge	1792	radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
		1793	enabled \|= 1 << radeon_connector->hpd.hpd;
1963	serge	1794	}
2997	Serge	1795	// radeon_irq_kms_enable_hpd(rdev, enabled);
1430	serge	1796	}
		1797
2997	Serge	1798	/**
		1799	* evergreen_hpd_fini - hpd tear down callback.
		1800	*
		1801	* @rdev: radeon_device pointer
		1802	*
		1803	* Tear down the hpd pins used by the card (evergreen+).
		1804	* Disable the hpd interrupts.
		1805	*/
1963	serge	1806	void evergreen_hpd_fini(struct radeon_device *rdev)
		1807	{
		1808	struct drm_device *dev = rdev->ddev;
		1809	struct drm_connector *connector;
2997	Serge	1810	unsigned disabled = 0;
1430	serge	1811
1963	serge	1812	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		1813	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		1814	switch (radeon_connector->hpd.hpd) {
		1815	case RADEON_HPD_1:
		1816	WREG32(DC_HPD1_CONTROL, 0);
		1817	break;
		1818	case RADEON_HPD_2:
		1819	WREG32(DC_HPD2_CONTROL, 0);
		1820	break;
		1821	case RADEON_HPD_3:
		1822	WREG32(DC_HPD3_CONTROL, 0);
		1823	break;
		1824	case RADEON_HPD_4:
		1825	WREG32(DC_HPD4_CONTROL, 0);
		1826	break;
		1827	case RADEON_HPD_5:
		1828	WREG32(DC_HPD5_CONTROL, 0);
		1829	break;
		1830	case RADEON_HPD_6:
		1831	WREG32(DC_HPD6_CONTROL, 0);
		1832	break;
		1833	default:
		1834	break;
		1835	}
2997	Serge	1836	disabled \|= 1 << radeon_connector->hpd.hpd;
1963	serge	1837	}
2997	Serge	1838	// radeon_irq_kms_disable_hpd(rdev, disabled);
1430	serge	1839	}
		1840
1986	serge	1841	/* watermark setup */
1963	serge	1842
1986	serge	1843	static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
		1844	struct radeon_crtc *radeon_crtc,
		1845	struct drm_display_mode *mode,
		1846	struct drm_display_mode *other_mode)
		1847	{
5078	serge	1848	u32 tmp, buffer_alloc, i;
		1849	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1986	serge	1850	/*
		1851	* Line Buffer Setup
		1852	* There are 3 line buffers, each one shared by 2 display controllers.
		1853	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
		1854	* the display controllers. The paritioning is done via one of four
		1855	* preset allocations specified in bits 2:0:
		1856	* first display controller
		1857	* 0 - first half of lb (3840 * 2)
		1858	* 1 - first 3/4 of lb (5760 * 2)
		1859	* 2 - whole lb (7680 * 2), other crtc must be disabled
		1860	* 3 - first 1/4 of lb (1920 * 2)
		1861	* second display controller
		1862	* 4 - second half of lb (3840 * 2)
		1863	* 5 - second 3/4 of lb (5760 * 2)
		1864	* 6 - whole lb (7680 * 2), other crtc must be disabled
		1865	* 7 - last 1/4 of lb (1920 * 2)
		1866	*/
		1867	/* this can get tricky if we have two large displays on a paired group
		1868	* of crtcs. Ideally for multiple large displays we'd assign them to
		1869	* non-linked crtcs for maximum line buffer allocation.
		1870	*/
		1871	if (radeon_crtc->base.enabled && mode) {
5078	serge	1872	if (other_mode) {
1986	serge	1873	tmp = 0; /* 1/2 */
5078	serge	1874	buffer_alloc = 1;
		1875	} else {
1986	serge	1876	tmp = 2; /* whole */
5078	serge	1877	buffer_alloc = 2;
		1878	}
		1879	} else {
1986	serge	1880	tmp = 0;
5078	serge	1881	buffer_alloc = 0;
		1882	}
1963	serge	1883
1986	serge	1884	/* second controller of the pair uses second half of the lb */
		1885	if (radeon_crtc->crtc_id % 2)
		1886	tmp += 4;
		1887	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
		1888
5078	serge	1889	if (ASIC_IS_DCE41(rdev) \|\| ASIC_IS_DCE5(rdev)) {
		1890	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
		1891	DMIF_BUFFERS_ALLOCATED(buffer_alloc));
		1892	for (i = 0; i < rdev->usec_timeout; i++) {
		1893	if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
		1894	DMIF_BUFFERS_ALLOCATED_COMPLETED)
		1895	break;
		1896	udelay(1);
		1897	}
		1898	}
		1899
1986	serge	1900	if (radeon_crtc->base.enabled && mode) {
		1901	switch (tmp) {
		1902	case 0:
		1903	case 4:
		1904	default:
		1905	if (ASIC_IS_DCE5(rdev))
		1906	return 4096 * 2;
		1907	else
		1908	return 3840 * 2;
		1909	case 1:
		1910	case 5:
		1911	if (ASIC_IS_DCE5(rdev))
		1912	return 6144 * 2;
		1913	else
		1914	return 5760 * 2;
		1915	case 2:
		1916	case 6:
		1917	if (ASIC_IS_DCE5(rdev))
		1918	return 8192 * 2;
		1919	else
		1920	return 7680 * 2;
		1921	case 3:
		1922	case 7:
		1923	if (ASIC_IS_DCE5(rdev))
		1924	return 2048 * 2;
		1925	else
		1926	return 1920 * 2;
		1927	}
		1928	}
		1929
		1930	/* controller not enabled, so no lb used */
		1931	return 0;
		1932	}
		1933
2997	Serge	1934	u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1986	serge	1935	{
		1936	u32 tmp = RREG32(MC_SHARED_CHMAP);
		1937
		1938	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		1939	case 0:
		1940	default:
		1941	return 1;
		1942	case 1:
		1943	return 2;
		1944	case 2:
		1945	return 4;
		1946	case 3:
		1947	return 8;
		1948	}
		1949	}
		1950
		1951	struct evergreen_wm_params {
		1952	u32 dram_channels; /* number of dram channels */
		1953	u32 yclk; /* bandwidth per dram data pin in kHz */
		1954	u32 sclk; /* engine clock in kHz */
		1955	u32 disp_clk; /* display clock in kHz */
		1956	u32 src_width; /* viewport width */
		1957	u32 active_time; /* active display time in ns */
		1958	u32 blank_time; /* blank time in ns */
		1959	bool interlaced; /* mode is interlaced */
		1960	fixed20_12 vsc; /* vertical scale ratio */
		1961	u32 num_heads; /* number of active crtcs */
		1962	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
		1963	u32 lb_size; /* line buffer allocated to pipe */
		1964	u32 vtaps; /* vertical scaler taps */
		1965	};
		1966
		1967	static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
		1968	{
		1969	/* Calculate DRAM Bandwidth and the part allocated to display. */
		1970	fixed20_12 dram_efficiency; /* 0.7 */
		1971	fixed20_12 yclk, dram_channels, bandwidth;
		1972	fixed20_12 a;
		1973
		1974	a.full = dfixed_const(1000);
		1975	yclk.full = dfixed_const(wm->yclk);
		1976	yclk.full = dfixed_div(yclk, a);
		1977	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		1978	a.full = dfixed_const(10);
		1979	dram_efficiency.full = dfixed_const(7);
		1980	dram_efficiency.full = dfixed_div(dram_efficiency, a);
		1981	bandwidth.full = dfixed_mul(dram_channels, yclk);
		1982	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
		1983
		1984	return dfixed_trunc(bandwidth);
		1985	}
		1986
		1987	static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		1988	{
		1989	/* Calculate DRAM Bandwidth and the part allocated to display. */
		1990	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
		1991	fixed20_12 yclk, dram_channels, bandwidth;
		1992	fixed20_12 a;
		1993
		1994	a.full = dfixed_const(1000);
		1995	yclk.full = dfixed_const(wm->yclk);
		1996	yclk.full = dfixed_div(yclk, a);
		1997	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		1998	a.full = dfixed_const(10);
		1999	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
		2000	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
		2001	bandwidth.full = dfixed_mul(dram_channels, yclk);
		2002	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
		2003
		2004	return dfixed_trunc(bandwidth);
		2005	}
		2006
		2007	static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
		2008	{
		2009	/* Calculate the display Data return Bandwidth */
		2010	fixed20_12 return_efficiency; /* 0.8 */
		2011	fixed20_12 sclk, bandwidth;
		2012	fixed20_12 a;
		2013
		2014	a.full = dfixed_const(1000);
		2015	sclk.full = dfixed_const(wm->sclk);
		2016	sclk.full = dfixed_div(sclk, a);
		2017	a.full = dfixed_const(10);
		2018	return_efficiency.full = dfixed_const(8);
		2019	return_efficiency.full = dfixed_div(return_efficiency, a);
		2020	a.full = dfixed_const(32);
		2021	bandwidth.full = dfixed_mul(a, sclk);
		2022	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
		2023
		2024	return dfixed_trunc(bandwidth);
		2025	}
		2026
		2027	static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
		2028	{
		2029	/* Calculate the DMIF Request Bandwidth */
		2030	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
		2031	fixed20_12 disp_clk, bandwidth;
		2032	fixed20_12 a;
		2033
		2034	a.full = dfixed_const(1000);
		2035	disp_clk.full = dfixed_const(wm->disp_clk);
		2036	disp_clk.full = dfixed_div(disp_clk, a);
		2037	a.full = dfixed_const(10);
		2038	disp_clk_request_efficiency.full = dfixed_const(8);
		2039	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
		2040	a.full = dfixed_const(32);
		2041	bandwidth.full = dfixed_mul(a, disp_clk);
		2042	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
		2043
		2044	return dfixed_trunc(bandwidth);
		2045	}
		2046
		2047	static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
		2048	{
		2049	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
		2050	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
		2051	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
		2052	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
		2053
		2054	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
		2055	}
		2056
		2057	static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
		2058	{
		2059	/* Calculate the display mode Average Bandwidth
		2060	* DisplayMode should contain the source and destination dimensions,
		2061	* timing, etc.
		2062	*/
		2063	fixed20_12 bpp;
		2064	fixed20_12 line_time;
		2065	fixed20_12 src_width;
		2066	fixed20_12 bandwidth;
		2067	fixed20_12 a;
		2068
		2069	a.full = dfixed_const(1000);
		2070	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
		2071	line_time.full = dfixed_div(line_time, a);
		2072	bpp.full = dfixed_const(wm->bytes_per_pixel);
		2073	src_width.full = dfixed_const(wm->src_width);
		2074	bandwidth.full = dfixed_mul(src_width, bpp);
		2075	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
		2076	bandwidth.full = dfixed_div(bandwidth, line_time);
		2077
		2078	return dfixed_trunc(bandwidth);
		2079	}
		2080
		2081	static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
		2082	{
		2083	/* First calcualte the latency in ns */
		2084	u32 mc_latency = 2000; /* 2000 ns. */
		2085	u32 available_bandwidth = evergreen_available_bandwidth(wm);
		2086	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
		2087	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
		2088	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
		2089	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
		2090	(wm->num_heads * cursor_line_pair_return_time);
		2091	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
		2092	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
		2093	fixed20_12 a, b, c;
		2094
		2095	if (wm->num_heads == 0)
		2096	return 0;
		2097
		2098	a.full = dfixed_const(2);
		2099	b.full = dfixed_const(1);
		2100	if ((wm->vsc.full > a.full) \|\|
		2101	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) \|\|
		2102	(wm->vtaps >= 5) \|\|
		2103	((wm->vsc.full >= a.full) && wm->interlaced))
		2104	max_src_lines_per_dst_line = 4;
		2105	else
		2106	max_src_lines_per_dst_line = 2;
		2107
		2108	a.full = dfixed_const(available_bandwidth);
		2109	b.full = dfixed_const(wm->num_heads);
		2110	a.full = dfixed_div(a, b);
		2111
		2112	b.full = dfixed_const(1000);
		2113	c.full = dfixed_const(wm->disp_clk);
		2114	b.full = dfixed_div(c, b);
		2115	c.full = dfixed_const(wm->bytes_per_pixel);
		2116	b.full = dfixed_mul(b, c);
		2117
		2118	lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
		2119
		2120	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
		2121	b.full = dfixed_const(1000);
		2122	c.full = dfixed_const(lb_fill_bw);
		2123	b.full = dfixed_div(c, b);
		2124	a.full = dfixed_div(a, b);
		2125	line_fill_time = dfixed_trunc(a);
		2126
		2127	if (line_fill_time < wm->active_time)
		2128	return latency;
		2129	else
		2130	return latency + (line_fill_time - wm->active_time);
		2131
		2132	}
		2133
		2134	static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
		2135	{
		2136	if (evergreen_average_bandwidth(wm) <=
		2137	(evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
		2138	return true;
		2139	else
		2140	return false;
		2141	};
		2142
		2143	static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
		2144	{
		2145	if (evergreen_average_bandwidth(wm) <=
		2146	(evergreen_available_bandwidth(wm) / wm->num_heads))
		2147	return true;
		2148	else
		2149	return false;
		2150	};
		2151
		2152	static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
		2153	{
		2154	u32 lb_partitions = wm->lb_size / wm->src_width;
		2155	u32 line_time = wm->active_time + wm->blank_time;
		2156	u32 latency_tolerant_lines;
		2157	u32 latency_hiding;
		2158	fixed20_12 a;
		2159
		2160	a.full = dfixed_const(1);
		2161	if (wm->vsc.full > a.full)
		2162	latency_tolerant_lines = 1;
		2163	else {
		2164	if (lb_partitions <= (wm->vtaps + 1))
		2165	latency_tolerant_lines = 1;
		2166	else
		2167	latency_tolerant_lines = 2;
		2168	}
		2169
		2170	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
		2171
		2172	if (evergreen_latency_watermark(wm) <= latency_hiding)
		2173	return true;
		2174	else
		2175	return false;
		2176	}
		2177
		2178	static void evergreen_program_watermarks(struct radeon_device *rdev,
		2179	struct radeon_crtc *radeon_crtc,
		2180	u32 lb_size, u32 num_heads)
		2181	{
		2182	struct drm_display_mode *mode = &radeon_crtc->base.mode;
5078	serge	2183	struct evergreen_wm_params wm_low, wm_high;
		2184	u32 dram_channels;
1986	serge	2185	u32 pixel_period;
		2186	u32 line_time = 0;
		2187	u32 latency_watermark_a = 0, latency_watermark_b = 0;
		2188	u32 priority_a_mark = 0, priority_b_mark = 0;
		2189	u32 priority_a_cnt = PRIORITY_OFF;
		2190	u32 priority_b_cnt = PRIORITY_OFF;
		2191	u32 pipe_offset = radeon_crtc->crtc_id * 16;
		2192	u32 tmp, arb_control3;
		2193	fixed20_12 a, b, c;
		2194
		2195	if (radeon_crtc->base.enabled && num_heads && mode) {
		2196	pixel_period = 1000000 / (u32)mode->clock;
		2197	line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
		2198	priority_a_cnt = 0;
		2199	priority_b_cnt = 0;
5078	serge	2200	dram_channels = evergreen_get_number_of_dram_channels(rdev);
1986	serge	2201
5078	serge	2202	/* watermark for high clocks */
		2203	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
		2204	wm_high.yclk =
		2205	radeon_dpm_get_mclk(rdev, false) * 10;
		2206	wm_high.sclk =
		2207	radeon_dpm_get_sclk(rdev, false) * 10;
		2208	} else {
		2209	wm_high.yclk = rdev->pm.current_mclk * 10;
		2210	wm_high.sclk = rdev->pm.current_sclk * 10;
		2211	}
		2212
		2213	wm_high.disp_clk = mode->clock;
		2214	wm_high.src_width = mode->crtc_hdisplay;
		2215	wm_high.active_time = mode->crtc_hdisplay * pixel_period;
		2216	wm_high.blank_time = line_time - wm_high.active_time;
		2217	wm_high.interlaced = false;
1986	serge	2218	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
5078	serge	2219	wm_high.interlaced = true;
		2220	wm_high.vsc = radeon_crtc->vsc;
		2221	wm_high.vtaps = 1;
1986	serge	2222	if (radeon_crtc->rmx_type != RMX_OFF)
5078	serge	2223	wm_high.vtaps = 2;
		2224	wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
		2225	wm_high.lb_size = lb_size;
		2226	wm_high.dram_channels = dram_channels;
		2227	wm_high.num_heads = num_heads;
1986	serge	2228
5078	serge	2229	/* watermark for low clocks */
		2230	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
		2231	wm_low.yclk =
		2232	radeon_dpm_get_mclk(rdev, true) * 10;
		2233	wm_low.sclk =
		2234	radeon_dpm_get_sclk(rdev, true) * 10;
		2235	} else {
		2236	wm_low.yclk = rdev->pm.current_mclk * 10;
		2237	wm_low.sclk = rdev->pm.current_sclk * 10;
		2238	}
		2239
		2240	wm_low.disp_clk = mode->clock;
		2241	wm_low.src_width = mode->crtc_hdisplay;
		2242	wm_low.active_time = mode->crtc_hdisplay * pixel_period;
		2243	wm_low.blank_time = line_time - wm_low.active_time;
		2244	wm_low.interlaced = false;
		2245	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		2246	wm_low.interlaced = true;
		2247	wm_low.vsc = radeon_crtc->vsc;
		2248	wm_low.vtaps = 1;
		2249	if (radeon_crtc->rmx_type != RMX_OFF)
		2250	wm_low.vtaps = 2;
		2251	wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
		2252	wm_low.lb_size = lb_size;
		2253	wm_low.dram_channels = dram_channels;
		2254	wm_low.num_heads = num_heads;
		2255
1986	serge	2256	/* set for high clocks */
5078	serge	2257	latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
1986	serge	2258	/* set for low clocks */
5078	serge	2259	latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
1986	serge	2260
		2261	/* possibly force display priority to high */
		2262	/* should really do this at mode validation time... */
5078	serge	2263	if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) \|\|
		2264	!evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) \|\|
		2265	!evergreen_check_latency_hiding(&wm_high) \|\|
1986	serge	2266	(rdev->disp_priority == 2)) {
5078	serge	2267	DRM_DEBUG_KMS("force priority a to high\n");
1986	serge	2268	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
5078	serge	2269	}
		2270	if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) \|\|
		2271	!evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) \|\|
		2272	!evergreen_check_latency_hiding(&wm_low) \|\|
		2273	(rdev->disp_priority == 2)) {
		2274	DRM_DEBUG_KMS("force priority b to high\n");
1986	serge	2275	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		2276	}
		2277
		2278	a.full = dfixed_const(1000);
		2279	b.full = dfixed_const(mode->clock);
		2280	b.full = dfixed_div(b, a);
		2281	c.full = dfixed_const(latency_watermark_a);
		2282	c.full = dfixed_mul(c, b);
		2283	c.full = dfixed_mul(c, radeon_crtc->hsc);
		2284	c.full = dfixed_div(c, a);
		2285	a.full = dfixed_const(16);
		2286	c.full = dfixed_div(c, a);
		2287	priority_a_mark = dfixed_trunc(c);
		2288	priority_a_cnt \|= priority_a_mark & PRIORITY_MARK_MASK;
		2289
		2290	a.full = dfixed_const(1000);
		2291	b.full = dfixed_const(mode->clock);
		2292	b.full = dfixed_div(b, a);
		2293	c.full = dfixed_const(latency_watermark_b);
		2294	c.full = dfixed_mul(c, b);
		2295	c.full = dfixed_mul(c, radeon_crtc->hsc);
		2296	c.full = dfixed_div(c, a);
		2297	a.full = dfixed_const(16);
		2298	c.full = dfixed_div(c, a);
		2299	priority_b_mark = dfixed_trunc(c);
		2300	priority_b_cnt \|= priority_b_mark & PRIORITY_MARK_MASK;
		2301	}
		2302
		2303	/* select wm A */
		2304	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		2305	tmp = arb_control3;
		2306	tmp &= ~LATENCY_WATERMARK_MASK(3);
		2307	tmp \|= LATENCY_WATERMARK_MASK(1);
		2308	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		2309	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		2310	(LATENCY_LOW_WATERMARK(latency_watermark_a) \|
		2311	LATENCY_HIGH_WATERMARK(line_time)));
		2312	/* select wm B */
		2313	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
		2314	tmp &= ~LATENCY_WATERMARK_MASK(3);
		2315	tmp \|= LATENCY_WATERMARK_MASK(2);
		2316	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
		2317	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
		2318	(LATENCY_LOW_WATERMARK(latency_watermark_b) \|
		2319	LATENCY_HIGH_WATERMARK(line_time)));
		2320	/* restore original selection */
		2321	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
		2322
		2323	/* write the priority marks */
		2324	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
		2325	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
		2326
5078	serge	2327	/* save values for DPM */
		2328	radeon_crtc->line_time = line_time;
		2329	radeon_crtc->wm_high = latency_watermark_a;
		2330	radeon_crtc->wm_low = latency_watermark_b;
1986	serge	2331	}
		2332
2997	Serge	2333	/**
		2334	* evergreen_bandwidth_update - update display watermarks callback.
		2335	*
		2336	* @rdev: radeon_device pointer
		2337	*
		2338	* Update the display watermarks based on the requested mode(s)
		2339	* (evergreen+).
		2340	*/
1963	serge	2341	void evergreen_bandwidth_update(struct radeon_device *rdev)
1430	serge	2342	{
1986	serge	2343	struct drm_display_mode *mode0 = NULL;
		2344	struct drm_display_mode *mode1 = NULL;
		2345	u32 num_heads = 0, lb_size;
		2346	int i;
		2347
5271	serge	2348	if (!rdev->mode_info.mode_config_initialized)
		2349	return;
		2350
1986	serge	2351	radeon_update_display_priority(rdev);
		2352
		2353	for (i = 0; i < rdev->num_crtc; i++) {
		2354	if (rdev->mode_info.crtcs[i]->base.enabled)
		2355	num_heads++;
		2356	}
		2357	for (i = 0; i < rdev->num_crtc; i += 2) {
		2358	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
		2359	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
		2360	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
		2361	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
		2362	lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
		2363	evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
		2364	}
1430	serge	2365	}
		2366
2997	Serge	2367	/**
		2368	* evergreen_mc_wait_for_idle - wait for MC idle callback.
		2369	*
		2370	* @rdev: radeon_device pointer
		2371	*
		2372	* Wait for the MC (memory controller) to be idle.
		2373	* (evergreen+).
		2374	* Returns 0 if the MC is idle, -1 if not.
		2375	*/
1963	serge	2376	int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1430	serge	2377	{
		2378	unsigned i;
		2379	u32 tmp;
		2380
		2381	for (i = 0; i < rdev->usec_timeout; i++) {
		2382	/* read MC_STATUS */
		2383	tmp = RREG32(SRBM_STATUS) & 0x1F00;
		2384	if (!tmp)
		2385	return 0;
		2386	udelay(1);
		2387	}
		2388	return -1;
		2389	}
		2390
		2391	/*
		2392	* GART
		2393	*/
1963	serge	2394	void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
		2395	{
		2396	unsigned i;
		2397	u32 tmp;
		2398
		2399	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		2400
		2401	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		2402	for (i = 0; i < rdev->usec_timeout; i++) {
		2403	/* read MC_STATUS */
		2404	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		2405	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		2406	if (tmp == 2) {
		2407	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		2408	return;
		2409	}
		2410	if (tmp) {
		2411	return;
		2412	}
		2413	udelay(1);
		2414	}
		2415	}
		2416
2997	Serge	2417	static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
1430	serge	2418	{
		2419	u32 tmp;
1963	serge	2420	int r;
1430	serge	2421
2997	Serge	2422	if (rdev->gart.robj == NULL) {
1430	serge	2423	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		2424	return -EINVAL;
		2425	}
		2426	r = radeon_gart_table_vram_pin(rdev);
		2427	if (r)
		2428	return r;
		2429	/* Setup L2 cache */
		2430	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		2431	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		2432	EFFECTIVE_L2_QUEUE_SIZE(7));
		2433	WREG32(VM_L2_CNTL2, 0);
		2434	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		2435	/* Setup TLB control */
		2436	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		2437	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		2438	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		2439	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
1963	serge	2440	if (rdev->flags & RADEON_IS_IGP) {
		2441	WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
		2442	WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
		2443	WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
		2444	} else {
3031	serge	2445	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		2446	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		2447	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2997	Serge	2448	if ((rdev->family == CHIP_JUNIPER) \|\|
		2449	(rdev->family == CHIP_CYPRESS) \|\|
		2450	(rdev->family == CHIP_HEMLOCK) \|\|
		2451	(rdev->family == CHIP_BARTS))
		2452	WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
1963	serge	2453	}
1430	serge	2454	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		2455	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		2456	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		2457	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
		2458	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		2459	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		2460	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		2461	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		2462	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		2463	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		2464	(u32)(rdev->dummy_page.addr >> 12));
1963	serge	2465	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	2466
1963	serge	2467	evergreen_pcie_gart_tlb_flush(rdev);
2997	Serge	2468	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		2469	(unsigned)(rdev->mc.gtt_size >> 20),
		2470	(unsigned long long)rdev->gart.table_addr);
1430	serge	2471	rdev->gart.ready = true;
		2472	return 0;
		2473	}
		2474
2997	Serge	2475	static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
1430	serge	2476	{
		2477	u32 tmp;
		2478
		2479	/* Disable all tables */
1963	serge	2480	WREG32(VM_CONTEXT0_CNTL, 0);
		2481	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	2482
		2483	/* Setup L2 cache */
		2484	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		2485	EFFECTIVE_L2_QUEUE_SIZE(7));
		2486	WREG32(VM_L2_CNTL2, 0);
		2487	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		2488	/* Setup TLB control */
		2489	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		2490	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		2491	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		2492	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		2493	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		2494	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		2495	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		2496	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2997	Serge	2497	radeon_gart_table_vram_unpin(rdev);
1430	serge	2498	}
		2499
2997	Serge	2500	static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
1430	serge	2501	{
		2502	evergreen_pcie_gart_disable(rdev);
		2503	radeon_gart_table_vram_free(rdev);
		2504	radeon_gart_fini(rdev);
		2505	}
		2506
		2507
2997	Serge	2508	static void evergreen_agp_enable(struct radeon_device *rdev)
1430	serge	2509	{
		2510	u32 tmp;
		2511
		2512	/* Setup L2 cache */
		2513	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		2514	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		2515	EFFECTIVE_L2_QUEUE_SIZE(7));
		2516	WREG32(VM_L2_CNTL2, 0);
		2517	WREG32(VM_L2_CNTL3, BANK_SELECT(0) \| CACHE_UPDATE_MODE(2));
		2518	/* Setup TLB control */
		2519	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		2520	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		2521	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU \|
		2522	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5);
		2523	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
		2524	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
		2525	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
		2526	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
		2527	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
		2528	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
		2529	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1963	serge	2530	WREG32(VM_CONTEXT0_CNTL, 0);
		2531	WREG32(VM_CONTEXT1_CNTL, 0);
1430	serge	2532	}
		2533
1963	serge	2534	void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	2535	{
2997	Serge	2536	u32 crtc_enabled, tmp, frame_count, blackout;
		2537	int i, j;
		2538
3764	Serge	2539	if (!ASIC_IS_NODCE(rdev)) {
1430	serge	2540	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
		2541	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
		2542
2997	Serge	2543	/* disable VGA render */
1430	serge	2544	WREG32(VGA_RENDER_CONTROL, 0);
3764	Serge	2545	}
2997	Serge	2546	/* blank the display controllers */
		2547	for (i = 0; i < rdev->num_crtc; i++) {
		2548	crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
		2549	if (crtc_enabled) {
		2550	save->crtc_enabled[i] = true;
		2551	if (ASIC_IS_DCE6(rdev)) {
		2552	tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
		2553	if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
		2554	radeon_wait_for_vblank(rdev, i);
3764	Serge	2555	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2997	Serge	2556	tmp \|= EVERGREEN_CRTC_BLANK_DATA_EN;
		2557	WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
5271	serge	2558	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
3031	serge	2559	}
2997	Serge	2560	} else {
		2561	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
		2562	if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
		2563	radeon_wait_for_vblank(rdev, i);
3764	Serge	2564	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2997	Serge	2565	tmp \|= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
		2566	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
3764	Serge	2567	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
3031	serge	2568	}
		2569	}
2997	Serge	2570	/* wait for the next frame */
		2571	frame_count = radeon_get_vblank_counter(rdev, i);
		2572	for (j = 0; j < rdev->usec_timeout; j++) {
		2573	if (radeon_get_vblank_counter(rdev, i) != frame_count)
		2574	break;
		2575	udelay(1);
3031	serge	2576	}
3764	Serge	2577
		2578	/* XXX this is a hack to avoid strange behavior with EFI on certain systems */
		2579	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
		2580	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
		2581	tmp &= ~EVERGREEN_CRTC_MASTER_EN;
		2582	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
		2583	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
		2584	save->crtc_enabled[i] = false;
		2585	/* ***** */
3120	serge	2586	} else {
		2587	save->crtc_enabled[i] = false;
3031	serge	2588	}
1963	serge	2589	}
1430	serge	2590
2997	Serge	2591	radeon_mc_wait_for_idle(rdev);
		2592
		2593	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
		2594	if ((blackout & BLACKOUT_MODE_MASK) != 1) {
		2595	/* Block CPU access */
		2596	WREG32(BIF_FB_EN, 0);
		2597	/* blackout the MC */
		2598	blackout &= ~BLACKOUT_MODE_MASK;
		2599	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout \| 1);
2005	serge	2600	}
3764	Serge	2601	/* wait for the MC to settle */
		2602	udelay(100);
		2603
		2604	/* lock double buffered regs */
		2605	for (i = 0; i < rdev->num_crtc; i++) {
		2606	if (save->crtc_enabled[i]) {
		2607	tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
		2608	if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
		2609	tmp \|= EVERGREEN_GRPH_UPDATE_LOCK;
		2610	WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
		2611	}
		2612	tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
		2613	if (!(tmp & 1)) {
		2614	tmp \|= 1;
		2615	WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
		2616	}
		2617	}
		2618	}
1430	serge	2619	}
		2620
1963	serge	2621	void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save)
1430	serge	2622	{
2997	Serge	2623	u32 tmp, frame_count;
		2624	int i, j;
1430	serge	2625
2997	Serge	2626	/* update crtc base addresses */
		2627	for (i = 0; i < rdev->num_crtc; i++) {
		2628	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
3031	serge	2629	upper_32_bits(rdev->mc.vram_start));
2997	Serge	2630	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
3031	serge	2631	upper_32_bits(rdev->mc.vram_start));
2997	Serge	2632	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
3031	serge	2633	(u32)rdev->mc.vram_start);
2997	Serge	2634	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
3031	serge	2635	(u32)rdev->mc.vram_start);
2997	Serge	2636	}
3764	Serge	2637
		2638	if (!ASIC_IS_NODCE(rdev)) {
2997	Serge	2639	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
		2640	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
3764	Serge	2641	}
1430	serge	2642
3764	Serge	2643	/* unlock regs and wait for update */
		2644	for (i = 0; i < rdev->num_crtc; i++) {
		2645	if (save->crtc_enabled[i]) {
		2646	tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
5078	serge	2647	if ((tmp & 0x7) != 3) {
		2648	tmp &= ~0x7;
		2649	tmp \|= 0x3;
3764	Serge	2650	WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
		2651	}
		2652	tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
		2653	if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
		2654	tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
		2655	WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
		2656	}
		2657	tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
		2658	if (tmp & 1) {
		2659	tmp &= ~1;
		2660	WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
		2661	}
		2662	for (j = 0; j < rdev->usec_timeout; j++) {
		2663	tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
		2664	if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
		2665	break;
		2666	udelay(1);
		2667	}
		2668	}
		2669	}
		2670
2997	Serge	2671	/* unblackout the MC */
		2672	tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
		2673	tmp &= ~BLACKOUT_MODE_MASK;
		2674	WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
		2675	/* allow CPU access */
		2676	WREG32(BIF_FB_EN, FB_READ_EN \| FB_WRITE_EN);
1430	serge	2677
2997	Serge	2678	for (i = 0; i < rdev->num_crtc; i++) {
3031	serge	2679	if (save->crtc_enabled[i]) {
2997	Serge	2680	if (ASIC_IS_DCE6(rdev)) {
		2681	tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
5078	serge	2682	tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
3764	Serge	2683	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2997	Serge	2684	WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
3764	Serge	2685	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2997	Serge	2686	} else {
		2687	tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
		2688	tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
3764	Serge	2689	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2997	Serge	2690	WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
3764	Serge	2691	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2997	Serge	2692	}
		2693	/* wait for the next frame */
		2694	frame_count = radeon_get_vblank_counter(rdev, i);
		2695	for (j = 0; j < rdev->usec_timeout; j++) {
		2696	if (radeon_get_vblank_counter(rdev, i) != frame_count)
		2697	break;
		2698	udelay(1);
		2699	}
3031	serge	2700	}
2005	serge	2701	}
3764	Serge	2702	if (!ASIC_IS_NODCE(rdev)) {
2997	Serge	2703	/* Unlock vga access */
1430	serge	2704	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
		2705	mdelay(1);
		2706	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
3764	Serge	2707	}
1430	serge	2708	}
		2709
1963	serge	2710	void evergreen_mc_program(struct radeon_device *rdev)
1430	serge	2711	{
		2712	struct evergreen_mc_save save;
		2713	u32 tmp;
		2714	int i, j;
		2715
		2716	/* Initialize HDP */
		2717	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		2718	WREG32((0x2c14 + j), 0x00000000);
		2719	WREG32((0x2c18 + j), 0x00000000);
		2720	WREG32((0x2c1c + j), 0x00000000);
		2721	WREG32((0x2c20 + j), 0x00000000);
		2722	WREG32((0x2c24 + j), 0x00000000);
		2723	}
		2724	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		2725
		2726	evergreen_mc_stop(rdev, &save);
		2727	if (evergreen_mc_wait_for_idle(rdev)) {
		2728	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		2729	}
		2730	/* Lockout access through VGA aperture*/
		2731	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		2732	/* Update configuration */
		2733	if (rdev->flags & RADEON_IS_AGP) {
		2734	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		2735	/* VRAM before AGP */
		2736	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		2737	rdev->mc.vram_start >> 12);
		2738	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		2739	rdev->mc.gtt_end >> 12);
		2740	} else {
		2741	/* VRAM after AGP */
		2742	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		2743	rdev->mc.gtt_start >> 12);
		2744	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		2745	rdev->mc.vram_end >> 12);
		2746	}
		2747	} else {
		2748	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		2749	rdev->mc.vram_start >> 12);
		2750	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		2751	rdev->mc.vram_end >> 12);
		2752	}
2997	Serge	2753	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
		2754	/* llano/ontario only */
		2755	if ((rdev->family == CHIP_PALM) \|\|
		2756	(rdev->family == CHIP_SUMO) \|\|
		2757	(rdev->family == CHIP_SUMO2)) {
1963	serge	2758	tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
		2759	tmp \|= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
		2760	tmp \|= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
		2761	WREG32(MC_FUS_VM_FB_OFFSET, tmp);
		2762	}
1430	serge	2763	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		2764	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		2765	WREG32(MC_VM_FB_LOCATION, tmp);
		2766	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1963	serge	2767	WREG32(HDP_NONSURFACE_INFO, (2 << 7) \| (1 << 30));
		2768	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1430	serge	2769	if (rdev->flags & RADEON_IS_AGP) {
		2770	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
		2771	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
		2772	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		2773	} else {
		2774	WREG32(MC_VM_AGP_BASE, 0);
		2775	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		2776	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		2777	}
		2778	if (evergreen_mc_wait_for_idle(rdev)) {
		2779	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		2780	}
		2781	evergreen_mc_resume(rdev, &save);
		2782	/* we need to own VRAM, so turn off the VGA renderer here
		2783	* to stop it overwriting our objects */
		2784	rv515_vga_render_disable(rdev);
		2785	}
		2786
		2787	/*
		2788	* CP.
		2789	*/
1986	serge	2790	void evergreen_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		2791	{
2997	Serge	2792	struct radeon_ring *ring = &rdev->ring[ib->ring];
		2793	u32 next_rptr;
		2794
1986	serge	2795	/* set to DX10/11 mode */
2997	Serge	2796	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
		2797	radeon_ring_write(ring, 1);
		2798
		2799	if (ring->rptr_save_reg) {
		2800	next_rptr = ring->wptr + 3 + 4;
		2801	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3764	Serge	2802	radeon_ring_write(ring, ((ring->rptr_save_reg -
2997	Serge	2803	PACKET3_SET_CONFIG_REG_START) >> 2));
		2804	radeon_ring_write(ring, next_rptr);
		2805	} else if (rdev->wb.enabled) {
		2806	next_rptr = ring->wptr + 5 + 4;
		2807	radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
		2808	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		2809	radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) \| (1 << 18));
		2810	radeon_ring_write(ring, next_rptr);
		2811	radeon_ring_write(ring, 0);
		2812	}
		2813
		2814	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		2815	radeon_ring_write(ring,
1986	serge	2816	#ifdef __BIG_ENDIAN
		2817	(2 << 0) \|
		2818	#endif
		2819	(ib->gpu_addr & 0xFFFFFFFC));
2997	Serge	2820	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
		2821	radeon_ring_write(ring, ib->length_dw);
1986	serge	2822	}
1963	serge	2823
1986	serge	2824
1963	serge	2825	static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1430	serge	2826	{
1963	serge	2827	const __be32 *fw_data;
		2828	int i;
		2829
		2830	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		2831	return -EINVAL;
		2832
		2833	r700_cp_stop(rdev);
		2834	WREG32(CP_RB_CNTL,
		2835	#ifdef __BIG_ENDIAN
		2836	BUF_SWAP_32BIT \|
		2837	#endif
		2838	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
		2839
		2840	fw_data = (const __be32 *)rdev->pfp_fw->data;
		2841	WREG32(CP_PFP_UCODE_ADDR, 0);
		2842	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
		2843	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		2844	WREG32(CP_PFP_UCODE_ADDR, 0);
		2845
		2846	fw_data = (const __be32 *)rdev->me_fw->data;
		2847	WREG32(CP_ME_RAM_WADDR, 0);
		2848	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
		2849	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		2850
		2851	WREG32(CP_PFP_UCODE_ADDR, 0);
		2852	WREG32(CP_ME_RAM_WADDR, 0);
		2853	WREG32(CP_ME_RAM_RADDR, 0);
		2854	return 0;
1430	serge	2855	}
		2856
1963	serge	2857	static int evergreen_cp_start(struct radeon_device *rdev)
		2858	{
2997	Serge	2859	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	2860	int r, i;
		2861	uint32_t cp_me;
1430	serge	2862
2997	Serge	2863	r = radeon_ring_lock(rdev, ring, 7);
1963	serge	2864	if (r) {
		2865	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		2866	return r;
		2867	}
2997	Serge	2868	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		2869	radeon_ring_write(ring, 0x1);
		2870	radeon_ring_write(ring, 0x0);
		2871	radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
		2872	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		2873	radeon_ring_write(ring, 0);
		2874	radeon_ring_write(ring, 0);
5078	serge	2875	radeon_ring_unlock_commit(rdev, ring, false);
1963	serge	2876
		2877	cp_me = 0xff;
		2878	WREG32(CP_ME_CNTL, cp_me);
		2879
2997	Serge	2880	r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
1963	serge	2881	if (r) {
		2882	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		2883	return r;
		2884	}
		2885
		2886	/* setup clear context state */
2997	Serge	2887	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		2888	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1963	serge	2889
		2890	for (i = 0; i < evergreen_default_size; i++)
2997	Serge	2891	radeon_ring_write(ring, evergreen_default_state[i]);
1963	serge	2892
2997	Serge	2893	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		2894	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1963	serge	2895
		2896	/* set clear context state */
2997	Serge	2897	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
		2898	radeon_ring_write(ring, 0);
1963	serge	2899
		2900	/* SQ_VTX_BASE_VTX_LOC */
2997	Serge	2901	radeon_ring_write(ring, 0xc0026f00);
		2902	radeon_ring_write(ring, 0x00000000);
		2903	radeon_ring_write(ring, 0x00000000);
		2904	radeon_ring_write(ring, 0x00000000);
1963	serge	2905
		2906	/* Clear consts */
2997	Serge	2907	radeon_ring_write(ring, 0xc0036f00);
		2908	radeon_ring_write(ring, 0x00000bc4);
		2909	radeon_ring_write(ring, 0xffffffff);
		2910	radeon_ring_write(ring, 0xffffffff);
		2911	radeon_ring_write(ring, 0xffffffff);
1963	serge	2912
2997	Serge	2913	radeon_ring_write(ring, 0xc0026900);
		2914	radeon_ring_write(ring, 0x00000316);
		2915	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		2916	radeon_ring_write(ring, 0x00000010); /* */
1963	serge	2917
5078	serge	2918	radeon_ring_unlock_commit(rdev, ring, false);
1963	serge	2919
		2920	return 0;
		2921	}
		2922
2997	Serge	2923	static int evergreen_cp_resume(struct radeon_device *rdev)
1430	serge	2924	{
2997	Serge	2925	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	2926	u32 tmp;
		2927	u32 rb_bufsz;
		2928	int r;
1430	serge	2929
1963	serge	2930	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
		2931	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP \|
		2932	SOFT_RESET_PA \|
		2933	SOFT_RESET_SH \|
		2934	SOFT_RESET_VGT \|
2160	serge	2935	SOFT_RESET_SPI \|
1963	serge	2936	SOFT_RESET_SX));
		2937	RREG32(GRBM_SOFT_RESET);
		2938	mdelay(15);
		2939	WREG32(GRBM_SOFT_RESET, 0);
		2940	RREG32(GRBM_SOFT_RESET);
		2941
		2942	/* Set ring buffer size */
5078	serge	2943	rb_bufsz = order_base_2(ring->ring_size / 8);
		2944	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
1963	serge	2945	#ifdef __BIG_ENDIAN
		2946	tmp \|= BUF_SWAP_32BIT;
		2947	#endif
		2948	WREG32(CP_RB_CNTL, tmp);
2997	Serge	2949	WREG32(CP_SEM_WAIT_TIMER, 0x0);
		2950	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1963	serge	2951
		2952	/* Set the write pointer delay */
		2953	WREG32(CP_RB_WPTR_DELAY, 0);
		2954
		2955	/* Initialize the ring buffer's read and write pointers */
		2956	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		2957	WREG32(CP_RB_RPTR_WR, 0);
2997	Serge	2958	ring->wptr = 0;
		2959	WREG32(CP_RB_WPTR, ring->wptr);
1963	serge	2960
3120	serge	2961	/* set the wb address whether it's enabled or not */
1963	serge	2962	WREG32(CP_RB_RPTR_ADDR,
		2963	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		2964	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		2965	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		2966
		2967	if (rdev->wb.enabled)
		2968	WREG32(SCRATCH_UMSK, 0xff);
		2969	else {
		2970	tmp \|= RB_NO_UPDATE;
		2971	WREG32(SCRATCH_UMSK, 0);
		2972	}
		2973
		2974	mdelay(1);
		2975	WREG32(CP_RB_CNTL, tmp);
		2976
2997	Serge	2977	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1963	serge	2978	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		2979
		2980	evergreen_cp_start(rdev);
2997	Serge	2981	ring->ready = true;
		2982	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
1963	serge	2983	if (r) {
2997	Serge	2984	ring->ready = false;
1963	serge	2985	return r;
		2986	}
1430	serge	2987	return 0;
		2988	}
		2989
		2990	/*
		2991	* Core functions
		2992	*/
		2993	static void evergreen_gpu_init(struct radeon_device *rdev)
		2994	{
2997	Serge	2995	u32 gb_addr_config;
1963	serge	2996	u32 mc_shared_chmap, mc_arb_ramcfg;
		2997	u32 sx_debug_1;
		2998	u32 smx_dc_ctl0;
		2999	u32 sq_config;
		3000	u32 sq_lds_resource_mgmt;
		3001	u32 sq_gpr_resource_mgmt_1;
		3002	u32 sq_gpr_resource_mgmt_2;
		3003	u32 sq_gpr_resource_mgmt_3;
		3004	u32 sq_thread_resource_mgmt;
		3005	u32 sq_thread_resource_mgmt_2;
		3006	u32 sq_stack_resource_mgmt_1;
		3007	u32 sq_stack_resource_mgmt_2;
		3008	u32 sq_stack_resource_mgmt_3;
		3009	u32 vgt_cache_invalidation;
		3010	u32 hdp_host_path_cntl, tmp;
2997	Serge	3011	u32 disabled_rb_mask;
5271	serge	3012	int i, j, ps_thread_count;
1963	serge	3013
		3014	switch (rdev->family) {
		3015	case CHIP_CYPRESS:
		3016	case CHIP_HEMLOCK:
		3017	rdev->config.evergreen.num_ses = 2;
		3018	rdev->config.evergreen.max_pipes = 4;
		3019	rdev->config.evergreen.max_tile_pipes = 8;
		3020	rdev->config.evergreen.max_simds = 10;
		3021	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		3022	rdev->config.evergreen.max_gprs = 256;
		3023	rdev->config.evergreen.max_threads = 248;
		3024	rdev->config.evergreen.max_gs_threads = 32;
		3025	rdev->config.evergreen.max_stack_entries = 512;
		3026	rdev->config.evergreen.sx_num_of_sets = 4;
		3027	rdev->config.evergreen.sx_max_export_size = 256;
		3028	rdev->config.evergreen.sx_max_export_pos_size = 64;
		3029	rdev->config.evergreen.sx_max_export_smx_size = 192;
		3030	rdev->config.evergreen.max_hw_contexts = 8;
		3031	rdev->config.evergreen.sq_num_cf_insts = 2;
		3032
		3033	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		3034	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		3035	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	3036	gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	3037	break;
		3038	case CHIP_JUNIPER:
		3039	rdev->config.evergreen.num_ses = 1;
		3040	rdev->config.evergreen.max_pipes = 4;
		3041	rdev->config.evergreen.max_tile_pipes = 4;
		3042	rdev->config.evergreen.max_simds = 10;
		3043	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		3044	rdev->config.evergreen.max_gprs = 256;
		3045	rdev->config.evergreen.max_threads = 248;
		3046	rdev->config.evergreen.max_gs_threads = 32;
		3047	rdev->config.evergreen.max_stack_entries = 512;
		3048	rdev->config.evergreen.sx_num_of_sets = 4;
		3049	rdev->config.evergreen.sx_max_export_size = 256;
		3050	rdev->config.evergreen.sx_max_export_pos_size = 64;
		3051	rdev->config.evergreen.sx_max_export_smx_size = 192;
		3052	rdev->config.evergreen.max_hw_contexts = 8;
		3053	rdev->config.evergreen.sq_num_cf_insts = 2;
		3054
		3055	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		3056	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		3057	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	3058	gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
1963	serge	3059	break;
		3060	case CHIP_REDWOOD:
		3061	rdev->config.evergreen.num_ses = 1;
		3062	rdev->config.evergreen.max_pipes = 4;
		3063	rdev->config.evergreen.max_tile_pipes = 4;
		3064	rdev->config.evergreen.max_simds = 5;
		3065	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		3066	rdev->config.evergreen.max_gprs = 256;
		3067	rdev->config.evergreen.max_threads = 248;
		3068	rdev->config.evergreen.max_gs_threads = 32;
		3069	rdev->config.evergreen.max_stack_entries = 256;
		3070	rdev->config.evergreen.sx_num_of_sets = 4;
		3071	rdev->config.evergreen.sx_max_export_size = 256;
		3072	rdev->config.evergreen.sx_max_export_pos_size = 64;
		3073	rdev->config.evergreen.sx_max_export_smx_size = 192;
		3074	rdev->config.evergreen.max_hw_contexts = 8;
		3075	rdev->config.evergreen.sq_num_cf_insts = 2;
		3076
		3077	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		3078	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		3079	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	3080	gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
1963	serge	3081	break;
		3082	case CHIP_CEDAR:
		3083	default:
		3084	rdev->config.evergreen.num_ses = 1;
		3085	rdev->config.evergreen.max_pipes = 2;
		3086	rdev->config.evergreen.max_tile_pipes = 2;
		3087	rdev->config.evergreen.max_simds = 2;
		3088	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		3089	rdev->config.evergreen.max_gprs = 256;
		3090	rdev->config.evergreen.max_threads = 192;
		3091	rdev->config.evergreen.max_gs_threads = 16;
		3092	rdev->config.evergreen.max_stack_entries = 256;
		3093	rdev->config.evergreen.sx_num_of_sets = 4;
		3094	rdev->config.evergreen.sx_max_export_size = 128;
		3095	rdev->config.evergreen.sx_max_export_pos_size = 32;
		3096	rdev->config.evergreen.sx_max_export_smx_size = 96;
		3097	rdev->config.evergreen.max_hw_contexts = 4;
		3098	rdev->config.evergreen.sq_num_cf_insts = 1;
		3099
		3100	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		3101	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		3102	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	3103	gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
1963	serge	3104	break;
		3105	case CHIP_PALM:
		3106	rdev->config.evergreen.num_ses = 1;
		3107	rdev->config.evergreen.max_pipes = 2;
		3108	rdev->config.evergreen.max_tile_pipes = 2;
		3109	rdev->config.evergreen.max_simds = 2;
		3110	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		3111	rdev->config.evergreen.max_gprs = 256;
		3112	rdev->config.evergreen.max_threads = 192;
		3113	rdev->config.evergreen.max_gs_threads = 16;
		3114	rdev->config.evergreen.max_stack_entries = 256;
		3115	rdev->config.evergreen.sx_num_of_sets = 4;
		3116	rdev->config.evergreen.sx_max_export_size = 128;
		3117	rdev->config.evergreen.sx_max_export_pos_size = 32;
		3118	rdev->config.evergreen.sx_max_export_smx_size = 96;
		3119	rdev->config.evergreen.max_hw_contexts = 4;
		3120	rdev->config.evergreen.sq_num_cf_insts = 1;
		3121
		3122	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		3123	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		3124	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	3125	gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
1963	serge	3126	break;
		3127	case CHIP_SUMO:
		3128	rdev->config.evergreen.num_ses = 1;
		3129	rdev->config.evergreen.max_pipes = 4;
3192	Serge	3130	rdev->config.evergreen.max_tile_pipes = 4;
1963	serge	3131	if (rdev->pdev->device == 0x9648)
		3132	rdev->config.evergreen.max_simds = 3;
		3133	else if ((rdev->pdev->device == 0x9647) \|\|
		3134	(rdev->pdev->device == 0x964a))
		3135	rdev->config.evergreen.max_simds = 4;
		3136	else
		3137	rdev->config.evergreen.max_simds = 5;
		3138	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		3139	rdev->config.evergreen.max_gprs = 256;
		3140	rdev->config.evergreen.max_threads = 248;
		3141	rdev->config.evergreen.max_gs_threads = 32;
		3142	rdev->config.evergreen.max_stack_entries = 256;
		3143	rdev->config.evergreen.sx_num_of_sets = 4;
		3144	rdev->config.evergreen.sx_max_export_size = 256;
		3145	rdev->config.evergreen.sx_max_export_pos_size = 64;
		3146	rdev->config.evergreen.sx_max_export_smx_size = 192;
		3147	rdev->config.evergreen.max_hw_contexts = 8;
		3148	rdev->config.evergreen.sq_num_cf_insts = 2;
		3149
		3150	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		3151	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		3152	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3192	Serge	3153	gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
1963	serge	3154	break;
		3155	case CHIP_SUMO2:
		3156	rdev->config.evergreen.num_ses = 1;
		3157	rdev->config.evergreen.max_pipes = 4;
		3158	rdev->config.evergreen.max_tile_pipes = 4;
		3159	rdev->config.evergreen.max_simds = 2;
		3160	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		3161	rdev->config.evergreen.max_gprs = 256;
		3162	rdev->config.evergreen.max_threads = 248;
		3163	rdev->config.evergreen.max_gs_threads = 32;
		3164	rdev->config.evergreen.max_stack_entries = 512;
		3165	rdev->config.evergreen.sx_num_of_sets = 4;
		3166	rdev->config.evergreen.sx_max_export_size = 256;
		3167	rdev->config.evergreen.sx_max_export_pos_size = 64;
		3168	rdev->config.evergreen.sx_max_export_smx_size = 192;
5078	serge	3169	rdev->config.evergreen.max_hw_contexts = 4;
1963	serge	3170	rdev->config.evergreen.sq_num_cf_insts = 2;
		3171
		3172	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		3173	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		3174	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3192	Serge	3175	gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
1963	serge	3176	break;
		3177	case CHIP_BARTS:
		3178	rdev->config.evergreen.num_ses = 2;
		3179	rdev->config.evergreen.max_pipes = 4;
		3180	rdev->config.evergreen.max_tile_pipes = 8;
		3181	rdev->config.evergreen.max_simds = 7;
		3182	rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
		3183	rdev->config.evergreen.max_gprs = 256;
		3184	rdev->config.evergreen.max_threads = 248;
		3185	rdev->config.evergreen.max_gs_threads = 32;
		3186	rdev->config.evergreen.max_stack_entries = 512;
		3187	rdev->config.evergreen.sx_num_of_sets = 4;
		3188	rdev->config.evergreen.sx_max_export_size = 256;
		3189	rdev->config.evergreen.sx_max_export_pos_size = 64;
		3190	rdev->config.evergreen.sx_max_export_smx_size = 192;
		3191	rdev->config.evergreen.max_hw_contexts = 8;
		3192	rdev->config.evergreen.sq_num_cf_insts = 2;
		3193
		3194	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		3195	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		3196	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	3197	gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	3198	break;
		3199	case CHIP_TURKS:
		3200	rdev->config.evergreen.num_ses = 1;
		3201	rdev->config.evergreen.max_pipes = 4;
		3202	rdev->config.evergreen.max_tile_pipes = 4;
		3203	rdev->config.evergreen.max_simds = 6;
		3204	rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
		3205	rdev->config.evergreen.max_gprs = 256;
		3206	rdev->config.evergreen.max_threads = 248;
		3207	rdev->config.evergreen.max_gs_threads = 32;
		3208	rdev->config.evergreen.max_stack_entries = 256;
		3209	rdev->config.evergreen.sx_num_of_sets = 4;
		3210	rdev->config.evergreen.sx_max_export_size = 256;
		3211	rdev->config.evergreen.sx_max_export_pos_size = 64;
		3212	rdev->config.evergreen.sx_max_export_smx_size = 192;
		3213	rdev->config.evergreen.max_hw_contexts = 8;
		3214	rdev->config.evergreen.sq_num_cf_insts = 2;
		3215
		3216	rdev->config.evergreen.sc_prim_fifo_size = 0x100;
		3217	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		3218	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	3219	gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	3220	break;
		3221	case CHIP_CAICOS:
		3222	rdev->config.evergreen.num_ses = 1;
3192	Serge	3223	rdev->config.evergreen.max_pipes = 2;
1963	serge	3224	rdev->config.evergreen.max_tile_pipes = 2;
		3225	rdev->config.evergreen.max_simds = 2;
		3226	rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
		3227	rdev->config.evergreen.max_gprs = 256;
		3228	rdev->config.evergreen.max_threads = 192;
		3229	rdev->config.evergreen.max_gs_threads = 16;
		3230	rdev->config.evergreen.max_stack_entries = 256;
		3231	rdev->config.evergreen.sx_num_of_sets = 4;
		3232	rdev->config.evergreen.sx_max_export_size = 128;
		3233	rdev->config.evergreen.sx_max_export_pos_size = 32;
		3234	rdev->config.evergreen.sx_max_export_smx_size = 96;
		3235	rdev->config.evergreen.max_hw_contexts = 4;
		3236	rdev->config.evergreen.sq_num_cf_insts = 1;
		3237
		3238	rdev->config.evergreen.sc_prim_fifo_size = 0x40;
		3239	rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
		3240	rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	3241	gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
1963	serge	3242	break;
		3243	}
		3244
		3245	/* Initialize HDP */
		3246	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		3247	WREG32((0x2c14 + j), 0x00000000);
		3248	WREG32((0x2c18 + j), 0x00000000);
		3249	WREG32((0x2c1c + j), 0x00000000);
		3250	WREG32((0x2c20 + j), 0x00000000);
		3251	WREG32((0x2c24 + j), 0x00000000);
		3252	}
		3253
		3254	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		3255
2997	Serge	3256	evergreen_fix_pci_max_read_req_size(rdev);
1963	serge	3257
		3258	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2997	Serge	3259	if ((rdev->family == CHIP_PALM) \|\|
		3260	(rdev->family == CHIP_SUMO) \|\|
		3261	(rdev->family == CHIP_SUMO2))
1963	serge	3262	mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
		3263	else
3031	serge	3264	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1963	serge	3265
		3266	/* setup tiling info dword. gb_addr_config is not adequate since it does
		3267	* not have bank info, so create a custom tiling dword.
		3268	* bits 3:0 num_pipes
		3269	* bits 7:4 num_banks
		3270	* bits 11:8 group_size
		3271	* bits 15:12 row_size
		3272	*/
		3273	rdev->config.evergreen.tile_config = 0;
		3274	switch (rdev->config.evergreen.max_tile_pipes) {
		3275	case 1:
		3276	default:
		3277	rdev->config.evergreen.tile_config \|= (0 << 0);
		3278	break;
		3279	case 2:
		3280	rdev->config.evergreen.tile_config \|= (1 << 0);
		3281	break;
		3282	case 4:
		3283	rdev->config.evergreen.tile_config \|= (2 << 0);
		3284	break;
		3285	case 8:
		3286	rdev->config.evergreen.tile_config \|= (3 << 0);
		3287	break;
		3288	}
1986	serge	3289	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1963	serge	3290	if (rdev->flags & RADEON_IS_IGP)
1986	serge	3291	rdev->config.evergreen.tile_config \|= 1 << 4;
2997	Serge	3292	else {
		3293	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
		3294	case 0: /* four banks */
		3295	rdev->config.evergreen.tile_config \|= 0 << 4;
		3296	break;
		3297	case 1: /* eight banks */
		3298	rdev->config.evergreen.tile_config \|= 1 << 4;
		3299	break;
		3300	case 2: /* sixteen banks */
		3301	default:
		3302	rdev->config.evergreen.tile_config \|= 2 << 4;
		3303	break;
		3304	}
		3305	}
		3306	rdev->config.evergreen.tile_config \|= 0 << 8;
1963	serge	3307	rdev->config.evergreen.tile_config \|=
		3308	((gb_addr_config & 0x30000000) >> 28) << 12;
		3309
2997	Serge	3310	if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
		3311	u32 efuse_straps_4;
		3312	u32 efuse_straps_3;
1963	serge	3313
5078	serge	3314	efuse_straps_4 = RREG32_RCU(0x204);
		3315	efuse_straps_3 = RREG32_RCU(0x203);
2997	Serge	3316	tmp = (((efuse_straps_4 & 0xf) << 4) \|
		3317	((efuse_straps_3 & 0xf0000000) >> 28));
		3318	} else {
		3319	tmp = 0;
		3320	for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
		3321	u32 rb_disable_bitmap;
1963	serge	3322
2997	Serge	3323	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		3324	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		3325	rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
		3326	tmp <<= 4;
		3327	tmp \|= rb_disable_bitmap;
1963	serge	3328	}
2997	Serge	3329	}
		3330	/* enabled rb are just the one not disabled :) */
		3331	disabled_rb_mask = tmp;
3764	Serge	3332	tmp = 0;
		3333	for (i = 0; i < rdev->config.evergreen.max_backends; i++)
		3334	tmp \|= (1 << i);
		3335	/* if all the backends are disabled, fix it up here */
		3336	if ((disabled_rb_mask & tmp) == tmp) {
		3337	for (i = 0; i < rdev->config.evergreen.max_backends; i++)
		3338	disabled_rb_mask &= ~(1 << i);
		3339	}
1963	serge	3340
5078	serge	3341	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
		3342	u32 simd_disable_bitmap;
		3343
		3344	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		3345	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		3346	simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
		3347	simd_disable_bitmap \|= 0xffffffff << rdev->config.evergreen.max_simds;
		3348	tmp <<= 16;
		3349	tmp \|= simd_disable_bitmap;
		3350	}
		3351	rdev->config.evergreen.active_simds = hweight32(~tmp);
		3352
2997	Serge	3353	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
		3354	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
1963	serge	3355
2997	Serge	3356	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		3357	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
		3358	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3192	Serge	3359	WREG32(DMA_TILING_CONFIG, gb_addr_config);
3764	Serge	3360	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
		3361	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
		3362	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1963	serge	3363
3764	Serge	3364	if ((rdev->config.evergreen.max_backends == 1) &&
		3365	(rdev->flags & RADEON_IS_IGP)) {
		3366	if ((disabled_rb_mask & 3) == 1) {
		3367	/* RB0 disabled, RB1 enabled */
		3368	tmp = 0x11111111;
		3369	} else {
		3370	/* RB1 disabled, RB0 enabled */
		3371	tmp = 0x00000000;
		3372	}
		3373	} else {
2997	Serge	3374	tmp = gb_addr_config & NUM_PIPES_MASK;
		3375	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
		3376	EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3764	Serge	3377	}
2997	Serge	3378	WREG32(GB_BACKEND_MAP, tmp);
1963	serge	3379
		3380	WREG32(CGTS_SYS_TCC_DISABLE, 0);
		3381	WREG32(CGTS_TCC_DISABLE, 0);
		3382	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
		3383	WREG32(CGTS_USER_TCC_DISABLE, 0);
		3384
		3385	/* set HW defaults for 3D engine */
		3386	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \|
		3387	ROQ_IB2_START(0x2b)));
		3388
		3389	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
		3390
		3391	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \|
		3392	SYNC_GRADIENT \|
		3393	SYNC_WALKER \|
		3394	SYNC_ALIGNER));
		3395
		3396	sx_debug_1 = RREG32(SX_DEBUG_1);
		3397	sx_debug_1 \|= ENABLE_NEW_SMX_ADDRESS;
		3398	WREG32(SX_DEBUG_1, sx_debug_1);
		3399
		3400
		3401	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
		3402	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
		3403	smx_dc_ctl0 \|= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
		3404	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
		3405
2997	Serge	3406	if (rdev->family <= CHIP_SUMO2)
		3407	WREG32(SMX_SAR_CTL0, 0x00010000);
		3408
1963	serge	3409	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) \|
		3410	POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) \|
		3411	SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
		3412
		3413	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) \|
		3414	SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) \|
		3415	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
		3416
		3417	WREG32(VGT_NUM_INSTANCES, 1);
		3418	WREG32(SPI_CONFIG_CNTL, 0);
		3419	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
		3420	WREG32(CP_PERFMON_CNTL, 0);
		3421
		3422	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) \|
		3423	FETCH_FIFO_HIWATER(0x4) \|
		3424	DONE_FIFO_HIWATER(0xe0) \|
		3425	ALU_UPDATE_FIFO_HIWATER(0x8)));
		3426
		3427	sq_config = RREG32(SQ_CONFIG);
		3428	sq_config &= ~(PS_PRIO(3) \|
		3429	VS_PRIO(3) \|
		3430	GS_PRIO(3) \|
		3431	ES_PRIO(3));
		3432	sq_config \|= (VC_ENABLE \|
		3433	EXPORT_SRC_C \|
		3434	PS_PRIO(0) \|
		3435	VS_PRIO(1) \|
		3436	GS_PRIO(2) \|
		3437	ES_PRIO(3));
		3438
		3439	switch (rdev->family) {
		3440	case CHIP_CEDAR:
		3441	case CHIP_PALM:
		3442	case CHIP_SUMO:
		3443	case CHIP_SUMO2:
		3444	case CHIP_CAICOS:
		3445	/* no vertex cache */
		3446	sq_config &= ~VC_ENABLE;
		3447	break;
		3448	default:
		3449	break;
		3450	}
		3451
		3452	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
		3453
		3454	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
		3455	sq_gpr_resource_mgmt_1 \|= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
		3456	sq_gpr_resource_mgmt_1 \|= NUM_CLAUSE_TEMP_GPRS(4);
		3457	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		3458	sq_gpr_resource_mgmt_2 \|= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
		3459	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		3460	sq_gpr_resource_mgmt_3 \|= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
		3461
		3462	switch (rdev->family) {
		3463	case CHIP_CEDAR:
		3464	case CHIP_PALM:
		3465	case CHIP_SUMO:
		3466	case CHIP_SUMO2:
		3467	ps_thread_count = 96;
		3468	break;
		3469	default:
		3470	ps_thread_count = 128;
		3471	break;
		3472	}
		3473
		3474	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
		3475	sq_thread_resource_mgmt \|= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		3476	sq_thread_resource_mgmt \|= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		3477	sq_thread_resource_mgmt \|= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		3478	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		3479	sq_thread_resource_mgmt_2 \|= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
		3480
		3481	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		3482	sq_stack_resource_mgmt_1 \|= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		3483	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		3484	sq_stack_resource_mgmt_2 \|= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		3485	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		3486	sq_stack_resource_mgmt_3 \|= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
		3487
		3488	WREG32(SQ_CONFIG, sq_config);
		3489	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		3490	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		3491	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
		3492	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		3493	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
		3494	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		3495	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		3496	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
		3497	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
		3498	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
		3499
		3500	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		3501	FORCE_EOV_MAX_REZ_CNT(255)));
		3502
		3503	switch (rdev->family) {
		3504	case CHIP_CEDAR:
		3505	case CHIP_PALM:
		3506	case CHIP_SUMO:
		3507	case CHIP_SUMO2:
		3508	case CHIP_CAICOS:
		3509	vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
		3510	break;
		3511	default:
		3512	vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
		3513	break;
		3514	}
		3515	vgt_cache_invalidation \|= AUTO_INVLD_EN(ES_AND_GS_AUTO);
		3516	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
		3517
		3518	WREG32(VGT_GS_VERTEX_REUSE, 16);
		3519	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
		3520	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		3521
		3522	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
		3523	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
		3524
		3525	WREG32(CB_PERF_CTR0_SEL_0, 0);
		3526	WREG32(CB_PERF_CTR0_SEL_1, 0);
		3527	WREG32(CB_PERF_CTR1_SEL_0, 0);
		3528	WREG32(CB_PERF_CTR1_SEL_1, 0);
		3529	WREG32(CB_PERF_CTR2_SEL_0, 0);
		3530	WREG32(CB_PERF_CTR2_SEL_1, 0);
		3531	WREG32(CB_PERF_CTR3_SEL_0, 0);
		3532	WREG32(CB_PERF_CTR3_SEL_1, 0);
		3533
		3534	/* clear render buffer base addresses */
		3535	WREG32(CB_COLOR0_BASE, 0);
		3536	WREG32(CB_COLOR1_BASE, 0);
		3537	WREG32(CB_COLOR2_BASE, 0);
		3538	WREG32(CB_COLOR3_BASE, 0);
		3539	WREG32(CB_COLOR4_BASE, 0);
		3540	WREG32(CB_COLOR5_BASE, 0);
		3541	WREG32(CB_COLOR6_BASE, 0);
		3542	WREG32(CB_COLOR7_BASE, 0);
		3543	WREG32(CB_COLOR8_BASE, 0);
		3544	WREG32(CB_COLOR9_BASE, 0);
		3545	WREG32(CB_COLOR10_BASE, 0);
		3546	WREG32(CB_COLOR11_BASE, 0);
		3547
		3548	/* set the shader const cache sizes to 0 */
		3549	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
		3550	WREG32(i, 0);
		3551	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
		3552	WREG32(i, 0);
		3553
		3554	tmp = RREG32(HDP_MISC_CNTL);
		3555	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		3556	WREG32(HDP_MISC_CNTL, tmp);
		3557
		3558	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		3559	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		3560
		3561	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		3562
		3563	udelay(50);
		3564
1430	serge	3565	}
		3566
		3567	int evergreen_mc_init(struct radeon_device *rdev)
		3568	{
		3569	u32 tmp;
		3570	int chansize, numchan;
		3571
		3572	/* Get VRAM informations */
		3573	rdev->mc.vram_is_ddr = true;
2997	Serge	3574	if ((rdev->family == CHIP_PALM) \|\|
		3575	(rdev->family == CHIP_SUMO) \|\|
		3576	(rdev->family == CHIP_SUMO2))
2004	serge	3577	tmp = RREG32(FUS_MC_ARB_RAMCFG);
		3578	else
3031	serge	3579	tmp = RREG32(MC_ARB_RAMCFG);
1430	serge	3580	if (tmp & CHANSIZE_OVERRIDE) {
		3581	chansize = 16;
		3582	} else if (tmp & CHANSIZE_MASK) {
		3583	chansize = 64;
		3584	} else {
		3585	chansize = 32;
		3586	}
		3587	tmp = RREG32(MC_SHARED_CHMAP);
		3588	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		3589	case 0:
		3590	default:
		3591	numchan = 1;
		3592	break;
		3593	case 1:
		3594	numchan = 2;
		3595	break;
		3596	case 2:
		3597	numchan = 4;
		3598	break;
		3599	case 3:
		3600	numchan = 8;
		3601	break;
		3602	}
		3603	rdev->mc.vram_width = numchan * chansize;
		3604	/* Could aper size report 0 ? */
1963	serge	3605	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		3606	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1430	serge	3607	/* Setup GPU memory space */
2997	Serge	3608	if ((rdev->family == CHIP_PALM) \|\|
		3609	(rdev->family == CHIP_SUMO) \|\|
		3610	(rdev->family == CHIP_SUMO2)) {
1963	serge	3611	/* size in bytes on fusion */
		3612	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		3613	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
		3614	} else {
2997	Serge	3615	/* size in MB on evergreen/cayman/tn */
3764	Serge	3616	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
		3617	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1963	serge	3618	}
1430	serge	3619	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1963	serge	3620	r700_vram_gtt_location(rdev, &rdev->mc);
		3621	radeon_update_bandwidth_info(rdev);
		3622
1430	serge	3623	return 0;
		3624	}
		3625
3764	Serge	3626	void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
1430	serge	3627	{
3764	Serge	3628	dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
1963	serge	3629	RREG32(GRBM_STATUS));
3764	Serge	3630	dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
1963	serge	3631	RREG32(GRBM_STATUS_SE0));
3764	Serge	3632	dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
1963	serge	3633	RREG32(GRBM_STATUS_SE1));
3764	Serge	3634	dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
1963	serge	3635	RREG32(SRBM_STATUS));
3764	Serge	3636	dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
		3637	RREG32(SRBM_STATUS2));
2997	Serge	3638	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		3639	RREG32(CP_STALLED_STAT1));
		3640	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		3641	RREG32(CP_STALLED_STAT2));
		3642	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		3643	RREG32(CP_BUSY_STAT));
		3644	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		3645	RREG32(CP_STAT));
3764	Serge	3646	dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
		3647	RREG32(DMA_STATUS_REG));
		3648	if (rdev->family >= CHIP_CAYMAN) {
		3649	dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
		3650	RREG32(DMA_STATUS_REG + 0x800));
		3651	}
		3652	}
		3653
		3654	bool evergreen_is_display_hung(struct radeon_device *rdev)
		3655	{
		3656	u32 crtc_hung = 0;
		3657	u32 crtc_status[6];
		3658	u32 i, j, tmp;
		3659
		3660	for (i = 0; i < rdev->num_crtc; i++) {
		3661	if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
		3662	crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
		3663	crtc_hung \|= (1 << i);
		3664	}
		3665	}
		3666
		3667	for (j = 0; j < 10; j++) {
		3668	for (i = 0; i < rdev->num_crtc; i++) {
		3669	if (crtc_hung & (1 << i)) {
		3670	tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
		3671	if (tmp != crtc_status[i])
		3672	crtc_hung &= ~(1 << i);
		3673	}
		3674	}
		3675	if (crtc_hung == 0)
		3676	return false;
		3677	udelay(100);
		3678	}
		3679
		3680	return true;
		3681	}
		3682
5078	serge	3683	u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3764	Serge	3684	{
		3685	u32 reset_mask = 0;
		3686	u32 tmp;
		3687
		3688	/* GRBM_STATUS */
		3689	tmp = RREG32(GRBM_STATUS);
		3690	if (tmp & (PA_BUSY \| SC_BUSY \|
		3691	SH_BUSY \| SX_BUSY \|
		3692	TA_BUSY \| VGT_BUSY \|
		3693	DB_BUSY \| CB_BUSY \|
		3694	SPI_BUSY \| VGT_BUSY_NO_DMA))
		3695	reset_mask \|= RADEON_RESET_GFX;
		3696
		3697	if (tmp & (CF_RQ_PENDING \| PF_RQ_PENDING \|
		3698	CP_BUSY \| CP_COHERENCY_BUSY))
		3699	reset_mask \|= RADEON_RESET_CP;
		3700
		3701	if (tmp & GRBM_EE_BUSY)
		3702	reset_mask \|= RADEON_RESET_GRBM \| RADEON_RESET_GFX \| RADEON_RESET_CP;
		3703
		3704	/* DMA_STATUS_REG */
		3705	tmp = RREG32(DMA_STATUS_REG);
		3706	if (!(tmp & DMA_IDLE))
		3707	reset_mask \|= RADEON_RESET_DMA;
		3708
		3709	/* SRBM_STATUS2 */
		3710	tmp = RREG32(SRBM_STATUS2);
		3711	if (tmp & DMA_BUSY)
		3712	reset_mask \|= RADEON_RESET_DMA;
		3713
		3714	/* SRBM_STATUS */
		3715	tmp = RREG32(SRBM_STATUS);
		3716	if (tmp & (RLC_RQ_PENDING \| RLC_BUSY))
		3717	reset_mask \|= RADEON_RESET_RLC;
		3718
		3719	if (tmp & IH_BUSY)
		3720	reset_mask \|= RADEON_RESET_IH;
		3721
		3722	if (tmp & SEM_BUSY)
		3723	reset_mask \|= RADEON_RESET_SEM;
		3724
		3725	if (tmp & GRBM_RQ_PENDING)
		3726	reset_mask \|= RADEON_RESET_GRBM;
		3727
		3728	if (tmp & VMC_BUSY)
		3729	reset_mask \|= RADEON_RESET_VMC;
		3730
		3731	if (tmp & (MCB_BUSY \| MCB_NON_DISPLAY_BUSY \|
		3732	MCC_BUSY \| MCD_BUSY))
		3733	reset_mask \|= RADEON_RESET_MC;
		3734
		3735	if (evergreen_is_display_hung(rdev))
		3736	reset_mask \|= RADEON_RESET_DISPLAY;
		3737
		3738	/* VM_L2_STATUS */
		3739	tmp = RREG32(VM_L2_STATUS);
		3740	if (tmp & L2_BUSY)
		3741	reset_mask \|= RADEON_RESET_VMC;
		3742
		3743	/* Skip MC reset as it's mostly likely not hung, just busy */
		3744	if (reset_mask & RADEON_RESET_MC) {
		3745	DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
		3746	reset_mask &= ~RADEON_RESET_MC;
		3747	}
		3748
		3749	return reset_mask;
		3750	}
		3751
		3752	static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
		3753	{
		3754	struct evergreen_mc_save save;
		3755	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
		3756	u32 tmp;
		3757
		3758	if (reset_mask == 0)
		3759	return;
		3760
		3761	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
		3762
		3763	evergreen_print_gpu_status_regs(rdev);
		3764
		3765	/* Disable CP parsing/prefetching */
		3766	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT);
		3767
		3768	if (reset_mask & RADEON_RESET_DMA) {
		3769	/* Disable DMA */
		3770	tmp = RREG32(DMA_RB_CNTL);
		3771	tmp &= ~DMA_RB_ENABLE;
		3772	WREG32(DMA_RB_CNTL, tmp);
		3773	}
		3774
		3775	udelay(50);
		3776
1963	serge	3777	evergreen_mc_stop(rdev, &save);
		3778	if (evergreen_mc_wait_for_idle(rdev)) {
		3779	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		3780	}
		3781
3764	Serge	3782	if (reset_mask & (RADEON_RESET_GFX \| RADEON_RESET_COMPUTE)) {
		3783	grbm_soft_reset \|= SOFT_RESET_DB \|
		3784	SOFT_RESET_CB \|
		3785	SOFT_RESET_PA \|
		3786	SOFT_RESET_SC \|
		3787	SOFT_RESET_SPI \|
		3788	SOFT_RESET_SX \|
		3789	SOFT_RESET_SH \|
		3790	SOFT_RESET_TC \|
		3791	SOFT_RESET_TA \|
		3792	SOFT_RESET_VC \|
		3793	SOFT_RESET_VGT;
		3794	}
1963	serge	3795
3764	Serge	3796	if (reset_mask & RADEON_RESET_CP) {
		3797	grbm_soft_reset \|= SOFT_RESET_CP \|
		3798	SOFT_RESET_VGT;
		3799
		3800	srbm_soft_reset \|= SOFT_RESET_GRBM;
		3801	}
		3802
		3803	if (reset_mask & RADEON_RESET_DMA)
		3804	srbm_soft_reset \|= SOFT_RESET_DMA;
		3805
		3806	if (reset_mask & RADEON_RESET_DISPLAY)
		3807	srbm_soft_reset \|= SOFT_RESET_DC;
		3808
		3809	if (reset_mask & RADEON_RESET_RLC)
		3810	srbm_soft_reset \|= SOFT_RESET_RLC;
		3811
		3812	if (reset_mask & RADEON_RESET_SEM)
		3813	srbm_soft_reset \|= SOFT_RESET_SEM;
		3814
		3815	if (reset_mask & RADEON_RESET_IH)
		3816	srbm_soft_reset \|= SOFT_RESET_IH;
		3817
		3818	if (reset_mask & RADEON_RESET_GRBM)
		3819	srbm_soft_reset \|= SOFT_RESET_GRBM;
		3820
		3821	if (reset_mask & RADEON_RESET_VMC)
		3822	srbm_soft_reset \|= SOFT_RESET_VMC;
		3823
		3824	if (!(rdev->flags & RADEON_IS_IGP)) {
		3825	if (reset_mask & RADEON_RESET_MC)
		3826	srbm_soft_reset \|= SOFT_RESET_MC;
		3827	}
		3828
		3829	if (grbm_soft_reset) {
		3830	tmp = RREG32(GRBM_SOFT_RESET);
		3831	tmp \|= grbm_soft_reset;
		3832	dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
		3833	WREG32(GRBM_SOFT_RESET, tmp);
		3834	tmp = RREG32(GRBM_SOFT_RESET);
		3835
		3836	udelay(50);
		3837
		3838	tmp &= ~grbm_soft_reset;
		3839	WREG32(GRBM_SOFT_RESET, tmp);
		3840	tmp = RREG32(GRBM_SOFT_RESET);
		3841	}
		3842
		3843	if (srbm_soft_reset) {
		3844	tmp = RREG32(SRBM_SOFT_RESET);
		3845	tmp \|= srbm_soft_reset;
		3846	dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
		3847	WREG32(SRBM_SOFT_RESET, tmp);
		3848	tmp = RREG32(SRBM_SOFT_RESET);
		3849
		3850	udelay(50);
		3851
		3852	tmp &= ~srbm_soft_reset;
		3853	WREG32(SRBM_SOFT_RESET, tmp);
		3854	tmp = RREG32(SRBM_SOFT_RESET);
		3855	}
		3856
1963	serge	3857	/* Wait a little for things to settle down */
		3858	udelay(50);
3764	Serge	3859
1963	serge	3860	evergreen_mc_resume(rdev, &save);
3764	Serge	3861	udelay(50);
		3862
		3863	evergreen_print_gpu_status_regs(rdev);
1430	serge	3864	}
		3865
5078	serge	3866	void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
		3867	{
		3868	struct evergreen_mc_save save;
		3869	u32 tmp, i;
		3870
		3871	dev_info(rdev->dev, "GPU pci config reset\n");
		3872
		3873	/* disable dpm? */
		3874
		3875	/* Disable CP parsing/prefetching */
		3876	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT);
		3877	udelay(50);
		3878	/* Disable DMA */
		3879	tmp = RREG32(DMA_RB_CNTL);
		3880	tmp &= ~DMA_RB_ENABLE;
		3881	WREG32(DMA_RB_CNTL, tmp);
		3882	/* XXX other engines? */
		3883
		3884	/* halt the rlc */
		3885	r600_rlc_stop(rdev);
		3886
		3887	udelay(50);
		3888
		3889	/* set mclk/sclk to bypass */
		3890	rv770_set_clk_bypass_mode(rdev);
		3891	/* disable BM */
		3892	pci_clear_master(rdev->pdev);
		3893	/* disable mem access */
		3894	evergreen_mc_stop(rdev, &save);
		3895	if (evergreen_mc_wait_for_idle(rdev)) {
		3896	dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
		3897	}
		3898	/* reset */
		3899	radeon_pci_config_reset(rdev);
		3900	/* wait for asic to come out of reset */
		3901	for (i = 0; i < rdev->usec_timeout; i++) {
		3902	if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
		3903	break;
		3904	udelay(1);
		3905	}
		3906	}
		3907
1963	serge	3908	int evergreen_asic_reset(struct radeon_device *rdev)
		3909	{
3764	Serge	3910	u32 reset_mask;
		3911
		3912	reset_mask = evergreen_gpu_check_soft_reset(rdev);
		3913
		3914	if (reset_mask)
		3915	r600_set_bios_scratch_engine_hung(rdev, true);
		3916
5078	serge	3917	/* try soft reset */
3764	Serge	3918	evergreen_gpu_soft_reset(rdev, reset_mask);
		3919
		3920	reset_mask = evergreen_gpu_check_soft_reset(rdev);
		3921
5078	serge	3922	/* try pci config reset */
		3923	if (reset_mask && radeon_hard_reset)
		3924	evergreen_gpu_pci_config_reset(rdev);
		3925
		3926	reset_mask = evergreen_gpu_check_soft_reset(rdev);
		3927
3764	Serge	3928	if (!reset_mask)
		3929	r600_set_bios_scratch_engine_hung(rdev, false);
		3930
		3931	return 0;
1963	serge	3932	}
		3933
3764	Serge	3934	/**
		3935	* evergreen_gfx_is_lockup - Check if the GFX engine is locked up
		3936	*
		3937	* @rdev: radeon_device pointer
		3938	* @ring: radeon_ring structure holding ring information
		3939	*
		3940	* Check if the GFX engine is locked up.
		3941	* Returns true if the engine appears to be locked up, false if not.
		3942	*/
		3943	bool evergreen_gfx_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
		3944	{
		3945	u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
		3946
		3947	if (!(reset_mask & (RADEON_RESET_GFX \|
		3948	RADEON_RESET_COMPUTE \|
		3949	RADEON_RESET_CP))) {
5078	serge	3950	radeon_ring_lockup_update(rdev, ring);
3764	Serge	3951	return false;
		3952	}
		3953	return radeon_ring_test_lockup(rdev, ring);
		3954	}
		3955
5078	serge	3956	/*
		3957	* RLC
3764	Serge	3958	*/
5078	serge	3959	#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
		3960	#define RLC_CLEAR_STATE_END_MARKER 0x00000001
		3961
		3962	void sumo_rlc_fini(struct radeon_device *rdev)
3764	Serge	3963	{
5078	serge	3964	int r;
3764	Serge	3965
5078	serge	3966	/* save restore block */
		3967	if (rdev->rlc.save_restore_obj) {
		3968	r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
		3969	if (unlikely(r != 0))
		3970	dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
		3971	radeon_bo_unpin(rdev->rlc.save_restore_obj);
		3972	radeon_bo_unreserve(rdev->rlc.save_restore_obj);
		3973
		3974	radeon_bo_unref(&rdev->rlc.save_restore_obj);
		3975	rdev->rlc.save_restore_obj = NULL;
3764	Serge	3976	}
5078	serge	3977
		3978	/* clear state block */
		3979	if (rdev->rlc.clear_state_obj) {
		3980	r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
		3981	if (unlikely(r != 0))
		3982	dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
		3983	radeon_bo_unpin(rdev->rlc.clear_state_obj);
		3984	radeon_bo_unreserve(rdev->rlc.clear_state_obj);
		3985
		3986	radeon_bo_unref(&rdev->rlc.clear_state_obj);
		3987	rdev->rlc.clear_state_obj = NULL;
		3988	}
		3989
		3990	/* clear state block */
		3991	if (rdev->rlc.cp_table_obj) {
		3992	r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
		3993	if (unlikely(r != 0))
		3994	dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
		3995	radeon_bo_unpin(rdev->rlc.cp_table_obj);
		3996	radeon_bo_unreserve(rdev->rlc.cp_table_obj);
		3997
		3998	radeon_bo_unref(&rdev->rlc.cp_table_obj);
		3999	rdev->rlc.cp_table_obj = NULL;
		4000	}
3764	Serge	4001	}
		4002
5078	serge	4003	#define CP_ME_TABLE_SIZE 96
		4004
		4005	int sumo_rlc_init(struct radeon_device *rdev)
		4006	{
		4007	const u32 *src_ptr;
		4008	volatile u32 *dst_ptr;
		4009	u32 dws, data, i, j, k, reg_num;
		4010	u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
		4011	u64 reg_list_mc_addr;
		4012	const struct cs_section_def *cs_data;
		4013	int r;
		4014
		4015	src_ptr = rdev->rlc.reg_list;
		4016	dws = rdev->rlc.reg_list_size;
		4017	if (rdev->family >= CHIP_BONAIRE) {
		4018	dws += (5 * 16) + 48 + 48 + 64;
		4019	}
		4020	cs_data = rdev->rlc.cs_data;
		4021
		4022	if (src_ptr) {
		4023	/* save restore block */
		4024	if (rdev->rlc.save_restore_obj == NULL) {
		4025	r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
		4026	RADEON_GEM_DOMAIN_VRAM, 0, NULL,
5271	serge	4027	NULL, &rdev->rlc.save_restore_obj);
5078	serge	4028	if (r) {
		4029	dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
		4030	return r;
		4031	}
		4032	}
		4033
		4034	r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
		4035	if (unlikely(r != 0)) {
		4036	sumo_rlc_fini(rdev);
		4037	return r;
		4038	}
		4039	r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
		4040	&rdev->rlc.save_restore_gpu_addr);
		4041	if (r) {
		4042	radeon_bo_unreserve(rdev->rlc.save_restore_obj);
		4043	dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
		4044	sumo_rlc_fini(rdev);
		4045	return r;
		4046	}
		4047
		4048	r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
		4049	if (r) {
		4050	dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
		4051	sumo_rlc_fini(rdev);
		4052	return r;
		4053	}
		4054	/* write the sr buffer */
		4055	dst_ptr = rdev->rlc.sr_ptr;
		4056	if (rdev->family >= CHIP_TAHITI) {
		4057	/* SI */
		4058	for (i = 0; i < rdev->rlc.reg_list_size; i++)
		4059	dst_ptr[i] = cpu_to_le32(src_ptr[i]);
		4060	} else {
		4061	/* ON/LN/TN */
		4062	/* format:
		4063	* dw0: (reg2 << 16) \| reg1
		4064	* dw1: reg1 save space
		4065	* dw2: reg2 save space
		4066	*/
		4067	for (i = 0; i < dws; i++) {
		4068	data = src_ptr[i] >> 2;
		4069	i++;
		4070	if (i < dws)
		4071	data \|= (src_ptr[i] >> 2) << 16;
		4072	j = (((i - 1) * 3) / 2);
		4073	dst_ptr[j] = cpu_to_le32(data);
		4074	}
		4075	j = ((i * 3) / 2);
		4076	dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
		4077	}
		4078	radeon_bo_kunmap(rdev->rlc.save_restore_obj);
		4079	radeon_bo_unreserve(rdev->rlc.save_restore_obj);
		4080	}
		4081
		4082	if (cs_data) {
		4083	/* clear state block */
		4084	if (rdev->family >= CHIP_BONAIRE) {
		4085	rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
		4086	} else if (rdev->family >= CHIP_TAHITI) {
		4087	rdev->rlc.clear_state_size = si_get_csb_size(rdev);
		4088	dws = rdev->rlc.clear_state_size + (256 / 4);
		4089	} else {
		4090	reg_list_num = 0;
		4091	dws = 0;
		4092	for (i = 0; cs_data[i].section != NULL; i++) {
		4093	for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
		4094	reg_list_num++;
		4095	dws += cs_data[i].section[j].reg_count;
		4096	}
		4097	}
		4098	reg_list_blk_index = (3 * reg_list_num + 2);
		4099	dws += reg_list_blk_index;
		4100	rdev->rlc.clear_state_size = dws;
		4101	}
		4102
		4103	if (rdev->rlc.clear_state_obj == NULL) {
		4104	r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
		4105	RADEON_GEM_DOMAIN_VRAM, 0, NULL,
5271	serge	4106	NULL, &rdev->rlc.clear_state_obj);
5078	serge	4107	if (r) {
		4108	dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
		4109	sumo_rlc_fini(rdev);
		4110	return r;
		4111	}
		4112	}
		4113	r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
		4114	if (unlikely(r != 0)) {
		4115	sumo_rlc_fini(rdev);
		4116	return r;
		4117	}
		4118	r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
		4119	&rdev->rlc.clear_state_gpu_addr);
		4120	if (r) {
		4121	radeon_bo_unreserve(rdev->rlc.clear_state_obj);
		4122	dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
		4123	sumo_rlc_fini(rdev);
		4124	return r;
		4125	}
		4126
		4127	r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
		4128	if (r) {
		4129	dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
		4130	sumo_rlc_fini(rdev);
		4131	return r;
		4132	}
		4133	/* set up the cs buffer */
		4134	dst_ptr = rdev->rlc.cs_ptr;
		4135	if (rdev->family >= CHIP_BONAIRE) {
		4136	cik_get_csb_buffer(rdev, dst_ptr);
		4137	} else if (rdev->family >= CHIP_TAHITI) {
		4138	reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
		4139	dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
		4140	dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
		4141	dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
		4142	si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
		4143	} else {
		4144	reg_list_hdr_blk_index = 0;
		4145	reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
		4146	data = upper_32_bits(reg_list_mc_addr);
		4147	dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
		4148	reg_list_hdr_blk_index++;
		4149	for (i = 0; cs_data[i].section != NULL; i++) {
		4150	for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
		4151	reg_num = cs_data[i].section[j].reg_count;
		4152	data = reg_list_mc_addr & 0xffffffff;
		4153	dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
		4154	reg_list_hdr_blk_index++;
		4155
		4156	data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
		4157	dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
		4158	reg_list_hdr_blk_index++;
		4159
		4160	data = 0x08000000 \| (reg_num * 4);
		4161	dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
		4162	reg_list_hdr_blk_index++;
		4163
		4164	for (k = 0; k < reg_num; k++) {
		4165	data = cs_data[i].section[j].extent[k];
		4166	dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
		4167	}
		4168	reg_list_mc_addr += reg_num * 4;
		4169	reg_list_blk_index += reg_num;
		4170	}
		4171	}
		4172	dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
		4173	}
		4174	radeon_bo_kunmap(rdev->rlc.clear_state_obj);
		4175	radeon_bo_unreserve(rdev->rlc.clear_state_obj);
		4176	}
		4177
		4178	if (rdev->rlc.cp_table_size) {
		4179	if (rdev->rlc.cp_table_obj == NULL) {
		4180	r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
		4181	PAGE_SIZE, true,
		4182	RADEON_GEM_DOMAIN_VRAM, 0, NULL,
5271	serge	4183	NULL, &rdev->rlc.cp_table_obj);
5078	serge	4184	if (r) {
		4185	dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
		4186	sumo_rlc_fini(rdev);
		4187	return r;
		4188	}
		4189	}
		4190
		4191	r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
		4192	if (unlikely(r != 0)) {
		4193	dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
		4194	sumo_rlc_fini(rdev);
		4195	return r;
		4196	}
		4197	r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
		4198	&rdev->rlc.cp_table_gpu_addr);
		4199	if (r) {
		4200	radeon_bo_unreserve(rdev->rlc.cp_table_obj);
		4201	dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
		4202	sumo_rlc_fini(rdev);
		4203	return r;
		4204	}
		4205	r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
		4206	if (r) {
		4207	dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
		4208	sumo_rlc_fini(rdev);
		4209	return r;
		4210	}
		4211
		4212	cik_init_cp_pg_table(rdev);
		4213
		4214	radeon_bo_kunmap(rdev->rlc.cp_table_obj);
		4215	radeon_bo_unreserve(rdev->rlc.cp_table_obj);
		4216
		4217	}
		4218
		4219	return 0;
		4220	}
		4221
		4222	static void evergreen_rlc_start(struct radeon_device *rdev)
		4223	{
		4224	u32 mask = RLC_ENABLE;
		4225
		4226	if (rdev->flags & RADEON_IS_IGP) {
		4227	mask \|= GFX_POWER_GATING_ENABLE \| GFX_POWER_GATING_SRC;
		4228	}
		4229
		4230	WREG32(RLC_CNTL, mask);
		4231	}
		4232
		4233	int evergreen_rlc_resume(struct radeon_device *rdev)
		4234	{
		4235	u32 i;
		4236	const __be32 *fw_data;
		4237
		4238	if (!rdev->rlc_fw)
		4239	return -EINVAL;
		4240
		4241	r600_rlc_stop(rdev);
		4242
		4243	WREG32(RLC_HB_CNTL, 0);
		4244
		4245	if (rdev->flags & RADEON_IS_IGP) {
		4246	if (rdev->family == CHIP_ARUBA) {
		4247	u32 always_on_bitmap =
		4248	3 \| (3 << (16 * rdev->config.cayman.max_shader_engines));
		4249	/* find out the number of active simds */
		4250	u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
		4251	tmp \|= 0xffffffff << rdev->config.cayman.max_simds_per_se;
		4252	tmp = hweight32(~tmp);
		4253	if (tmp == rdev->config.cayman.max_simds_per_se) {
		4254	WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
		4255	WREG32(TN_RLC_LB_PARAMS, 0x00601004);
		4256	WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
		4257	WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
		4258	WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
		4259	}
		4260	} else {
		4261	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
		4262	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
		4263	}
		4264	WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		4265	WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		4266	} else {
		4267	WREG32(RLC_HB_BASE, 0);
		4268	WREG32(RLC_HB_RPTR, 0);
		4269	WREG32(RLC_HB_WPTR, 0);
		4270	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
		4271	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
		4272	}
		4273	WREG32(RLC_MC_CNTL, 0);
		4274	WREG32(RLC_UCODE_CNTL, 0);
		4275
		4276	fw_data = (const __be32 *)rdev->rlc_fw->data;
		4277	if (rdev->family >= CHIP_ARUBA) {
		4278	for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
		4279	WREG32(RLC_UCODE_ADDR, i);
		4280	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		4281	}
		4282	} else if (rdev->family >= CHIP_CAYMAN) {
		4283	for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
		4284	WREG32(RLC_UCODE_ADDR, i);
		4285	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		4286	}
		4287	} else {
		4288	for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
		4289	WREG32(RLC_UCODE_ADDR, i);
		4290	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		4291	}
		4292	}
		4293	WREG32(RLC_UCODE_ADDR, 0);
		4294
		4295	evergreen_rlc_start(rdev);
		4296
		4297	return 0;
		4298	}
		4299
1963	serge	4300	/* Interrupts */
		4301
		4302	u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
		4303	{
2997	Serge	4304	if (crtc >= rdev->num_crtc)
3031	serge	4305	return 0;
2997	Serge	4306	else
		4307	return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
1963	serge	4308	}
		4309
		4310	void evergreen_disable_interrupt_state(struct radeon_device *rdev)
		4311	{
		4312	u32 tmp;
		4313
2997	Serge	4314	if (rdev->family >= CHIP_CAYMAN) {
		4315	cayman_cp_int_cntl_setup(rdev, 0,
		4316	CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		4317	cayman_cp_int_cntl_setup(rdev, 1, 0);
		4318	cayman_cp_int_cntl_setup(rdev, 2, 0);
3192	Serge	4319	tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
		4320	WREG32(CAYMAN_DMA1_CNTL, tmp);
2997	Serge	4321	} else
3031	serge	4322	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
3192	Serge	4323	tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
		4324	WREG32(DMA_CNTL, tmp);
1963	serge	4325	WREG32(GRBM_INT_CNTL, 0);
		4326	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		4327	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005	serge	4328	if (rdev->num_crtc >= 4) {
3031	serge	4329	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		4330	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005	serge	4331	}
		4332	if (rdev->num_crtc >= 6) {
3031	serge	4333	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		4334	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	4335	}
		4336
		4337	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		4338	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005	serge	4339	if (rdev->num_crtc >= 4) {
3031	serge	4340	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		4341	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005	serge	4342	}
		4343	if (rdev->num_crtc >= 6) {
3031	serge	4344	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		4345	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963	serge	4346	}
		4347
5078	serge	4348	/* only one DAC on DCE5 */
		4349	if (!ASIC_IS_DCE5(rdev))
3031	serge	4350	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
1963	serge	4351	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		4352
		4353	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4354	WREG32(DC_HPD1_INT_CONTROL, tmp);
		4355	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4356	WREG32(DC_HPD2_INT_CONTROL, tmp);
		4357	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4358	WREG32(DC_HPD3_INT_CONTROL, tmp);
		4359	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4360	WREG32(DC_HPD4_INT_CONTROL, tmp);
		4361	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4362	WREG32(DC_HPD5_INT_CONTROL, tmp);
		4363	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4364	WREG32(DC_HPD6_INT_CONTROL, tmp);
		4365
		4366	}
2005	serge	4367
		4368	int evergreen_irq_set(struct radeon_device *rdev)
		4369	{
		4370	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
2997	Serge	4371	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
2005	serge	4372	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
		4373	u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
		4374	u32 grbm_int_cntl = 0;
2997	Serge	4375	u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
3192	Serge	4376	u32 dma_cntl, dma_cntl1 = 0;
5078	serge	4377	u32 thermal_int = 0;
2005	serge	4378
		4379	if (!rdev->irq.installed) {
		4380	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		4381	return -EINVAL;
		4382	}
		4383	/* don't enable anything if the ih is disabled */
		4384	if (!rdev->ih.enabled) {
		4385	r600_disable_interrupts(rdev);
		4386	/* force the active interrupt state to all disabled */
		4387	evergreen_disable_interrupt_state(rdev);
		4388	return 0;
		4389	}
		4390
		4391	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		4392	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		4393	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		4394	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		4395	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		4396	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5078	serge	4397	if (rdev->family == CHIP_ARUBA)
		4398	thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
		4399	~(THERM_INT_MASK_HIGH \| THERM_INT_MASK_LOW);
		4400	else
		4401	thermal_int = RREG32(CG_THERMAL_INT) &
		4402	~(THERM_INT_MASK_HIGH \| THERM_INT_MASK_LOW);
2005	serge	4403
2997	Serge	4404	afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		4405	afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		4406	afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		4407	afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		4408	afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		4409	afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		4410
3192	Serge	4411	dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
		4412
2997	Serge	4413	if (rdev->family >= CHIP_CAYMAN) {
		4414	/* enable CP interrupts on all rings */
		4415	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		4416	DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
		4417	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		4418	}
		4419	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
		4420	DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
		4421	cp_int_cntl1 \|= TIME_STAMP_INT_ENABLE;
		4422	}
		4423	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
		4424	DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
		4425	cp_int_cntl2 \|= TIME_STAMP_INT_ENABLE;
		4426	}
		4427	} else {
		4428	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		4429	DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
3031	serge	4430	cp_int_cntl \|= RB_INT_ENABLE;
		4431	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		4432	}
2005	serge	4433	}
2997	Serge	4434
3192	Serge	4435	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
		4436	DRM_DEBUG("r600_irq_set: sw int dma\n");
		4437	dma_cntl \|= TRAP_ENABLE;
		4438	}
		4439
		4440	if (rdev->family >= CHIP_CAYMAN) {
		4441	dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
		4442	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
		4443	DRM_DEBUG("r600_irq_set: sw int dma1\n");
		4444	dma_cntl1 \|= TRAP_ENABLE;
		4445	}
		4446	}
		4447
5078	serge	4448	if (rdev->irq.dpm_thermal) {
		4449	DRM_DEBUG("dpm thermal\n");
		4450	thermal_int \|= THERM_INT_MASK_HIGH \| THERM_INT_MASK_LOW;
		4451	}
		4452
2005	serge	4453	if (rdev->irq.crtc_vblank_int[0] \|\|
2997	Serge	4454	atomic_read(&rdev->irq.pflip[0])) {
2005	serge	4455	DRM_DEBUG("evergreen_irq_set: vblank 0\n");
		4456	crtc1 \|= VBLANK_INT_MASK;
		4457	}
		4458	if (rdev->irq.crtc_vblank_int[1] \|\|
2997	Serge	4459	atomic_read(&rdev->irq.pflip[1])) {
2005	serge	4460	DRM_DEBUG("evergreen_irq_set: vblank 1\n");
		4461	crtc2 \|= VBLANK_INT_MASK;
		4462	}
		4463	if (rdev->irq.crtc_vblank_int[2] \|\|
2997	Serge	4464	atomic_read(&rdev->irq.pflip[2])) {
2005	serge	4465	DRM_DEBUG("evergreen_irq_set: vblank 2\n");
		4466	crtc3 \|= VBLANK_INT_MASK;
		4467	}
		4468	if (rdev->irq.crtc_vblank_int[3] \|\|
2997	Serge	4469	atomic_read(&rdev->irq.pflip[3])) {
2005	serge	4470	DRM_DEBUG("evergreen_irq_set: vblank 3\n");
		4471	crtc4 \|= VBLANK_INT_MASK;
		4472	}
		4473	if (rdev->irq.crtc_vblank_int[4] \|\|
2997	Serge	4474	atomic_read(&rdev->irq.pflip[4])) {
2005	serge	4475	DRM_DEBUG("evergreen_irq_set: vblank 4\n");
		4476	crtc5 \|= VBLANK_INT_MASK;
		4477	}
		4478	if (rdev->irq.crtc_vblank_int[5] \|\|
2997	Serge	4479	atomic_read(&rdev->irq.pflip[5])) {
2005	serge	4480	DRM_DEBUG("evergreen_irq_set: vblank 5\n");
		4481	crtc6 \|= VBLANK_INT_MASK;
		4482	}
		4483	if (rdev->irq.hpd[0]) {
		4484	DRM_DEBUG("evergreen_irq_set: hpd 1\n");
		4485	hpd1 \|= DC_HPDx_INT_EN;
		4486	}
		4487	if (rdev->irq.hpd[1]) {
		4488	DRM_DEBUG("evergreen_irq_set: hpd 2\n");
		4489	hpd2 \|= DC_HPDx_INT_EN;
		4490	}
		4491	if (rdev->irq.hpd[2]) {
		4492	DRM_DEBUG("evergreen_irq_set: hpd 3\n");
		4493	hpd3 \|= DC_HPDx_INT_EN;
		4494	}
		4495	if (rdev->irq.hpd[3]) {
		4496	DRM_DEBUG("evergreen_irq_set: hpd 4\n");
		4497	hpd4 \|= DC_HPDx_INT_EN;
		4498	}
		4499	if (rdev->irq.hpd[4]) {
		4500	DRM_DEBUG("evergreen_irq_set: hpd 5\n");
		4501	hpd5 \|= DC_HPDx_INT_EN;
		4502	}
		4503	if (rdev->irq.hpd[5]) {
		4504	DRM_DEBUG("evergreen_irq_set: hpd 6\n");
		4505	hpd6 \|= DC_HPDx_INT_EN;
		4506	}
2997	Serge	4507	if (rdev->irq.afmt[0]) {
		4508	DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
		4509	afmt1 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
2005	serge	4510	}
2997	Serge	4511	if (rdev->irq.afmt[1]) {
		4512	DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
		4513	afmt2 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		4514	}
		4515	if (rdev->irq.afmt[2]) {
		4516	DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
		4517	afmt3 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		4518	}
		4519	if (rdev->irq.afmt[3]) {
		4520	DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
		4521	afmt4 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		4522	}
		4523	if (rdev->irq.afmt[4]) {
		4524	DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
		4525	afmt5 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		4526	}
		4527	if (rdev->irq.afmt[5]) {
		4528	DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
		4529	afmt6 \|= AFMT_AZ_FORMAT_WTRIG_MASK;
		4530	}
2005	serge	4531
2997	Serge	4532	if (rdev->family >= CHIP_CAYMAN) {
		4533	cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
		4534	cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
		4535	cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
		4536	} else
3031	serge	4537	WREG32(CP_INT_CNTL, cp_int_cntl);
3192	Serge	4538
		4539	WREG32(DMA_CNTL, dma_cntl);
		4540
		4541	if (rdev->family >= CHIP_CAYMAN)
		4542	WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
		4543
2005	serge	4544	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		4545
		4546	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
		4547	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
		4548	if (rdev->num_crtc >= 4) {
		4549	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
		4550	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
		4551	}
		4552	if (rdev->num_crtc >= 6) {
		4553	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
		4554	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
		4555	}
		4556
5078	serge	4557	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
		4558	GRPH_PFLIP_INT_MASK);
		4559	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
		4560	GRPH_PFLIP_INT_MASK);
2005	serge	4561	if (rdev->num_crtc >= 4) {
5078	serge	4562	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
		4563	GRPH_PFLIP_INT_MASK);
		4564	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
		4565	GRPH_PFLIP_INT_MASK);
2005	serge	4566	}
		4567	if (rdev->num_crtc >= 6) {
5078	serge	4568	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
		4569	GRPH_PFLIP_INT_MASK);
		4570	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
		4571	GRPH_PFLIP_INT_MASK);
2005	serge	4572	}
		4573
		4574	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		4575	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		4576	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		4577	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		4578	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		4579	WREG32(DC_HPD6_INT_CONTROL, hpd6);
5078	serge	4580	if (rdev->family == CHIP_ARUBA)
		4581	WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
		4582	else
		4583	WREG32(CG_THERMAL_INT, thermal_int);
2005	serge	4584
2997	Serge	4585	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
		4586	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
		4587	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
		4588	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
		4589	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
		4590	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
		4591
2005	serge	4592	return 0;
		4593	}
		4594
2997	Serge	4595	static void evergreen_irq_ack(struct radeon_device *rdev)
2005	serge	4596	{
		4597	u32 tmp;
		4598
		4599	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		4600	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		4601	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
		4602	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
		4603	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
		4604	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
		4605	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		4606	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		4607	if (rdev->num_crtc >= 4) {
		4608	rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		4609	rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		4610	}
		4611	if (rdev->num_crtc >= 6) {
		4612	rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		4613	rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		4614	}
		4615
2997	Serge	4616	rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		4617	rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		4618	rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		4619	rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		4620	rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		4621	rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		4622
2005	serge	4623	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
		4624	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4625	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
		4626	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4627	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
		4628	WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
		4629	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
		4630	WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
		4631	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
		4632	WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
		4633	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
		4634	WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
		4635
		4636	if (rdev->num_crtc >= 4) {
		4637	if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
		4638	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4639	if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
		4640	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4641	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
		4642	WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
		4643	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
		4644	WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
		4645	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
		4646	WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
		4647	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
		4648	WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
		4649	}
		4650
		4651	if (rdev->num_crtc >= 6) {
		4652	if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
		4653	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4654	if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
		4655	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4656	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
		4657	WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
		4658	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
		4659	WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
		4660	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
		4661	WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
		4662	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
		4663	WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
		4664	}
		4665
		4666	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		4667	tmp = RREG32(DC_HPD1_INT_CONTROL);
		4668	tmp \|= DC_HPDx_INT_ACK;
		4669	WREG32(DC_HPD1_INT_CONTROL, tmp);
		4670	}
		4671	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		4672	tmp = RREG32(DC_HPD2_INT_CONTROL);
		4673	tmp \|= DC_HPDx_INT_ACK;
		4674	WREG32(DC_HPD2_INT_CONTROL, tmp);
		4675	}
		4676	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		4677	tmp = RREG32(DC_HPD3_INT_CONTROL);
		4678	tmp \|= DC_HPDx_INT_ACK;
		4679	WREG32(DC_HPD3_INT_CONTROL, tmp);
		4680	}
		4681	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		4682	tmp = RREG32(DC_HPD4_INT_CONTROL);
		4683	tmp \|= DC_HPDx_INT_ACK;
		4684	WREG32(DC_HPD4_INT_CONTROL, tmp);
		4685	}
		4686	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		4687	tmp = RREG32(DC_HPD5_INT_CONTROL);
		4688	tmp \|= DC_HPDx_INT_ACK;
		4689	WREG32(DC_HPD5_INT_CONTROL, tmp);
		4690	}
		4691	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		4692	tmp = RREG32(DC_HPD5_INT_CONTROL);
		4693	tmp \|= DC_HPDx_INT_ACK;
		4694	WREG32(DC_HPD6_INT_CONTROL, tmp);
		4695	}
2997	Serge	4696	if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
		4697	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
		4698	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		4699	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
		4700	}
		4701	if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
		4702	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
		4703	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		4704	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
		4705	}
		4706	if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
		4707	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
		4708	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		4709	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
		4710	}
		4711	if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
		4712	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
		4713	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		4714	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
		4715	}
		4716	if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
		4717	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
		4718	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		4719	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
		4720	}
		4721	if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
		4722	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
		4723	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		4724	WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
		4725	}
2005	serge	4726	}
2997	Serge	4727
		4728	static void evergreen_irq_disable(struct radeon_device *rdev)
2005	serge	4729	{
2997	Serge	4730	r600_disable_interrupts(rdev);
		4731	/* Wait and acknowledge irq */
		4732	mdelay(1);
		4733	evergreen_irq_ack(rdev);
		4734	evergreen_disable_interrupt_state(rdev);
		4735	}
		4736
		4737	void evergreen_irq_suspend(struct radeon_device *rdev)
		4738	{
		4739	evergreen_irq_disable(rdev);
		4740	r600_rlc_stop(rdev);
		4741	}
		4742
		4743	static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
		4744	{
2005	serge	4745	u32 wptr, tmp;
		4746
		4747	if (rdev->wb.enabled)
		4748	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		4749	else
		4750	wptr = RREG32(IH_RB_WPTR);
		4751
		4752	if (wptr & RB_OVERFLOW) {
5179	serge	4753	wptr &= ~RB_OVERFLOW;
2005	serge	4754	/* When a ring buffer overflow happen start parsing interrupt
		4755	* from the last not overwritten vector (wptr + 16). Hopefully
		4756	* this should allow us to catchup.
		4757	*/
5179	serge	4758	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
		4759	wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
2005	serge	4760	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		4761	tmp = RREG32(IH_RB_CNTL);
		4762	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		4763	WREG32(IH_RB_CNTL, tmp);
		4764	}
		4765	return (wptr & rdev->ih.ptr_mask);
		4766	}
		4767
		4768	int evergreen_irq_process(struct radeon_device *rdev)
		4769	{
		4770	u32 wptr;
		4771	u32 rptr;
		4772	u32 src_id, src_data;
		4773	u32 ring_index;
		4774	bool queue_hotplug = false;
2997	Serge	4775	bool queue_hdmi = false;
5078	serge	4776	bool queue_thermal = false;
		4777	u32 status, addr;
2005	serge	4778
		4779	if (!rdev->ih.enabled \|\| rdev->shutdown)
		4780	return IRQ_NONE;
		4781
		4782	wptr = evergreen_get_ih_wptr(rdev);
2997	Serge	4783
		4784	restart_ih:
		4785	/* is somebody else already processing irqs? */
		4786	if (atomic_xchg(&rdev->ih.lock, 1))
		4787	return IRQ_NONE;
		4788
2005	serge	4789	rptr = rdev->ih.rptr;
		4790	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		4791
2175	serge	4792	/* Order reading of wptr vs. reading of IH ring data */
		4793	rmb();
		4794
2005	serge	4795	/* display interrupts */
		4796	evergreen_irq_ack(rdev);
		4797
		4798	while (rptr != wptr) {
		4799	/* wptr/rptr are in bytes! */
		4800	ring_index = rptr / 4;
		4801	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		4802	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		4803
		4804	switch (src_id) {
		4805	case 1: /* D1 vblank/vline */
		4806	switch (src_data) {
		4807	case 0: /* D1 vblank */
		4808	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
		4809	if (rdev->irq.crtc_vblank_int[0]) {
		4810	// drm_handle_vblank(rdev->ddev, 0);
		4811	rdev->pm.vblank_sync = true;
		4812	// wake_up(&rdev->irq.vblank_queue);
		4813	}
		4814	// if (rdev->irq.pflip[0])
		4815	// radeon_crtc_handle_flip(rdev, 0);
		4816	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		4817	DRM_DEBUG("IH: D1 vblank\n");
		4818	}
		4819	break;
		4820	case 1: /* D1 vline */
		4821	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
		4822	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		4823	DRM_DEBUG("IH: D1 vline\n");
		4824	}
		4825	break;
		4826	default:
		4827	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4828	break;
		4829	}
		4830	break;
		4831	case 2: /* D2 vblank/vline */
		4832	switch (src_data) {
		4833	case 0: /* D2 vblank */
		4834	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
		4835	if (rdev->irq.crtc_vblank_int[1]) {
		4836	// drm_handle_vblank(rdev->ddev, 1);
		4837	rdev->pm.vblank_sync = true;
		4838	// wake_up(&rdev->irq.vblank_queue);
		4839	}
		4840	// if (rdev->irq.pflip[1])
		4841	// radeon_crtc_handle_flip(rdev, 1);
		4842	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
		4843	DRM_DEBUG("IH: D2 vblank\n");
		4844	}
		4845	break;
		4846	case 1: /* D2 vline */
		4847	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
		4848	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
		4849	DRM_DEBUG("IH: D2 vline\n");
		4850	}
		4851	break;
		4852	default:
		4853	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4854	break;
		4855	}
		4856	break;
		4857	case 3: /* D3 vblank/vline */
		4858	switch (src_data) {
		4859	case 0: /* D3 vblank */
		4860	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
		4861	if (rdev->irq.crtc_vblank_int[2]) {
		4862	// drm_handle_vblank(rdev->ddev, 2);
		4863	rdev->pm.vblank_sync = true;
		4864	// wake_up(&rdev->irq.vblank_queue);
		4865	}
		4866	// if (rdev->irq.pflip[2])
		4867	// radeon_crtc_handle_flip(rdev, 2);
		4868	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
		4869	DRM_DEBUG("IH: D3 vblank\n");
		4870	}
		4871	break;
		4872	case 1: /* D3 vline */
		4873	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
		4874	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
		4875	DRM_DEBUG("IH: D3 vline\n");
		4876	}
		4877	break;
		4878	default:
		4879	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4880	break;
		4881	}
		4882	break;
		4883	case 4: /* D4 vblank/vline */
		4884	switch (src_data) {
		4885	case 0: /* D4 vblank */
		4886	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
		4887	if (rdev->irq.crtc_vblank_int[3]) {
		4888	// drm_handle_vblank(rdev->ddev, 3);
		4889	rdev->pm.vblank_sync = true;
		4890	// wake_up(&rdev->irq.vblank_queue);
		4891	}
		4892	// if (rdev->irq.pflip[3])
		4893	// radeon_crtc_handle_flip(rdev, 3);
		4894	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
		4895	DRM_DEBUG("IH: D4 vblank\n");
		4896	}
		4897	break;
		4898	case 1: /* D4 vline */
		4899	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
		4900	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
		4901	DRM_DEBUG("IH: D4 vline\n");
		4902	}
		4903	break;
		4904	default:
		4905	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4906	break;
		4907	}
		4908	break;
		4909	case 5: /* D5 vblank/vline */
		4910	switch (src_data) {
		4911	case 0: /* D5 vblank */
		4912	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
		4913	if (rdev->irq.crtc_vblank_int[4]) {
		4914	// drm_handle_vblank(rdev->ddev, 4);
		4915	rdev->pm.vblank_sync = true;
		4916	// wake_up(&rdev->irq.vblank_queue);
		4917	}
		4918	// if (rdev->irq.pflip[4])
		4919	// radeon_crtc_handle_flip(rdev, 4);
		4920	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
		4921	DRM_DEBUG("IH: D5 vblank\n");
		4922	}
		4923	break;
		4924	case 1: /* D5 vline */
		4925	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
		4926	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
		4927	DRM_DEBUG("IH: D5 vline\n");
		4928	}
		4929	break;
		4930	default:
		4931	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4932	break;
		4933	}
		4934	break;
		4935	case 6: /* D6 vblank/vline */
		4936	switch (src_data) {
		4937	case 0: /* D6 vblank */
		4938	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
		4939	if (rdev->irq.crtc_vblank_int[5]) {
		4940	// drm_handle_vblank(rdev->ddev, 5);
		4941	rdev->pm.vblank_sync = true;
		4942	// wake_up(&rdev->irq.vblank_queue);
		4943	}
		4944	// if (rdev->irq.pflip[5])
		4945	// radeon_crtc_handle_flip(rdev, 5);
		4946	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
		4947	DRM_DEBUG("IH: D6 vblank\n");
		4948	}
		4949	break;
		4950	case 1: /* D6 vline */
		4951	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
		4952	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
		4953	DRM_DEBUG("IH: D6 vline\n");
		4954	}
		4955	break;
		4956	default:
		4957	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4958	break;
		4959	}
		4960	break;
5078	serge	4961	case 8: /* D1 page flip */
		4962	case 10: /* D2 page flip */
		4963	case 12: /* D3 page flip */
		4964	case 14: /* D4 page flip */
		4965	case 16: /* D5 page flip */
		4966	case 18: /* D6 page flip */
		4967	DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
		4968	break;
2005	serge	4969	case 42: /* HPD hotplug */
		4970	switch (src_data) {
		4971	case 0:
		4972	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		4973	rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
		4974	queue_hotplug = true;
		4975	DRM_DEBUG("IH: HPD1\n");
		4976	}
		4977	break;
		4978	case 1:
		4979	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		4980	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
		4981	queue_hotplug = true;
		4982	DRM_DEBUG("IH: HPD2\n");
		4983	}
		4984	break;
		4985	case 2:
		4986	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		4987	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
		4988	queue_hotplug = true;
		4989	DRM_DEBUG("IH: HPD3\n");
		4990	}
		4991	break;
		4992	case 3:
		4993	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		4994	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
		4995	queue_hotplug = true;
		4996	DRM_DEBUG("IH: HPD4\n");
		4997	}
		4998	break;
		4999	case 4:
		5000	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		5001	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
		5002	queue_hotplug = true;
		5003	DRM_DEBUG("IH: HPD5\n");
		5004	}
		5005	break;
		5006	case 5:
		5007	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		5008	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
		5009	queue_hotplug = true;
		5010	DRM_DEBUG("IH: HPD6\n");
		5011	}
		5012	break;
		5013	default:
		5014	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		5015	break;
		5016	}
		5017	break;
2997	Serge	5018	case 44: /* hdmi */
		5019	switch (src_data) {
		5020	case 0:
		5021	if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
		5022	rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
		5023	queue_hdmi = true;
		5024	DRM_DEBUG("IH: HDMI0\n");
		5025	}
		5026	break;
		5027	case 1:
		5028	if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
		5029	rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
		5030	queue_hdmi = true;
		5031	DRM_DEBUG("IH: HDMI1\n");
		5032	}
		5033	break;
		5034	case 2:
		5035	if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
		5036	rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
		5037	queue_hdmi = true;
		5038	DRM_DEBUG("IH: HDMI2\n");
		5039	}
		5040	break;
		5041	case 3:
		5042	if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
		5043	rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
		5044	queue_hdmi = true;
		5045	DRM_DEBUG("IH: HDMI3\n");
		5046	}
		5047	break;
		5048	case 4:
		5049	if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
		5050	rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
		5051	queue_hdmi = true;
		5052	DRM_DEBUG("IH: HDMI4\n");
		5053	}
		5054	break;
		5055	case 5:
		5056	if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
		5057	rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
		5058	queue_hdmi = true;
		5059	DRM_DEBUG("IH: HDMI5\n");
		5060	}
		5061	break;
		5062	default:
		5063	DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
		5064	break;
		5065	}
3764	Serge	5066	case 124: /* UVD */
		5067	DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
		5068	radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
2997	Serge	5069	break;
3192	Serge	5070	case 146:
		5071	case 147:
5078	serge	5072	addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
		5073	status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
		5074	/* reset addr and status */
		5075	WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
		5076	if (addr == 0x0 && status == 0x0)
		5077	break;
3192	Serge	5078	dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
		5079	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5078	serge	5080	addr);
3192	Serge	5081	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5078	serge	5082	status);
		5083	cayman_vm_decode_fault(rdev, status, addr);
3192	Serge	5084	break;
2005	serge	5085	case 176: /* CP_INT in ring buffer */
		5086	case 177: /* CP_INT in IB1 */
		5087	case 178: /* CP_INT in IB2 */
		5088	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2997	Serge	5089	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2005	serge	5090	break;
		5091	case 181: /* CP EOP event */
		5092	DRM_DEBUG("IH: CP EOP\n");
2997	Serge	5093	if (rdev->family >= CHIP_CAYMAN) {
		5094	switch (src_data) {
		5095	case 0:
		5096	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
		5097	break;
		5098	case 1:
		5099	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		5100	break;
		5101	case 2:
		5102	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		5103	break;
		5104	}
		5105	} else
		5106	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2005	serge	5107	break;
3192	Serge	5108	case 224: /* DMA trap event */
		5109	DRM_DEBUG("IH: DMA trap\n");
		5110	radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
		5111	break;
5078	serge	5112	case 230: /* thermal low to high */
		5113	DRM_DEBUG("IH: thermal low to high\n");
		5114	rdev->pm.dpm.thermal.high_to_low = false;
		5115	queue_thermal = true;
		5116	break;
		5117	case 231: /* thermal high to low */
		5118	DRM_DEBUG("IH: thermal high to low\n");
		5119	rdev->pm.dpm.thermal.high_to_low = true;
		5120	queue_thermal = true;
		5121	break;
2005	serge	5122	case 233: /* GUI IDLE */
		5123	DRM_DEBUG("IH: GUI idle\n");
		5124	break;
3192	Serge	5125	case 244: /* DMA trap event */
		5126	if (rdev->family >= CHIP_CAYMAN) {
		5127	DRM_DEBUG("IH: DMA1 trap\n");
		5128	radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
		5129	}
		5130	break;
2005	serge	5131	default:
		5132	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		5133	break;
		5134	}
		5135
		5136	/* wptr/rptr are in bytes! */
		5137	rptr += 16;
		5138	rptr &= rdev->ih.ptr_mask;
5271	serge	5139	WREG32(IH_RB_RPTR, rptr);
2005	serge	5140	}
2997	Serge	5141	rdev->ih.rptr = rptr;
		5142	atomic_set(&rdev->ih.lock, 0);
		5143
2005	serge	5144	/* make sure wptr hasn't changed while processing */
		5145	wptr = evergreen_get_ih_wptr(rdev);
2997	Serge	5146	if (wptr != rptr)
2005	serge	5147	goto restart_ih;
2997	Serge	5148
2005	serge	5149	return IRQ_HANDLED;
		5150	}
		5151
1430	serge	5152	static int evergreen_startup(struct radeon_device *rdev)
		5153	{
3764	Serge	5154	struct radeon_ring *ring;
1430	serge	5155	int r;
		5156
1990	serge	5157	/* enable pcie gen2 link */
3031	serge	5158	evergreen_pcie_gen2_enable(rdev);
5078	serge	5159	/* enable aspm */
		5160	evergreen_program_aspm(rdev);
1990	serge	5161
5078	serge	5162	/* scratch needs to be initialized before MC */
		5163	r = r600_vram_scratch_init(rdev);
		5164	if (r)
		5165	return r;
		5166
		5167	evergreen_mc_program(rdev);
		5168
		5169	if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
1990	serge	5170	r = ni_mc_load_microcode(rdev);
		5171	if (r) {
		5172	DRM_ERROR("Failed to load MC firmware!\n");
		5173	return r;
		5174	}
1430	serge	5175	}
1963	serge	5176
1430	serge	5177	if (rdev->flags & RADEON_IS_AGP) {
1963	serge	5178	evergreen_agp_enable(rdev);
1430	serge	5179	} else {
		5180	r = evergreen_pcie_gart_enable(rdev);
		5181	if (r)
		5182	return r;
		5183	}
		5184	evergreen_gpu_init(rdev);
2005	serge	5185
5078	serge	5186	/* allocate rlc buffers */
		5187	if (rdev->flags & RADEON_IS_IGP) {
		5188	rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
		5189	rdev->rlc.reg_list_size =
		5190	(u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
		5191	rdev->rlc.cs_data = evergreen_cs_data;
		5192	r = sumo_rlc_init(rdev);
1963	serge	5193	if (r) {
5078	serge	5194	DRM_ERROR("Failed to init rlc BOs!\n");
		5195	return r;
		5196	}
1963	serge	5197	}
1430	serge	5198
1963	serge	5199	/* allocate wb buffer */
		5200	r = radeon_wb_init(rdev);
		5201	if (r)
		5202	return r;
		5203
3192	Serge	5204	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
		5205	if (r) {
		5206	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		5207	return r;
		5208	}
		5209
		5210	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
		5211	if (r) {
		5212	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		5213	return r;
		5214	}
		5215
3764	Serge	5216	// r = rv770_uvd_resume(rdev);
		5217	// if (!r) {
		5218	// r = radeon_fence_driver_start_ring(rdev,
		5219	// R600_RING_TYPE_UVD_INDEX);
		5220	// if (r)
		5221	// dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
		5222	// }
		5223
5078	serge	5224	if (r)
		5225	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
3764	Serge	5226
1963	serge	5227	/* Enable IRQ */
3764	Serge	5228	if (!rdev->irq.installed) {
		5229	r = radeon_irq_kms_init(rdev);
		5230	if (r)
		5231	return r;
		5232	}
		5233
2005	serge	5234	r = r600_irq_init(rdev);
		5235	if (r) {
		5236	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		5237	// radeon_irq_kms_fini(rdev);
		5238	return r;
		5239	}
		5240	evergreen_irq_set(rdev);
1963	serge	5241
3764	Serge	5242	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2997	Serge	5243	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5078	serge	5244	RADEON_CP_PACKET2);
1430	serge	5245	if (r)
		5246	return r;
3192	Serge	5247
		5248	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		5249	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5078	serge	5250	DMA_PACKET(DMA_PACKET_NOP, 0, 0));
3192	Serge	5251	if (r)
		5252	return r;
		5253
1430	serge	5254	r = evergreen_cp_load_microcode(rdev);
		5255	if (r)
		5256	return r;
1963	serge	5257	r = evergreen_cp_resume(rdev);
1430	serge	5258	if (r)
		5259	return r;
3192	Serge	5260	r = r600_dma_resume(rdev);
		5261	if (r)
		5262	return r;
1963	serge	5263
3764	Serge	5264
3192	Serge	5265	r = radeon_ib_pool_init(rdev);
		5266	if (r) {
		5267	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		5268	return r;
		5269	}
		5270
1430	serge	5271	return 0;
		5272	}
		5273
		5274
		5275
		5276
		5277	/* Plan is to move initialization in that function and use
		5278	* helper function so that radeon_device_init pretty much
		5279	* do nothing more than calling asic specific function. This
		5280	* should also allow to remove a bunch of callback function
		5281	* like vram_info.
		5282	*/
		5283	int evergreen_init(struct radeon_device *rdev)
		5284	{
		5285	int r;
		5286
		5287	/* Read BIOS */
		5288	if (!radeon_get_bios(rdev)) {
		5289	if (ASIC_IS_AVIVO(rdev))
		5290	return -EINVAL;
		5291	}
		5292	/* Must be an ATOMBIOS */
		5293	if (!rdev->is_atom_bios) {
1986	serge	5294	dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
1430	serge	5295	return -EINVAL;
		5296	}
		5297	r = radeon_atombios_init(rdev);
		5298	if (r)
		5299	return r;
1986	serge	5300	/* reset the asic, the gfx blocks are often in a bad state
		5301	* after the driver is unloaded or after a resume
		5302	*/
		5303	if (radeon_asic_reset(rdev))
		5304	dev_warn(rdev->dev, "GPU reset failed !\n");
1430	serge	5305	/* Post card if necessary */
1986	serge	5306	if (!radeon_card_posted(rdev)) {
1430	serge	5307	if (!rdev->bios) {
		5308	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		5309	return -EINVAL;
		5310	}
		5311	DRM_INFO("GPU not posted. posting now...\n");
		5312	atom_asic_init(rdev->mode_info.atom_context);
		5313	}
3764	Serge	5314	/* init golden registers */
		5315	evergreen_init_golden_registers(rdev);
1430	serge	5316	/* Initialize scratch registers */
		5317	r600_scratch_init(rdev);
		5318	/* Initialize surface registers */
		5319	radeon_surface_init(rdev);
		5320	/* Initialize clocks */
		5321	radeon_get_clock_info(rdev->ddev);
		5322	/* Fence driver */
2005	serge	5323	r = radeon_fence_driver_init(rdev);
		5324	if (r)
		5325	return r;
3031	serge	5326	/* initialize AGP */
1430	serge	5327	if (rdev->flags & RADEON_IS_AGP) {
		5328	r = radeon_agp_init(rdev);
		5329	if (r)
		5330	radeon_agp_disable(rdev);
		5331	}
		5332	/* initialize memory controller */
		5333	r = evergreen_mc_init(rdev);
		5334	if (r)
		5335	return r;
		5336	/* Memory manager */
		5337	r = radeon_bo_init(rdev);
		5338	if (r)
		5339	return r;
1963	serge	5340
5078	serge	5341	if (ASIC_IS_DCE5(rdev)) {
		5342	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw \|\| !rdev->mc_fw) {
		5343	r = ni_init_microcode(rdev);
		5344	if (r) {
		5345	DRM_ERROR("Failed to load firmware!\n");
		5346	return r;
		5347	}
		5348	}
		5349	} else {
		5350	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		5351	r = r600_init_microcode(rdev);
		5352	if (r) {
		5353	DRM_ERROR("Failed to load firmware!\n");
		5354	return r;
		5355	}
		5356	}
		5357	}
		5358
		5359	/* Initialize power management */
		5360	radeon_pm_init(rdev);
		5361
2997	Serge	5362	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
		5363	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1430	serge	5364
3192	Serge	5365	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
		5366	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
		5367
3764	Serge	5368	// r = radeon_uvd_init(rdev);
		5369	// if (!r) {
		5370	// rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
		5371	// r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
		5372	// 4096);
		5373	// }
		5374
2005	serge	5375	rdev->ih.ring_obj = NULL;
		5376	r600_ih_ring_init(rdev, 64 * 1024);
1430	serge	5377
		5378	r = r600_pcie_gart_init(rdev);
		5379	if (r)
		5380	return r;
1963	serge	5381
		5382	rdev->accel_working = true;
1430	serge	5383	r = evergreen_startup(rdev);
		5384	if (r) {
1963	serge	5385	dev_err(rdev->dev, "disabling GPU acceleration\n");
1430	serge	5386	rdev->accel_working = false;
		5387	}
2997	Serge	5388
		5389	/* Don't start up if the MC ucode is missing on BTC parts.
		5390	* The default clocks and voltages before the MC ucode
		5391	* is loaded are not suffient for advanced operations.
		5392	*/
		5393	if (ASIC_IS_DCE5(rdev)) {
		5394	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
		5395	DRM_ERROR("radeon: MC ucode required for NI+.\n");
		5396	return -EINVAL;
2005	serge	5397	}
1430	serge	5398	}
2997	Serge	5399
1430	serge	5400	return 0;
		5401	}
		5402
1986	serge	5403
2997	Serge	5404	void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
1430	serge	5405	{
3764	Serge	5406	u32 link_width_cntl, speed_cntl;
1986	serge	5407
		5408	if (radeon_pcie_gen2 == 0)
		5409	return;
		5410
		5411	if (rdev->flags & RADEON_IS_IGP)
		5412	return;
		5413
		5414	if (!(rdev->flags & RADEON_IS_PCIE))
		5415	return;
		5416
		5417	/* x2 cards have a special sequence */
		5418	if (ASIC_IS_X2(rdev))
		5419	return;
		5420
3764	Serge	5421	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
		5422	(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
2997	Serge	5423	return;
		5424
3764	Serge	5425	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2997	Serge	5426	if (speed_cntl & LC_CURRENT_DATA_RATE) {
		5427	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
		5428	return;
		5429	}
		5430
		5431	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
		5432
1986	serge	5433	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) \|\|
		5434	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		5435
3764	Serge	5436	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1986	serge	5437	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3764	Serge	5438	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1986	serge	5439
3764	Serge	5440	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1986	serge	5441	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3764	Serge	5442	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1986	serge	5443
3764	Serge	5444	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1986	serge	5445	speed_cntl \|= LC_CLR_FAILED_SPD_CHANGE_CNT;
3764	Serge	5446	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1986	serge	5447
3764	Serge	5448	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1986	serge	5449	speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3764	Serge	5450	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1986	serge	5451
3764	Serge	5452	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1986	serge	5453	speed_cntl \|= LC_GEN2_EN_STRAP;
3764	Serge	5454	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1986	serge	5455
		5456	} else {
3764	Serge	5457	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1986	serge	5458	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		5459	if (1)
		5460	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		5461	else
		5462	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3764	Serge	5463	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1986	serge	5464	}
1430	serge	5465	}
5078	serge	5466
		5467	void evergreen_program_aspm(struct radeon_device *rdev)
		5468	{
		5469	u32 data, orig;
		5470	u32 pcie_lc_cntl, pcie_lc_cntl_old;
		5471	bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
		5472	/* fusion_platform = true
		5473	* if the system is a fusion system
		5474	* (APU or DGPU in a fusion system).
		5475	* todo: check if the system is a fusion platform.
		5476	*/
		5477	bool fusion_platform = false;
		5478
		5479	if (radeon_aspm == 0)
		5480	return;
		5481
		5482	if (!(rdev->flags & RADEON_IS_PCIE))
		5483	return;
		5484
		5485	switch (rdev->family) {
		5486	case CHIP_CYPRESS:
		5487	case CHIP_HEMLOCK:
		5488	case CHIP_JUNIPER:
		5489	case CHIP_REDWOOD:
		5490	case CHIP_CEDAR:
		5491	case CHIP_SUMO:
		5492	case CHIP_SUMO2:
		5493	case CHIP_PALM:
		5494	case CHIP_ARUBA:
		5495	disable_l0s = true;
		5496	break;
		5497	default:
		5498	disable_l0s = false;
		5499	break;
		5500	}
		5501
		5502	if (rdev->flags & RADEON_IS_IGP)
		5503	fusion_platform = true; /* XXX also dGPUs in a fusion system */
		5504
		5505	data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
		5506	if (fusion_platform)
		5507	data &= ~MULTI_PIF;
		5508	else
		5509	data \|= MULTI_PIF;
		5510	if (data != orig)
		5511	WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
		5512
		5513	data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
		5514	if (fusion_platform)
		5515	data &= ~MULTI_PIF;
		5516	else
		5517	data \|= MULTI_PIF;
		5518	if (data != orig)
		5519	WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
		5520
		5521	pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
		5522	pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK \| LC_L1_INACTIVITY_MASK);
		5523	if (!disable_l0s) {
		5524	if (rdev->family >= CHIP_BARTS)
		5525	pcie_lc_cntl \|= LC_L0S_INACTIVITY(7);
		5526	else
		5527	pcie_lc_cntl \|= LC_L0S_INACTIVITY(3);
		5528	}
		5529
		5530	if (!disable_l1) {
		5531	if (rdev->family >= CHIP_BARTS)
		5532	pcie_lc_cntl \|= LC_L1_INACTIVITY(7);
		5533	else
		5534	pcie_lc_cntl \|= LC_L1_INACTIVITY(8);
		5535
		5536	if (!disable_plloff_in_l1) {
		5537	data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
		5538	data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK \| PLL_POWER_STATE_IN_TXS2_0_MASK);
		5539	data \|= PLL_POWER_STATE_IN_OFF_0(7) \| PLL_POWER_STATE_IN_TXS2_0(7);
		5540	if (data != orig)
		5541	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
		5542
		5543	data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
		5544	data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK \| PLL_POWER_STATE_IN_TXS2_1_MASK);
		5545	data \|= PLL_POWER_STATE_IN_OFF_1(7) \| PLL_POWER_STATE_IN_TXS2_1(7);
		5546	if (data != orig)
		5547	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
		5548
		5549	data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
		5550	data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK \| PLL_POWER_STATE_IN_TXS2_0_MASK);
		5551	data \|= PLL_POWER_STATE_IN_OFF_0(7) \| PLL_POWER_STATE_IN_TXS2_0(7);
		5552	if (data != orig)
		5553	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
		5554
		5555	data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
		5556	data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK \| PLL_POWER_STATE_IN_TXS2_1_MASK);
		5557	data \|= PLL_POWER_STATE_IN_OFF_1(7) \| PLL_POWER_STATE_IN_TXS2_1(7);
		5558	if (data != orig)
		5559	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
		5560
		5561	if (rdev->family >= CHIP_BARTS) {
		5562	data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
		5563	data &= ~PLL_RAMP_UP_TIME_0_MASK;
		5564	data \|= PLL_RAMP_UP_TIME_0(4);
		5565	if (data != orig)
		5566	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
		5567
		5568	data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
		5569	data &= ~PLL_RAMP_UP_TIME_1_MASK;
		5570	data \|= PLL_RAMP_UP_TIME_1(4);
		5571	if (data != orig)
		5572	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
		5573
		5574	data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
		5575	data &= ~PLL_RAMP_UP_TIME_0_MASK;
		5576	data \|= PLL_RAMP_UP_TIME_0(4);
		5577	if (data != orig)
		5578	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
		5579
		5580	data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
		5581	data &= ~PLL_RAMP_UP_TIME_1_MASK;
		5582	data \|= PLL_RAMP_UP_TIME_1(4);
		5583	if (data != orig)
		5584	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
		5585	}
		5586
		5587	data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
		5588	data &= ~LC_DYN_LANES_PWR_STATE_MASK;
		5589	data \|= LC_DYN_LANES_PWR_STATE(3);
		5590	if (data != orig)
		5591	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
		5592
		5593	if (rdev->family >= CHIP_BARTS) {
		5594	data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
		5595	data &= ~LS2_EXIT_TIME_MASK;
		5596	data \|= LS2_EXIT_TIME(1);
		5597	if (data != orig)
		5598	WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
		5599
		5600	data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
		5601	data &= ~LS2_EXIT_TIME_MASK;
		5602	data \|= LS2_EXIT_TIME(1);
		5603	if (data != orig)
		5604	WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
		5605	}
		5606	}
		5607	}
		5608
		5609	/* evergreen parts only */
		5610	if (rdev->family < CHIP_BARTS)
		5611	pcie_lc_cntl \|= LC_PMI_TO_L1_DIS;
		5612
		5613	if (pcie_lc_cntl != pcie_lc_cntl_old)
		5614	WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
		5615	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/evergreen.c – Rev 5271