WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/si.c

Rev	Author	Line No.	Line
2997	Serge	1	/*
		2	* Copyright 2011 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	#include
		26	#include
		27	#include
		28	#include "radeon.h"
		29	#include "radeon_asic.h"
6104	serge	30	#include "radeon_audio.h"
2997	Serge	31	#include
		32	#include "sid.h"
		33	#include "atom.h"
		34	#include "si_blit_shaders.h"
5078	serge	35	#include "clearstate_si.h"
		36	#include "radeon_ucode.h"
2997	Serge	37
		38
		39	MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
		40	MODULE_FIRMWARE("radeon/TAHITI_me.bin");
		41	MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
		42	MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
5078	serge	43	MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
2997	Serge	44	MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
5078	serge	45	MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
		46
		47	MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
		48	MODULE_FIRMWARE("radeon/tahiti_me.bin");
		49	MODULE_FIRMWARE("radeon/tahiti_ce.bin");
		50	MODULE_FIRMWARE("radeon/tahiti_mc.bin");
		51	MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
		52	MODULE_FIRMWARE("radeon/tahiti_smc.bin");
		53
2997	Serge	54	MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
		55	MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
		56	MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
		57	MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
5078	serge	58	MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
2997	Serge	59	MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
5078	serge	60	MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
		61
		62	MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
		63	MODULE_FIRMWARE("radeon/pitcairn_me.bin");
		64	MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
		65	MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
		66	MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
		67	MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
		68
2997	Serge	69	MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
		70	MODULE_FIRMWARE("radeon/VERDE_me.bin");
		71	MODULE_FIRMWARE("radeon/VERDE_ce.bin");
		72	MODULE_FIRMWARE("radeon/VERDE_mc.bin");
5078	serge	73	MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
2997	Serge	74	MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
5078	serge	75	MODULE_FIRMWARE("radeon/VERDE_smc.bin");
		76
		77	MODULE_FIRMWARE("radeon/verde_pfp.bin");
		78	MODULE_FIRMWARE("radeon/verde_me.bin");
		79	MODULE_FIRMWARE("radeon/verde_ce.bin");
		80	MODULE_FIRMWARE("radeon/verde_mc.bin");
		81	MODULE_FIRMWARE("radeon/verde_rlc.bin");
		82	MODULE_FIRMWARE("radeon/verde_smc.bin");
		83
3764	Serge	84	MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
		85	MODULE_FIRMWARE("radeon/OLAND_me.bin");
		86	MODULE_FIRMWARE("radeon/OLAND_ce.bin");
		87	MODULE_FIRMWARE("radeon/OLAND_mc.bin");
5078	serge	88	MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
3764	Serge	89	MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
5078	serge	90	MODULE_FIRMWARE("radeon/OLAND_smc.bin");
		91
		92	MODULE_FIRMWARE("radeon/oland_pfp.bin");
		93	MODULE_FIRMWARE("radeon/oland_me.bin");
		94	MODULE_FIRMWARE("radeon/oland_ce.bin");
		95	MODULE_FIRMWARE("radeon/oland_mc.bin");
		96	MODULE_FIRMWARE("radeon/oland_rlc.bin");
		97	MODULE_FIRMWARE("radeon/oland_smc.bin");
		98
3764	Serge	99	MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
		100	MODULE_FIRMWARE("radeon/HAINAN_me.bin");
		101	MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
		102	MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
5078	serge	103	MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
3764	Serge	104	MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
5078	serge	105	MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
2997	Serge	106
5078	serge	107	MODULE_FIRMWARE("radeon/hainan_pfp.bin");
		108	MODULE_FIRMWARE("radeon/hainan_me.bin");
		109	MODULE_FIRMWARE("radeon/hainan_ce.bin");
		110	MODULE_FIRMWARE("radeon/hainan_mc.bin");
		111	MODULE_FIRMWARE("radeon/hainan_rlc.bin");
		112	MODULE_FIRMWARE("radeon/hainan_smc.bin");
		113
		114	static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
		115	static void si_pcie_gen3_enable(struct radeon_device *rdev);
		116	static void si_program_aspm(struct radeon_device *rdev);
		117	extern void sumo_rlc_fini(struct radeon_device *rdev);
		118	extern int sumo_rlc_init(struct radeon_device *rdev);
2997	Serge	119	extern int r600_ih_ring_alloc(struct radeon_device *rdev);
		120	extern void r600_ih_ring_fini(struct radeon_device *rdev);
		121	extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
		122	extern void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save);
		123	extern void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save);
		124	extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
3764	Serge	125	extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
		126	extern bool evergreen_is_display_hung(struct radeon_device *rdev);
5078	serge	127	static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
		128	bool enable);
		129	static void si_init_pg(struct radeon_device *rdev);
		130	static void si_init_cg(struct radeon_device *rdev);
		131	static void si_fini_pg(struct radeon_device *rdev);
		132	static void si_fini_cg(struct radeon_device *rdev);
		133	static void si_rlc_stop(struct radeon_device *rdev);
2997	Serge	134
5078	serge	135	static const u32 verde_rlc_save_restore_register_list[] =
		136	{
		137	(0x8000 << 16) \| (0x98f4 >> 2),
		138	0x00000000,
		139	(0x8040 << 16) \| (0x98f4 >> 2),
		140	0x00000000,
		141	(0x8000 << 16) \| (0xe80 >> 2),
		142	0x00000000,
		143	(0x8040 << 16) \| (0xe80 >> 2),
		144	0x00000000,
		145	(0x8000 << 16) \| (0x89bc >> 2),
		146	0x00000000,
		147	(0x8040 << 16) \| (0x89bc >> 2),
		148	0x00000000,
		149	(0x8000 << 16) \| (0x8c1c >> 2),
		150	0x00000000,
		151	(0x8040 << 16) \| (0x8c1c >> 2),
		152	0x00000000,
		153	(0x9c00 << 16) \| (0x98f0 >> 2),
		154	0x00000000,
		155	(0x9c00 << 16) \| (0xe7c >> 2),
		156	0x00000000,
		157	(0x8000 << 16) \| (0x9148 >> 2),
		158	0x00000000,
		159	(0x8040 << 16) \| (0x9148 >> 2),
		160	0x00000000,
		161	(0x9c00 << 16) \| (0x9150 >> 2),
		162	0x00000000,
		163	(0x9c00 << 16) \| (0x897c >> 2),
		164	0x00000000,
		165	(0x9c00 << 16) \| (0x8d8c >> 2),
		166	0x00000000,
		167	(0x9c00 << 16) \| (0xac54 >> 2),
		168	0X00000000,
		169	0x3,
		170	(0x9c00 << 16) \| (0x98f8 >> 2),
		171	0x00000000,
		172	(0x9c00 << 16) \| (0x9910 >> 2),
		173	0x00000000,
		174	(0x9c00 << 16) \| (0x9914 >> 2),
		175	0x00000000,
		176	(0x9c00 << 16) \| (0x9918 >> 2),
		177	0x00000000,
		178	(0x9c00 << 16) \| (0x991c >> 2),
		179	0x00000000,
		180	(0x9c00 << 16) \| (0x9920 >> 2),
		181	0x00000000,
		182	(0x9c00 << 16) \| (0x9924 >> 2),
		183	0x00000000,
		184	(0x9c00 << 16) \| (0x9928 >> 2),
		185	0x00000000,
		186	(0x9c00 << 16) \| (0x992c >> 2),
		187	0x00000000,
		188	(0x9c00 << 16) \| (0x9930 >> 2),
		189	0x00000000,
		190	(0x9c00 << 16) \| (0x9934 >> 2),
		191	0x00000000,
		192	(0x9c00 << 16) \| (0x9938 >> 2),
		193	0x00000000,
		194	(0x9c00 << 16) \| (0x993c >> 2),
		195	0x00000000,
		196	(0x9c00 << 16) \| (0x9940 >> 2),
		197	0x00000000,
		198	(0x9c00 << 16) \| (0x9944 >> 2),
		199	0x00000000,
		200	(0x9c00 << 16) \| (0x9948 >> 2),
		201	0x00000000,
		202	(0x9c00 << 16) \| (0x994c >> 2),
		203	0x00000000,
		204	(0x9c00 << 16) \| (0x9950 >> 2),
		205	0x00000000,
		206	(0x9c00 << 16) \| (0x9954 >> 2),
		207	0x00000000,
		208	(0x9c00 << 16) \| (0x9958 >> 2),
		209	0x00000000,
		210	(0x9c00 << 16) \| (0x995c >> 2),
		211	0x00000000,
		212	(0x9c00 << 16) \| (0x9960 >> 2),
		213	0x00000000,
		214	(0x9c00 << 16) \| (0x9964 >> 2),
		215	0x00000000,
		216	(0x9c00 << 16) \| (0x9968 >> 2),
		217	0x00000000,
		218	(0x9c00 << 16) \| (0x996c >> 2),
		219	0x00000000,
		220	(0x9c00 << 16) \| (0x9970 >> 2),
		221	0x00000000,
		222	(0x9c00 << 16) \| (0x9974 >> 2),
		223	0x00000000,
		224	(0x9c00 << 16) \| (0x9978 >> 2),
		225	0x00000000,
		226	(0x9c00 << 16) \| (0x997c >> 2),
		227	0x00000000,
		228	(0x9c00 << 16) \| (0x9980 >> 2),
		229	0x00000000,
		230	(0x9c00 << 16) \| (0x9984 >> 2),
		231	0x00000000,
		232	(0x9c00 << 16) \| (0x9988 >> 2),
		233	0x00000000,
		234	(0x9c00 << 16) \| (0x998c >> 2),
		235	0x00000000,
		236	(0x9c00 << 16) \| (0x8c00 >> 2),
		237	0x00000000,
		238	(0x9c00 << 16) \| (0x8c14 >> 2),
		239	0x00000000,
		240	(0x9c00 << 16) \| (0x8c04 >> 2),
		241	0x00000000,
		242	(0x9c00 << 16) \| (0x8c08 >> 2),
		243	0x00000000,
		244	(0x8000 << 16) \| (0x9b7c >> 2),
		245	0x00000000,
		246	(0x8040 << 16) \| (0x9b7c >> 2),
		247	0x00000000,
		248	(0x8000 << 16) \| (0xe84 >> 2),
		249	0x00000000,
		250	(0x8040 << 16) \| (0xe84 >> 2),
		251	0x00000000,
		252	(0x8000 << 16) \| (0x89c0 >> 2),
		253	0x00000000,
		254	(0x8040 << 16) \| (0x89c0 >> 2),
		255	0x00000000,
		256	(0x8000 << 16) \| (0x914c >> 2),
		257	0x00000000,
		258	(0x8040 << 16) \| (0x914c >> 2),
		259	0x00000000,
		260	(0x8000 << 16) \| (0x8c20 >> 2),
		261	0x00000000,
		262	(0x8040 << 16) \| (0x8c20 >> 2),
		263	0x00000000,
		264	(0x8000 << 16) \| (0x9354 >> 2),
		265	0x00000000,
		266	(0x8040 << 16) \| (0x9354 >> 2),
		267	0x00000000,
		268	(0x9c00 << 16) \| (0x9060 >> 2),
		269	0x00000000,
		270	(0x9c00 << 16) \| (0x9364 >> 2),
		271	0x00000000,
		272	(0x9c00 << 16) \| (0x9100 >> 2),
		273	0x00000000,
		274	(0x9c00 << 16) \| (0x913c >> 2),
		275	0x00000000,
		276	(0x8000 << 16) \| (0x90e0 >> 2),
		277	0x00000000,
		278	(0x8000 << 16) \| (0x90e4 >> 2),
		279	0x00000000,
		280	(0x8000 << 16) \| (0x90e8 >> 2),
		281	0x00000000,
		282	(0x8040 << 16) \| (0x90e0 >> 2),
		283	0x00000000,
		284	(0x8040 << 16) \| (0x90e4 >> 2),
		285	0x00000000,
		286	(0x8040 << 16) \| (0x90e8 >> 2),
		287	0x00000000,
		288	(0x9c00 << 16) \| (0x8bcc >> 2),
		289	0x00000000,
		290	(0x9c00 << 16) \| (0x8b24 >> 2),
		291	0x00000000,
		292	(0x9c00 << 16) \| (0x88c4 >> 2),
		293	0x00000000,
		294	(0x9c00 << 16) \| (0x8e50 >> 2),
		295	0x00000000,
		296	(0x9c00 << 16) \| (0x8c0c >> 2),
		297	0x00000000,
		298	(0x9c00 << 16) \| (0x8e58 >> 2),
		299	0x00000000,
		300	(0x9c00 << 16) \| (0x8e5c >> 2),
		301	0x00000000,
		302	(0x9c00 << 16) \| (0x9508 >> 2),
		303	0x00000000,
		304	(0x9c00 << 16) \| (0x950c >> 2),
		305	0x00000000,
		306	(0x9c00 << 16) \| (0x9494 >> 2),
		307	0x00000000,
		308	(0x9c00 << 16) \| (0xac0c >> 2),
		309	0x00000000,
		310	(0x9c00 << 16) \| (0xac10 >> 2),
		311	0x00000000,
		312	(0x9c00 << 16) \| (0xac14 >> 2),
		313	0x00000000,
		314	(0x9c00 << 16) \| (0xae00 >> 2),
		315	0x00000000,
		316	(0x9c00 << 16) \| (0xac08 >> 2),
		317	0x00000000,
		318	(0x9c00 << 16) \| (0x88d4 >> 2),
		319	0x00000000,
		320	(0x9c00 << 16) \| (0x88c8 >> 2),
		321	0x00000000,
		322	(0x9c00 << 16) \| (0x88cc >> 2),
		323	0x00000000,
		324	(0x9c00 << 16) \| (0x89b0 >> 2),
		325	0x00000000,
		326	(0x9c00 << 16) \| (0x8b10 >> 2),
		327	0x00000000,
		328	(0x9c00 << 16) \| (0x8a14 >> 2),
		329	0x00000000,
		330	(0x9c00 << 16) \| (0x9830 >> 2),
		331	0x00000000,
		332	(0x9c00 << 16) \| (0x9834 >> 2),
		333	0x00000000,
		334	(0x9c00 << 16) \| (0x9838 >> 2),
		335	0x00000000,
		336	(0x9c00 << 16) \| (0x9a10 >> 2),
		337	0x00000000,
		338	(0x8000 << 16) \| (0x9870 >> 2),
		339	0x00000000,
		340	(0x8000 << 16) \| (0x9874 >> 2),
		341	0x00000000,
		342	(0x8001 << 16) \| (0x9870 >> 2),
		343	0x00000000,
		344	(0x8001 << 16) \| (0x9874 >> 2),
		345	0x00000000,
		346	(0x8040 << 16) \| (0x9870 >> 2),
		347	0x00000000,
		348	(0x8040 << 16) \| (0x9874 >> 2),
		349	0x00000000,
		350	(0x8041 << 16) \| (0x9870 >> 2),
		351	0x00000000,
		352	(0x8041 << 16) \| (0x9874 >> 2),
		353	0x00000000,
		354	0x00000000
		355	};
		356
3764	Serge	357	static const u32 tahiti_golden_rlc_registers[] =
		358	{
		359	0xc424, 0xffffffff, 0x00601005,
		360	0xc47c, 0xffffffff, 0x10104040,
		361	0xc488, 0xffffffff, 0x0100000a,
		362	0xc314, 0xffffffff, 0x00000800,
		363	0xc30c, 0xffffffff, 0x800000f4,
		364	0xf4a8, 0xffffffff, 0x00000000
		365	};
		366
		367	static const u32 tahiti_golden_registers[] =
		368	{
		369	0x9a10, 0x00010000, 0x00018208,
		370	0x9830, 0xffffffff, 0x00000000,
		371	0x9834, 0xf00fffff, 0x00000400,
		372	0x9838, 0x0002021c, 0x00020200,
		373	0xc78, 0x00000080, 0x00000000,
		374	0xd030, 0x000300c0, 0x00800040,
		375	0xd830, 0x000300c0, 0x00800040,
		376	0x5bb0, 0x000000f0, 0x00000070,
		377	0x5bc0, 0x00200000, 0x50100000,
		378	0x7030, 0x31000311, 0x00000011,
		379	0x277c, 0x00000003, 0x000007ff,
		380	0x240c, 0x000007ff, 0x00000000,
		381	0x8a14, 0xf000001f, 0x00000007,
		382	0x8b24, 0xffffffff, 0x00ffffff,
		383	0x8b10, 0x0000ff0f, 0x00000000,
		384	0x28a4c, 0x07ffffff, 0x4e000000,
		385	0x28350, 0x3f3f3fff, 0x2a00126a,
		386	0x30, 0x000000ff, 0x0040,
		387	0x34, 0x00000040, 0x00004040,
		388	0x9100, 0x07ffffff, 0x03000000,
		389	0x8e88, 0x01ff1f3f, 0x00000000,
		390	0x8e84, 0x01ff1f3f, 0x00000000,
		391	0x9060, 0x0000007f, 0x00000020,
		392	0x9508, 0x00010000, 0x00010000,
		393	0xac14, 0x00000200, 0x000002fb,
		394	0xac10, 0xffffffff, 0x0000543b,
		395	0xac0c, 0xffffffff, 0xa9210876,
		396	0x88d0, 0xffffffff, 0x000fff40,
		397	0x88d4, 0x0000001f, 0x00000010,
		398	0x1410, 0x20000000, 0x20fffed8,
		399	0x15c0, 0x000c0fc0, 0x000c0400
		400	};
		401
		402	static const u32 tahiti_golden_registers2[] =
		403	{
		404	0xc64, 0x00000001, 0x00000001
		405	};
		406
		407	static const u32 pitcairn_golden_rlc_registers[] =
		408	{
		409	0xc424, 0xffffffff, 0x00601004,
		410	0xc47c, 0xffffffff, 0x10102020,
		411	0xc488, 0xffffffff, 0x01000020,
		412	0xc314, 0xffffffff, 0x00000800,
		413	0xc30c, 0xffffffff, 0x800000a4
		414	};
		415
		416	static const u32 pitcairn_golden_registers[] =
		417	{
		418	0x9a10, 0x00010000, 0x00018208,
		419	0x9830, 0xffffffff, 0x00000000,
		420	0x9834, 0xf00fffff, 0x00000400,
		421	0x9838, 0x0002021c, 0x00020200,
		422	0xc78, 0x00000080, 0x00000000,
		423	0xd030, 0x000300c0, 0x00800040,
		424	0xd830, 0x000300c0, 0x00800040,
		425	0x5bb0, 0x000000f0, 0x00000070,
		426	0x5bc0, 0x00200000, 0x50100000,
		427	0x7030, 0x31000311, 0x00000011,
		428	0x2ae4, 0x00073ffe, 0x000022a2,
		429	0x240c, 0x000007ff, 0x00000000,
		430	0x8a14, 0xf000001f, 0x00000007,
		431	0x8b24, 0xffffffff, 0x00ffffff,
		432	0x8b10, 0x0000ff0f, 0x00000000,
		433	0x28a4c, 0x07ffffff, 0x4e000000,
		434	0x28350, 0x3f3f3fff, 0x2a00126a,
		435	0x30, 0x000000ff, 0x0040,
		436	0x34, 0x00000040, 0x00004040,
		437	0x9100, 0x07ffffff, 0x03000000,
		438	0x9060, 0x0000007f, 0x00000020,
		439	0x9508, 0x00010000, 0x00010000,
		440	0xac14, 0x000003ff, 0x000000f7,
		441	0xac10, 0xffffffff, 0x00000000,
		442	0xac0c, 0xffffffff, 0x32761054,
		443	0x88d4, 0x0000001f, 0x00000010,
		444	0x15c0, 0x000c0fc0, 0x000c0400
		445	};
		446
		447	static const u32 verde_golden_rlc_registers[] =
		448	{
		449	0xc424, 0xffffffff, 0x033f1005,
		450	0xc47c, 0xffffffff, 0x10808020,
		451	0xc488, 0xffffffff, 0x00800008,
		452	0xc314, 0xffffffff, 0x00001000,
		453	0xc30c, 0xffffffff, 0x80010014
		454	};
		455
		456	static const u32 verde_golden_registers[] =
		457	{
		458	0x9a10, 0x00010000, 0x00018208,
		459	0x9830, 0xffffffff, 0x00000000,
		460	0x9834, 0xf00fffff, 0x00000400,
		461	0x9838, 0x0002021c, 0x00020200,
		462	0xc78, 0x00000080, 0x00000000,
		463	0xd030, 0x000300c0, 0x00800040,
		464	0xd030, 0x000300c0, 0x00800040,
		465	0xd830, 0x000300c0, 0x00800040,
		466	0xd830, 0x000300c0, 0x00800040,
		467	0x5bb0, 0x000000f0, 0x00000070,
		468	0x5bc0, 0x00200000, 0x50100000,
		469	0x7030, 0x31000311, 0x00000011,
		470	0x2ae4, 0x00073ffe, 0x000022a2,
		471	0x2ae4, 0x00073ffe, 0x000022a2,
		472	0x2ae4, 0x00073ffe, 0x000022a2,
		473	0x240c, 0x000007ff, 0x00000000,
		474	0x240c, 0x000007ff, 0x00000000,
		475	0x240c, 0x000007ff, 0x00000000,
		476	0x8a14, 0xf000001f, 0x00000007,
		477	0x8a14, 0xf000001f, 0x00000007,
		478	0x8a14, 0xf000001f, 0x00000007,
		479	0x8b24, 0xffffffff, 0x00ffffff,
		480	0x8b10, 0x0000ff0f, 0x00000000,
		481	0x28a4c, 0x07ffffff, 0x4e000000,
		482	0x28350, 0x3f3f3fff, 0x0000124a,
		483	0x28350, 0x3f3f3fff, 0x0000124a,
		484	0x28350, 0x3f3f3fff, 0x0000124a,
		485	0x30, 0x000000ff, 0x0040,
		486	0x34, 0x00000040, 0x00004040,
		487	0x9100, 0x07ffffff, 0x03000000,
		488	0x9100, 0x07ffffff, 0x03000000,
		489	0x8e88, 0x01ff1f3f, 0x00000000,
		490	0x8e88, 0x01ff1f3f, 0x00000000,
		491	0x8e88, 0x01ff1f3f, 0x00000000,
		492	0x8e84, 0x01ff1f3f, 0x00000000,
		493	0x8e84, 0x01ff1f3f, 0x00000000,
		494	0x8e84, 0x01ff1f3f, 0x00000000,
		495	0x9060, 0x0000007f, 0x00000020,
		496	0x9508, 0x00010000, 0x00010000,
		497	0xac14, 0x000003ff, 0x00000003,
		498	0xac14, 0x000003ff, 0x00000003,
		499	0xac14, 0x000003ff, 0x00000003,
		500	0xac10, 0xffffffff, 0x00000000,
		501	0xac10, 0xffffffff, 0x00000000,
		502	0xac10, 0xffffffff, 0x00000000,
		503	0xac0c, 0xffffffff, 0x00001032,
		504	0xac0c, 0xffffffff, 0x00001032,
		505	0xac0c, 0xffffffff, 0x00001032,
		506	0x88d4, 0x0000001f, 0x00000010,
		507	0x88d4, 0x0000001f, 0x00000010,
		508	0x88d4, 0x0000001f, 0x00000010,
		509	0x15c0, 0x000c0fc0, 0x000c0400
		510	};
		511
		512	static const u32 oland_golden_rlc_registers[] =
		513	{
		514	0xc424, 0xffffffff, 0x00601005,
		515	0xc47c, 0xffffffff, 0x10104040,
		516	0xc488, 0xffffffff, 0x0100000a,
		517	0xc314, 0xffffffff, 0x00000800,
		518	0xc30c, 0xffffffff, 0x800000f4
		519	};
		520
		521	static const u32 oland_golden_registers[] =
		522	{
		523	0x9a10, 0x00010000, 0x00018208,
		524	0x9830, 0xffffffff, 0x00000000,
		525	0x9834, 0xf00fffff, 0x00000400,
		526	0x9838, 0x0002021c, 0x00020200,
		527	0xc78, 0x00000080, 0x00000000,
		528	0xd030, 0x000300c0, 0x00800040,
		529	0xd830, 0x000300c0, 0x00800040,
		530	0x5bb0, 0x000000f0, 0x00000070,
		531	0x5bc0, 0x00200000, 0x50100000,
		532	0x7030, 0x31000311, 0x00000011,
		533	0x2ae4, 0x00073ffe, 0x000022a2,
		534	0x240c, 0x000007ff, 0x00000000,
		535	0x8a14, 0xf000001f, 0x00000007,
		536	0x8b24, 0xffffffff, 0x00ffffff,
		537	0x8b10, 0x0000ff0f, 0x00000000,
		538	0x28a4c, 0x07ffffff, 0x4e000000,
		539	0x28350, 0x3f3f3fff, 0x00000082,
		540	0x30, 0x000000ff, 0x0040,
		541	0x34, 0x00000040, 0x00004040,
		542	0x9100, 0x07ffffff, 0x03000000,
		543	0x9060, 0x0000007f, 0x00000020,
		544	0x9508, 0x00010000, 0x00010000,
		545	0xac14, 0x000003ff, 0x000000f3,
		546	0xac10, 0xffffffff, 0x00000000,
		547	0xac0c, 0xffffffff, 0x00003210,
		548	0x88d4, 0x0000001f, 0x00000010,
		549	0x15c0, 0x000c0fc0, 0x000c0400
		550	};
		551
		552	static const u32 hainan_golden_registers[] =
		553	{
		554	0x9a10, 0x00010000, 0x00018208,
		555	0x9830, 0xffffffff, 0x00000000,
		556	0x9834, 0xf00fffff, 0x00000400,
		557	0x9838, 0x0002021c, 0x00020200,
		558	0xd0c0, 0xff000fff, 0x00000100,
		559	0xd030, 0x000300c0, 0x00800040,
		560	0xd8c0, 0xff000fff, 0x00000100,
		561	0xd830, 0x000300c0, 0x00800040,
		562	0x2ae4, 0x00073ffe, 0x000022a2,
		563	0x240c, 0x000007ff, 0x00000000,
		564	0x8a14, 0xf000001f, 0x00000007,
		565	0x8b24, 0xffffffff, 0x00ffffff,
		566	0x8b10, 0x0000ff0f, 0x00000000,
		567	0x28a4c, 0x07ffffff, 0x4e000000,
		568	0x28350, 0x3f3f3fff, 0x00000000,
		569	0x30, 0x000000ff, 0x0040,
		570	0x34, 0x00000040, 0x00004040,
		571	0x9100, 0x03e00000, 0x03600000,
		572	0x9060, 0x0000007f, 0x00000020,
		573	0x9508, 0x00010000, 0x00010000,
		574	0xac14, 0x000003ff, 0x000000f1,
		575	0xac10, 0xffffffff, 0x00000000,
		576	0xac0c, 0xffffffff, 0x00003210,
		577	0x88d4, 0x0000001f, 0x00000010,
		578	0x15c0, 0x000c0fc0, 0x000c0400
		579	};
		580
		581	static const u32 hainan_golden_registers2[] =
		582	{
		583	0x98f8, 0xffffffff, 0x02010001
		584	};
		585
		586	static const u32 tahiti_mgcg_cgcg_init[] =
		587	{
		588	0xc400, 0xffffffff, 0xfffffffc,
		589	0x802c, 0xffffffff, 0xe0000000,
		590	0x9a60, 0xffffffff, 0x00000100,
		591	0x92a4, 0xffffffff, 0x00000100,
		592	0xc164, 0xffffffff, 0x00000100,
		593	0x9774, 0xffffffff, 0x00000100,
		594	0x8984, 0xffffffff, 0x06000100,
		595	0x8a18, 0xffffffff, 0x00000100,
		596	0x92a0, 0xffffffff, 0x00000100,
		597	0xc380, 0xffffffff, 0x00000100,
		598	0x8b28, 0xffffffff, 0x00000100,
		599	0x9144, 0xffffffff, 0x00000100,
		600	0x8d88, 0xffffffff, 0x00000100,
		601	0x8d8c, 0xffffffff, 0x00000100,
		602	0x9030, 0xffffffff, 0x00000100,
		603	0x9034, 0xffffffff, 0x00000100,
		604	0x9038, 0xffffffff, 0x00000100,
		605	0x903c, 0xffffffff, 0x00000100,
		606	0xad80, 0xffffffff, 0x00000100,
		607	0xac54, 0xffffffff, 0x00000100,
		608	0x897c, 0xffffffff, 0x06000100,
		609	0x9868, 0xffffffff, 0x00000100,
		610	0x9510, 0xffffffff, 0x00000100,
		611	0xaf04, 0xffffffff, 0x00000100,
		612	0xae04, 0xffffffff, 0x00000100,
		613	0x949c, 0xffffffff, 0x00000100,
		614	0x802c, 0xffffffff, 0xe0000000,
		615	0x9160, 0xffffffff, 0x00010000,
		616	0x9164, 0xffffffff, 0x00030002,
		617	0x9168, 0xffffffff, 0x00040007,
		618	0x916c, 0xffffffff, 0x00060005,
		619	0x9170, 0xffffffff, 0x00090008,
		620	0x9174, 0xffffffff, 0x00020001,
		621	0x9178, 0xffffffff, 0x00040003,
		622	0x917c, 0xffffffff, 0x00000007,
		623	0x9180, 0xffffffff, 0x00060005,
		624	0x9184, 0xffffffff, 0x00090008,
		625	0x9188, 0xffffffff, 0x00030002,
		626	0x918c, 0xffffffff, 0x00050004,
		627	0x9190, 0xffffffff, 0x00000008,
		628	0x9194, 0xffffffff, 0x00070006,
		629	0x9198, 0xffffffff, 0x000a0009,
		630	0x919c, 0xffffffff, 0x00040003,
		631	0x91a0, 0xffffffff, 0x00060005,
		632	0x91a4, 0xffffffff, 0x00000009,
		633	0x91a8, 0xffffffff, 0x00080007,
		634	0x91ac, 0xffffffff, 0x000b000a,
		635	0x91b0, 0xffffffff, 0x00050004,
		636	0x91b4, 0xffffffff, 0x00070006,
		637	0x91b8, 0xffffffff, 0x0008000b,
		638	0x91bc, 0xffffffff, 0x000a0009,
		639	0x91c0, 0xffffffff, 0x000d000c,
		640	0x91c4, 0xffffffff, 0x00060005,
		641	0x91c8, 0xffffffff, 0x00080007,
		642	0x91cc, 0xffffffff, 0x0000000b,
		643	0x91d0, 0xffffffff, 0x000a0009,
		644	0x91d4, 0xffffffff, 0x000d000c,
		645	0x91d8, 0xffffffff, 0x00070006,
		646	0x91dc, 0xffffffff, 0x00090008,
		647	0x91e0, 0xffffffff, 0x0000000c,
		648	0x91e4, 0xffffffff, 0x000b000a,
		649	0x91e8, 0xffffffff, 0x000e000d,
		650	0x91ec, 0xffffffff, 0x00080007,
		651	0x91f0, 0xffffffff, 0x000a0009,
		652	0x91f4, 0xffffffff, 0x0000000d,
		653	0x91f8, 0xffffffff, 0x000c000b,
		654	0x91fc, 0xffffffff, 0x000f000e,
		655	0x9200, 0xffffffff, 0x00090008,
		656	0x9204, 0xffffffff, 0x000b000a,
		657	0x9208, 0xffffffff, 0x000c000f,
		658	0x920c, 0xffffffff, 0x000e000d,
		659	0x9210, 0xffffffff, 0x00110010,
		660	0x9214, 0xffffffff, 0x000a0009,
		661	0x9218, 0xffffffff, 0x000c000b,
		662	0x921c, 0xffffffff, 0x0000000f,
		663	0x9220, 0xffffffff, 0x000e000d,
		664	0x9224, 0xffffffff, 0x00110010,
		665	0x9228, 0xffffffff, 0x000b000a,
		666	0x922c, 0xffffffff, 0x000d000c,
		667	0x9230, 0xffffffff, 0x00000010,
		668	0x9234, 0xffffffff, 0x000f000e,
		669	0x9238, 0xffffffff, 0x00120011,
		670	0x923c, 0xffffffff, 0x000c000b,
		671	0x9240, 0xffffffff, 0x000e000d,
		672	0x9244, 0xffffffff, 0x00000011,
		673	0x9248, 0xffffffff, 0x0010000f,
		674	0x924c, 0xffffffff, 0x00130012,
		675	0x9250, 0xffffffff, 0x000d000c,
		676	0x9254, 0xffffffff, 0x000f000e,
		677	0x9258, 0xffffffff, 0x00100013,
		678	0x925c, 0xffffffff, 0x00120011,
		679	0x9260, 0xffffffff, 0x00150014,
		680	0x9264, 0xffffffff, 0x000e000d,
		681	0x9268, 0xffffffff, 0x0010000f,
		682	0x926c, 0xffffffff, 0x00000013,
		683	0x9270, 0xffffffff, 0x00120011,
		684	0x9274, 0xffffffff, 0x00150014,
		685	0x9278, 0xffffffff, 0x000f000e,
		686	0x927c, 0xffffffff, 0x00110010,
		687	0x9280, 0xffffffff, 0x00000014,
		688	0x9284, 0xffffffff, 0x00130012,
		689	0x9288, 0xffffffff, 0x00160015,
		690	0x928c, 0xffffffff, 0x0010000f,
		691	0x9290, 0xffffffff, 0x00120011,
		692	0x9294, 0xffffffff, 0x00000015,
		693	0x9298, 0xffffffff, 0x00140013,
		694	0x929c, 0xffffffff, 0x00170016,
		695	0x9150, 0xffffffff, 0x96940200,
		696	0x8708, 0xffffffff, 0x00900100,
		697	0xc478, 0xffffffff, 0x00000080,
		698	0xc404, 0xffffffff, 0x0020003f,
		699	0x30, 0xffffffff, 0x0000001c,
		700	0x34, 0x000f0000, 0x000f0000,
		701	0x160c, 0xffffffff, 0x00000100,
		702	0x1024, 0xffffffff, 0x00000100,
		703	0x102c, 0x00000101, 0x00000000,
		704	0x20a8, 0xffffffff, 0x00000104,
		705	0x264c, 0x000c0000, 0x000c0000,
		706	0x2648, 0x000c0000, 0x000c0000,
		707	0x55e4, 0xff000fff, 0x00000100,
		708	0x55e8, 0x00000001, 0x00000001,
		709	0x2f50, 0x00000001, 0x00000001,
		710	0x30cc, 0xc0000fff, 0x00000104,
		711	0xc1e4, 0x00000001, 0x00000001,
		712	0xd0c0, 0xfffffff0, 0x00000100,
		713	0xd8c0, 0xfffffff0, 0x00000100
		714	};
		715
		716	static const u32 pitcairn_mgcg_cgcg_init[] =
		717	{
		718	0xc400, 0xffffffff, 0xfffffffc,
		719	0x802c, 0xffffffff, 0xe0000000,
		720	0x9a60, 0xffffffff, 0x00000100,
		721	0x92a4, 0xffffffff, 0x00000100,
		722	0xc164, 0xffffffff, 0x00000100,
		723	0x9774, 0xffffffff, 0x00000100,
		724	0x8984, 0xffffffff, 0x06000100,
		725	0x8a18, 0xffffffff, 0x00000100,
		726	0x92a0, 0xffffffff, 0x00000100,
		727	0xc380, 0xffffffff, 0x00000100,
		728	0x8b28, 0xffffffff, 0x00000100,
		729	0x9144, 0xffffffff, 0x00000100,
		730	0x8d88, 0xffffffff, 0x00000100,
		731	0x8d8c, 0xffffffff, 0x00000100,
		732	0x9030, 0xffffffff, 0x00000100,
		733	0x9034, 0xffffffff, 0x00000100,
		734	0x9038, 0xffffffff, 0x00000100,
		735	0x903c, 0xffffffff, 0x00000100,
		736	0xad80, 0xffffffff, 0x00000100,
		737	0xac54, 0xffffffff, 0x00000100,
		738	0x897c, 0xffffffff, 0x06000100,
		739	0x9868, 0xffffffff, 0x00000100,
		740	0x9510, 0xffffffff, 0x00000100,
		741	0xaf04, 0xffffffff, 0x00000100,
		742	0xae04, 0xffffffff, 0x00000100,
		743	0x949c, 0xffffffff, 0x00000100,
		744	0x802c, 0xffffffff, 0xe0000000,
		745	0x9160, 0xffffffff, 0x00010000,
		746	0x9164, 0xffffffff, 0x00030002,
		747	0x9168, 0xffffffff, 0x00040007,
		748	0x916c, 0xffffffff, 0x00060005,
		749	0x9170, 0xffffffff, 0x00090008,
		750	0x9174, 0xffffffff, 0x00020001,
		751	0x9178, 0xffffffff, 0x00040003,
		752	0x917c, 0xffffffff, 0x00000007,
		753	0x9180, 0xffffffff, 0x00060005,
		754	0x9184, 0xffffffff, 0x00090008,
		755	0x9188, 0xffffffff, 0x00030002,
		756	0x918c, 0xffffffff, 0x00050004,
		757	0x9190, 0xffffffff, 0x00000008,
		758	0x9194, 0xffffffff, 0x00070006,
		759	0x9198, 0xffffffff, 0x000a0009,
		760	0x919c, 0xffffffff, 0x00040003,
		761	0x91a0, 0xffffffff, 0x00060005,
		762	0x91a4, 0xffffffff, 0x00000009,
		763	0x91a8, 0xffffffff, 0x00080007,
		764	0x91ac, 0xffffffff, 0x000b000a,
		765	0x91b0, 0xffffffff, 0x00050004,
		766	0x91b4, 0xffffffff, 0x00070006,
		767	0x91b8, 0xffffffff, 0x0008000b,
		768	0x91bc, 0xffffffff, 0x000a0009,
		769	0x91c0, 0xffffffff, 0x000d000c,
		770	0x9200, 0xffffffff, 0x00090008,
		771	0x9204, 0xffffffff, 0x000b000a,
		772	0x9208, 0xffffffff, 0x000c000f,
		773	0x920c, 0xffffffff, 0x000e000d,
		774	0x9210, 0xffffffff, 0x00110010,
		775	0x9214, 0xffffffff, 0x000a0009,
		776	0x9218, 0xffffffff, 0x000c000b,
		777	0x921c, 0xffffffff, 0x0000000f,
		778	0x9220, 0xffffffff, 0x000e000d,
		779	0x9224, 0xffffffff, 0x00110010,
		780	0x9228, 0xffffffff, 0x000b000a,
		781	0x922c, 0xffffffff, 0x000d000c,
		782	0x9230, 0xffffffff, 0x00000010,
		783	0x9234, 0xffffffff, 0x000f000e,
		784	0x9238, 0xffffffff, 0x00120011,
		785	0x923c, 0xffffffff, 0x000c000b,
		786	0x9240, 0xffffffff, 0x000e000d,
		787	0x9244, 0xffffffff, 0x00000011,
		788	0x9248, 0xffffffff, 0x0010000f,
		789	0x924c, 0xffffffff, 0x00130012,
		790	0x9250, 0xffffffff, 0x000d000c,
		791	0x9254, 0xffffffff, 0x000f000e,
		792	0x9258, 0xffffffff, 0x00100013,
		793	0x925c, 0xffffffff, 0x00120011,
		794	0x9260, 0xffffffff, 0x00150014,
		795	0x9150, 0xffffffff, 0x96940200,
		796	0x8708, 0xffffffff, 0x00900100,
		797	0xc478, 0xffffffff, 0x00000080,
		798	0xc404, 0xffffffff, 0x0020003f,
		799	0x30, 0xffffffff, 0x0000001c,
		800	0x34, 0x000f0000, 0x000f0000,
		801	0x160c, 0xffffffff, 0x00000100,
		802	0x1024, 0xffffffff, 0x00000100,
		803	0x102c, 0x00000101, 0x00000000,
		804	0x20a8, 0xffffffff, 0x00000104,
		805	0x55e4, 0xff000fff, 0x00000100,
		806	0x55e8, 0x00000001, 0x00000001,
		807	0x2f50, 0x00000001, 0x00000001,
		808	0x30cc, 0xc0000fff, 0x00000104,
		809	0xc1e4, 0x00000001, 0x00000001,
		810	0xd0c0, 0xfffffff0, 0x00000100,
		811	0xd8c0, 0xfffffff0, 0x00000100
		812	};
		813
		814	static const u32 verde_mgcg_cgcg_init[] =
		815	{
		816	0xc400, 0xffffffff, 0xfffffffc,
		817	0x802c, 0xffffffff, 0xe0000000,
		818	0x9a60, 0xffffffff, 0x00000100,
		819	0x92a4, 0xffffffff, 0x00000100,
		820	0xc164, 0xffffffff, 0x00000100,
		821	0x9774, 0xffffffff, 0x00000100,
		822	0x8984, 0xffffffff, 0x06000100,
		823	0x8a18, 0xffffffff, 0x00000100,
		824	0x92a0, 0xffffffff, 0x00000100,
		825	0xc380, 0xffffffff, 0x00000100,
		826	0x8b28, 0xffffffff, 0x00000100,
		827	0x9144, 0xffffffff, 0x00000100,
		828	0x8d88, 0xffffffff, 0x00000100,
		829	0x8d8c, 0xffffffff, 0x00000100,
		830	0x9030, 0xffffffff, 0x00000100,
		831	0x9034, 0xffffffff, 0x00000100,
		832	0x9038, 0xffffffff, 0x00000100,
		833	0x903c, 0xffffffff, 0x00000100,
		834	0xad80, 0xffffffff, 0x00000100,
		835	0xac54, 0xffffffff, 0x00000100,
		836	0x897c, 0xffffffff, 0x06000100,
		837	0x9868, 0xffffffff, 0x00000100,
		838	0x9510, 0xffffffff, 0x00000100,
		839	0xaf04, 0xffffffff, 0x00000100,
		840	0xae04, 0xffffffff, 0x00000100,
		841	0x949c, 0xffffffff, 0x00000100,
		842	0x802c, 0xffffffff, 0xe0000000,
		843	0x9160, 0xffffffff, 0x00010000,
		844	0x9164, 0xffffffff, 0x00030002,
		845	0x9168, 0xffffffff, 0x00040007,
		846	0x916c, 0xffffffff, 0x00060005,
		847	0x9170, 0xffffffff, 0x00090008,
		848	0x9174, 0xffffffff, 0x00020001,
		849	0x9178, 0xffffffff, 0x00040003,
		850	0x917c, 0xffffffff, 0x00000007,
		851	0x9180, 0xffffffff, 0x00060005,
		852	0x9184, 0xffffffff, 0x00090008,
		853	0x9188, 0xffffffff, 0x00030002,
		854	0x918c, 0xffffffff, 0x00050004,
		855	0x9190, 0xffffffff, 0x00000008,
		856	0x9194, 0xffffffff, 0x00070006,
		857	0x9198, 0xffffffff, 0x000a0009,
		858	0x919c, 0xffffffff, 0x00040003,
		859	0x91a0, 0xffffffff, 0x00060005,
		860	0x91a4, 0xffffffff, 0x00000009,
		861	0x91a8, 0xffffffff, 0x00080007,
		862	0x91ac, 0xffffffff, 0x000b000a,
		863	0x91b0, 0xffffffff, 0x00050004,
		864	0x91b4, 0xffffffff, 0x00070006,
		865	0x91b8, 0xffffffff, 0x0008000b,
		866	0x91bc, 0xffffffff, 0x000a0009,
		867	0x91c0, 0xffffffff, 0x000d000c,
		868	0x9200, 0xffffffff, 0x00090008,
		869	0x9204, 0xffffffff, 0x000b000a,
		870	0x9208, 0xffffffff, 0x000c000f,
		871	0x920c, 0xffffffff, 0x000e000d,
		872	0x9210, 0xffffffff, 0x00110010,
		873	0x9214, 0xffffffff, 0x000a0009,
		874	0x9218, 0xffffffff, 0x000c000b,
		875	0x921c, 0xffffffff, 0x0000000f,
		876	0x9220, 0xffffffff, 0x000e000d,
		877	0x9224, 0xffffffff, 0x00110010,
		878	0x9228, 0xffffffff, 0x000b000a,
		879	0x922c, 0xffffffff, 0x000d000c,
		880	0x9230, 0xffffffff, 0x00000010,
		881	0x9234, 0xffffffff, 0x000f000e,
		882	0x9238, 0xffffffff, 0x00120011,
		883	0x923c, 0xffffffff, 0x000c000b,
		884	0x9240, 0xffffffff, 0x000e000d,
		885	0x9244, 0xffffffff, 0x00000011,
		886	0x9248, 0xffffffff, 0x0010000f,
		887	0x924c, 0xffffffff, 0x00130012,
		888	0x9250, 0xffffffff, 0x000d000c,
		889	0x9254, 0xffffffff, 0x000f000e,
		890	0x9258, 0xffffffff, 0x00100013,
		891	0x925c, 0xffffffff, 0x00120011,
		892	0x9260, 0xffffffff, 0x00150014,
		893	0x9150, 0xffffffff, 0x96940200,
		894	0x8708, 0xffffffff, 0x00900100,
		895	0xc478, 0xffffffff, 0x00000080,
		896	0xc404, 0xffffffff, 0x0020003f,
		897	0x30, 0xffffffff, 0x0000001c,
		898	0x34, 0x000f0000, 0x000f0000,
		899	0x160c, 0xffffffff, 0x00000100,
		900	0x1024, 0xffffffff, 0x00000100,
		901	0x102c, 0x00000101, 0x00000000,
		902	0x20a8, 0xffffffff, 0x00000104,
		903	0x264c, 0x000c0000, 0x000c0000,
		904	0x2648, 0x000c0000, 0x000c0000,
		905	0x55e4, 0xff000fff, 0x00000100,
		906	0x55e8, 0x00000001, 0x00000001,
		907	0x2f50, 0x00000001, 0x00000001,
		908	0x30cc, 0xc0000fff, 0x00000104,
		909	0xc1e4, 0x00000001, 0x00000001,
		910	0xd0c0, 0xfffffff0, 0x00000100,
		911	0xd8c0, 0xfffffff0, 0x00000100
		912	};
		913
		914	static const u32 oland_mgcg_cgcg_init[] =
		915	{
		916	0xc400, 0xffffffff, 0xfffffffc,
		917	0x802c, 0xffffffff, 0xe0000000,
		918	0x9a60, 0xffffffff, 0x00000100,
		919	0x92a4, 0xffffffff, 0x00000100,
		920	0xc164, 0xffffffff, 0x00000100,
		921	0x9774, 0xffffffff, 0x00000100,
		922	0x8984, 0xffffffff, 0x06000100,
		923	0x8a18, 0xffffffff, 0x00000100,
		924	0x92a0, 0xffffffff, 0x00000100,
		925	0xc380, 0xffffffff, 0x00000100,
		926	0x8b28, 0xffffffff, 0x00000100,
		927	0x9144, 0xffffffff, 0x00000100,
		928	0x8d88, 0xffffffff, 0x00000100,
		929	0x8d8c, 0xffffffff, 0x00000100,
		930	0x9030, 0xffffffff, 0x00000100,
		931	0x9034, 0xffffffff, 0x00000100,
		932	0x9038, 0xffffffff, 0x00000100,
		933	0x903c, 0xffffffff, 0x00000100,
		934	0xad80, 0xffffffff, 0x00000100,
		935	0xac54, 0xffffffff, 0x00000100,
		936	0x897c, 0xffffffff, 0x06000100,
		937	0x9868, 0xffffffff, 0x00000100,
		938	0x9510, 0xffffffff, 0x00000100,
		939	0xaf04, 0xffffffff, 0x00000100,
		940	0xae04, 0xffffffff, 0x00000100,
		941	0x949c, 0xffffffff, 0x00000100,
		942	0x802c, 0xffffffff, 0xe0000000,
		943	0x9160, 0xffffffff, 0x00010000,
		944	0x9164, 0xffffffff, 0x00030002,
		945	0x9168, 0xffffffff, 0x00040007,
		946	0x916c, 0xffffffff, 0x00060005,
		947	0x9170, 0xffffffff, 0x00090008,
		948	0x9174, 0xffffffff, 0x00020001,
		949	0x9178, 0xffffffff, 0x00040003,
		950	0x917c, 0xffffffff, 0x00000007,
		951	0x9180, 0xffffffff, 0x00060005,
		952	0x9184, 0xffffffff, 0x00090008,
		953	0x9188, 0xffffffff, 0x00030002,
		954	0x918c, 0xffffffff, 0x00050004,
		955	0x9190, 0xffffffff, 0x00000008,
		956	0x9194, 0xffffffff, 0x00070006,
		957	0x9198, 0xffffffff, 0x000a0009,
		958	0x919c, 0xffffffff, 0x00040003,
		959	0x91a0, 0xffffffff, 0x00060005,
		960	0x91a4, 0xffffffff, 0x00000009,
		961	0x91a8, 0xffffffff, 0x00080007,
		962	0x91ac, 0xffffffff, 0x000b000a,
		963	0x91b0, 0xffffffff, 0x00050004,
		964	0x91b4, 0xffffffff, 0x00070006,
		965	0x91b8, 0xffffffff, 0x0008000b,
		966	0x91bc, 0xffffffff, 0x000a0009,
		967	0x91c0, 0xffffffff, 0x000d000c,
		968	0x91c4, 0xffffffff, 0x00060005,
		969	0x91c8, 0xffffffff, 0x00080007,
		970	0x91cc, 0xffffffff, 0x0000000b,
		971	0x91d0, 0xffffffff, 0x000a0009,
		972	0x91d4, 0xffffffff, 0x000d000c,
		973	0x9150, 0xffffffff, 0x96940200,
		974	0x8708, 0xffffffff, 0x00900100,
		975	0xc478, 0xffffffff, 0x00000080,
		976	0xc404, 0xffffffff, 0x0020003f,
		977	0x30, 0xffffffff, 0x0000001c,
		978	0x34, 0x000f0000, 0x000f0000,
		979	0x160c, 0xffffffff, 0x00000100,
		980	0x1024, 0xffffffff, 0x00000100,
		981	0x102c, 0x00000101, 0x00000000,
		982	0x20a8, 0xffffffff, 0x00000104,
		983	0x264c, 0x000c0000, 0x000c0000,
		984	0x2648, 0x000c0000, 0x000c0000,
		985	0x55e4, 0xff000fff, 0x00000100,
		986	0x55e8, 0x00000001, 0x00000001,
		987	0x2f50, 0x00000001, 0x00000001,
		988	0x30cc, 0xc0000fff, 0x00000104,
		989	0xc1e4, 0x00000001, 0x00000001,
		990	0xd0c0, 0xfffffff0, 0x00000100,
		991	0xd8c0, 0xfffffff0, 0x00000100
		992	};
		993
		994	static const u32 hainan_mgcg_cgcg_init[] =
		995	{
		996	0xc400, 0xffffffff, 0xfffffffc,
		997	0x802c, 0xffffffff, 0xe0000000,
		998	0x9a60, 0xffffffff, 0x00000100,
		999	0x92a4, 0xffffffff, 0x00000100,
		1000	0xc164, 0xffffffff, 0x00000100,
		1001	0x9774, 0xffffffff, 0x00000100,
		1002	0x8984, 0xffffffff, 0x06000100,
		1003	0x8a18, 0xffffffff, 0x00000100,
		1004	0x92a0, 0xffffffff, 0x00000100,
		1005	0xc380, 0xffffffff, 0x00000100,
		1006	0x8b28, 0xffffffff, 0x00000100,
		1007	0x9144, 0xffffffff, 0x00000100,
		1008	0x8d88, 0xffffffff, 0x00000100,
		1009	0x8d8c, 0xffffffff, 0x00000100,
		1010	0x9030, 0xffffffff, 0x00000100,
		1011	0x9034, 0xffffffff, 0x00000100,
		1012	0x9038, 0xffffffff, 0x00000100,
		1013	0x903c, 0xffffffff, 0x00000100,
		1014	0xad80, 0xffffffff, 0x00000100,
		1015	0xac54, 0xffffffff, 0x00000100,
		1016	0x897c, 0xffffffff, 0x06000100,
		1017	0x9868, 0xffffffff, 0x00000100,
		1018	0x9510, 0xffffffff, 0x00000100,
		1019	0xaf04, 0xffffffff, 0x00000100,
		1020	0xae04, 0xffffffff, 0x00000100,
		1021	0x949c, 0xffffffff, 0x00000100,
		1022	0x802c, 0xffffffff, 0xe0000000,
		1023	0x9160, 0xffffffff, 0x00010000,
		1024	0x9164, 0xffffffff, 0x00030002,
		1025	0x9168, 0xffffffff, 0x00040007,
		1026	0x916c, 0xffffffff, 0x00060005,
		1027	0x9170, 0xffffffff, 0x00090008,
		1028	0x9174, 0xffffffff, 0x00020001,
		1029	0x9178, 0xffffffff, 0x00040003,
		1030	0x917c, 0xffffffff, 0x00000007,
		1031	0x9180, 0xffffffff, 0x00060005,
		1032	0x9184, 0xffffffff, 0x00090008,
		1033	0x9188, 0xffffffff, 0x00030002,
		1034	0x918c, 0xffffffff, 0x00050004,
		1035	0x9190, 0xffffffff, 0x00000008,
		1036	0x9194, 0xffffffff, 0x00070006,
		1037	0x9198, 0xffffffff, 0x000a0009,
		1038	0x919c, 0xffffffff, 0x00040003,
		1039	0x91a0, 0xffffffff, 0x00060005,
		1040	0x91a4, 0xffffffff, 0x00000009,
		1041	0x91a8, 0xffffffff, 0x00080007,
		1042	0x91ac, 0xffffffff, 0x000b000a,
		1043	0x91b0, 0xffffffff, 0x00050004,
		1044	0x91b4, 0xffffffff, 0x00070006,
		1045	0x91b8, 0xffffffff, 0x0008000b,
		1046	0x91bc, 0xffffffff, 0x000a0009,
		1047	0x91c0, 0xffffffff, 0x000d000c,
		1048	0x91c4, 0xffffffff, 0x00060005,
		1049	0x91c8, 0xffffffff, 0x00080007,
		1050	0x91cc, 0xffffffff, 0x0000000b,
		1051	0x91d0, 0xffffffff, 0x000a0009,
		1052	0x91d4, 0xffffffff, 0x000d000c,
		1053	0x9150, 0xffffffff, 0x96940200,
		1054	0x8708, 0xffffffff, 0x00900100,
		1055	0xc478, 0xffffffff, 0x00000080,
		1056	0xc404, 0xffffffff, 0x0020003f,
		1057	0x30, 0xffffffff, 0x0000001c,
		1058	0x34, 0x000f0000, 0x000f0000,
		1059	0x160c, 0xffffffff, 0x00000100,
		1060	0x1024, 0xffffffff, 0x00000100,
		1061	0x20a8, 0xffffffff, 0x00000104,
		1062	0x264c, 0x000c0000, 0x000c0000,
		1063	0x2648, 0x000c0000, 0x000c0000,
		1064	0x2f50, 0x00000001, 0x00000001,
		1065	0x30cc, 0xc0000fff, 0x00000104,
		1066	0xc1e4, 0x00000001, 0x00000001,
		1067	0xd0c0, 0xfffffff0, 0x00000100,
		1068	0xd8c0, 0xfffffff0, 0x00000100
		1069	};
		1070
		1071	static u32 verde_pg_init[] =
		1072	{
		1073	0x353c, 0xffffffff, 0x40000,
		1074	0x3538, 0xffffffff, 0x200010ff,
		1075	0x353c, 0xffffffff, 0x0,
		1076	0x353c, 0xffffffff, 0x0,
		1077	0x353c, 0xffffffff, 0x0,
		1078	0x353c, 0xffffffff, 0x0,
		1079	0x353c, 0xffffffff, 0x0,
		1080	0x353c, 0xffffffff, 0x7007,
		1081	0x3538, 0xffffffff, 0x300010ff,
		1082	0x353c, 0xffffffff, 0x0,
		1083	0x353c, 0xffffffff, 0x0,
		1084	0x353c, 0xffffffff, 0x0,
		1085	0x353c, 0xffffffff, 0x0,
		1086	0x353c, 0xffffffff, 0x0,
		1087	0x353c, 0xffffffff, 0x400000,
		1088	0x3538, 0xffffffff, 0x100010ff,
		1089	0x353c, 0xffffffff, 0x0,
		1090	0x353c, 0xffffffff, 0x0,
		1091	0x353c, 0xffffffff, 0x0,
		1092	0x353c, 0xffffffff, 0x0,
		1093	0x353c, 0xffffffff, 0x0,
		1094	0x353c, 0xffffffff, 0x120200,
		1095	0x3538, 0xffffffff, 0x500010ff,
		1096	0x353c, 0xffffffff, 0x0,
		1097	0x353c, 0xffffffff, 0x0,
		1098	0x353c, 0xffffffff, 0x0,
		1099	0x353c, 0xffffffff, 0x0,
		1100	0x353c, 0xffffffff, 0x0,
		1101	0x353c, 0xffffffff, 0x1e1e16,
		1102	0x3538, 0xffffffff, 0x600010ff,
		1103	0x353c, 0xffffffff, 0x0,
		1104	0x353c, 0xffffffff, 0x0,
		1105	0x353c, 0xffffffff, 0x0,
		1106	0x353c, 0xffffffff, 0x0,
		1107	0x353c, 0xffffffff, 0x0,
		1108	0x353c, 0xffffffff, 0x171f1e,
		1109	0x3538, 0xffffffff, 0x700010ff,
		1110	0x353c, 0xffffffff, 0x0,
		1111	0x353c, 0xffffffff, 0x0,
		1112	0x353c, 0xffffffff, 0x0,
		1113	0x353c, 0xffffffff, 0x0,
		1114	0x353c, 0xffffffff, 0x0,
		1115	0x353c, 0xffffffff, 0x0,
		1116	0x3538, 0xffffffff, 0x9ff,
		1117	0x3500, 0xffffffff, 0x0,
		1118	0x3504, 0xffffffff, 0x10000800,
		1119	0x3504, 0xffffffff, 0xf,
		1120	0x3504, 0xffffffff, 0xf,
		1121	0x3500, 0xffffffff, 0x4,
		1122	0x3504, 0xffffffff, 0x1000051e,
		1123	0x3504, 0xffffffff, 0xffff,
		1124	0x3504, 0xffffffff, 0xffff,
		1125	0x3500, 0xffffffff, 0x8,
		1126	0x3504, 0xffffffff, 0x80500,
		1127	0x3500, 0xffffffff, 0x12,
		1128	0x3504, 0xffffffff, 0x9050c,
		1129	0x3500, 0xffffffff, 0x1d,
		1130	0x3504, 0xffffffff, 0xb052c,
		1131	0x3500, 0xffffffff, 0x2a,
		1132	0x3504, 0xffffffff, 0x1053e,
		1133	0x3500, 0xffffffff, 0x2d,
		1134	0x3504, 0xffffffff, 0x10546,
		1135	0x3500, 0xffffffff, 0x30,
		1136	0x3504, 0xffffffff, 0xa054e,
		1137	0x3500, 0xffffffff, 0x3c,
		1138	0x3504, 0xffffffff, 0x1055f,
		1139	0x3500, 0xffffffff, 0x3f,
		1140	0x3504, 0xffffffff, 0x10567,
		1141	0x3500, 0xffffffff, 0x42,
		1142	0x3504, 0xffffffff, 0x1056f,
		1143	0x3500, 0xffffffff, 0x45,
		1144	0x3504, 0xffffffff, 0x10572,
		1145	0x3500, 0xffffffff, 0x48,
		1146	0x3504, 0xffffffff, 0x20575,
		1147	0x3500, 0xffffffff, 0x4c,
		1148	0x3504, 0xffffffff, 0x190801,
		1149	0x3500, 0xffffffff, 0x67,
		1150	0x3504, 0xffffffff, 0x1082a,
		1151	0x3500, 0xffffffff, 0x6a,
		1152	0x3504, 0xffffffff, 0x1b082d,
		1153	0x3500, 0xffffffff, 0x87,
		1154	0x3504, 0xffffffff, 0x310851,
		1155	0x3500, 0xffffffff, 0xba,
		1156	0x3504, 0xffffffff, 0x891,
		1157	0x3500, 0xffffffff, 0xbc,
		1158	0x3504, 0xffffffff, 0x893,
		1159	0x3500, 0xffffffff, 0xbe,
		1160	0x3504, 0xffffffff, 0x20895,
		1161	0x3500, 0xffffffff, 0xc2,
		1162	0x3504, 0xffffffff, 0x20899,
		1163	0x3500, 0xffffffff, 0xc6,
		1164	0x3504, 0xffffffff, 0x2089d,
		1165	0x3500, 0xffffffff, 0xca,
		1166	0x3504, 0xffffffff, 0x8a1,
		1167	0x3500, 0xffffffff, 0xcc,
		1168	0x3504, 0xffffffff, 0x8a3,
		1169	0x3500, 0xffffffff, 0xce,
		1170	0x3504, 0xffffffff, 0x308a5,
		1171	0x3500, 0xffffffff, 0xd3,
		1172	0x3504, 0xffffffff, 0x6d08cd,
		1173	0x3500, 0xffffffff, 0x142,
		1174	0x3504, 0xffffffff, 0x2000095a,
		1175	0x3504, 0xffffffff, 0x1,
		1176	0x3500, 0xffffffff, 0x144,
		1177	0x3504, 0xffffffff, 0x301f095b,
		1178	0x3500, 0xffffffff, 0x165,
		1179	0x3504, 0xffffffff, 0xc094d,
		1180	0x3500, 0xffffffff, 0x173,
		1181	0x3504, 0xffffffff, 0xf096d,
		1182	0x3500, 0xffffffff, 0x184,
		1183	0x3504, 0xffffffff, 0x15097f,
		1184	0x3500, 0xffffffff, 0x19b,
		1185	0x3504, 0xffffffff, 0xc0998,
		1186	0x3500, 0xffffffff, 0x1a9,
		1187	0x3504, 0xffffffff, 0x409a7,
		1188	0x3500, 0xffffffff, 0x1af,
		1189	0x3504, 0xffffffff, 0xcdc,
		1190	0x3500, 0xffffffff, 0x1b1,
		1191	0x3504, 0xffffffff, 0x800,
		1192	0x3508, 0xffffffff, 0x6c9b2000,
		1193	0x3510, 0xfc00, 0x2000,
		1194	0x3544, 0xffffffff, 0xfc0,
		1195	0x28d4, 0x00000100, 0x100
		1196	};
		1197
		1198	static void si_init_golden_registers(struct radeon_device *rdev)
		1199	{
		1200	switch (rdev->family) {
		1201	case CHIP_TAHITI:
		1202	radeon_program_register_sequence(rdev,
		1203	tahiti_golden_registers,
		1204	(const u32)ARRAY_SIZE(tahiti_golden_registers));
		1205	radeon_program_register_sequence(rdev,
		1206	tahiti_golden_rlc_registers,
		1207	(const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
		1208	radeon_program_register_sequence(rdev,
		1209	tahiti_mgcg_cgcg_init,
		1210	(const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
		1211	radeon_program_register_sequence(rdev,
		1212	tahiti_golden_registers2,
		1213	(const u32)ARRAY_SIZE(tahiti_golden_registers2));
		1214	break;
		1215	case CHIP_PITCAIRN:
		1216	radeon_program_register_sequence(rdev,
		1217	pitcairn_golden_registers,
		1218	(const u32)ARRAY_SIZE(pitcairn_golden_registers));
		1219	radeon_program_register_sequence(rdev,
		1220	pitcairn_golden_rlc_registers,
		1221	(const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
		1222	radeon_program_register_sequence(rdev,
		1223	pitcairn_mgcg_cgcg_init,
		1224	(const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
		1225	break;
		1226	case CHIP_VERDE:
		1227	radeon_program_register_sequence(rdev,
		1228	verde_golden_registers,
		1229	(const u32)ARRAY_SIZE(verde_golden_registers));
		1230	radeon_program_register_sequence(rdev,
		1231	verde_golden_rlc_registers,
		1232	(const u32)ARRAY_SIZE(verde_golden_rlc_registers));
		1233	radeon_program_register_sequence(rdev,
		1234	verde_mgcg_cgcg_init,
		1235	(const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
		1236	radeon_program_register_sequence(rdev,
		1237	verde_pg_init,
		1238	(const u32)ARRAY_SIZE(verde_pg_init));
		1239	break;
		1240	case CHIP_OLAND:
		1241	radeon_program_register_sequence(rdev,
		1242	oland_golden_registers,
		1243	(const u32)ARRAY_SIZE(oland_golden_registers));
		1244	radeon_program_register_sequence(rdev,
		1245	oland_golden_rlc_registers,
		1246	(const u32)ARRAY_SIZE(oland_golden_rlc_registers));
		1247	radeon_program_register_sequence(rdev,
		1248	oland_mgcg_cgcg_init,
		1249	(const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
		1250	break;
		1251	case CHIP_HAINAN:
		1252	radeon_program_register_sequence(rdev,
		1253	hainan_golden_registers,
		1254	(const u32)ARRAY_SIZE(hainan_golden_registers));
		1255	radeon_program_register_sequence(rdev,
		1256	hainan_golden_registers2,
		1257	(const u32)ARRAY_SIZE(hainan_golden_registers2));
		1258	radeon_program_register_sequence(rdev,
		1259	hainan_mgcg_cgcg_init,
		1260	(const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
		1261	break;
		1262	default:
		1263	break;
		1264	}
		1265	}
		1266
6104	serge	1267	/**
		1268	* si_get_allowed_info_register - fetch the register for the info ioctl
		1269	*
		1270	* @rdev: radeon_device pointer
		1271	* @reg: register offset in bytes
		1272	* @val: register value
		1273	*
		1274	* Returns 0 for success or -EINVAL for an invalid register
		1275	*
		1276	*/
		1277	int si_get_allowed_info_register(struct radeon_device *rdev,
		1278	u32 reg, u32 *val)
		1279	{
		1280	switch (reg) {
		1281	case GRBM_STATUS:
		1282	case GRBM_STATUS2:
		1283	case GRBM_STATUS_SE0:
		1284	case GRBM_STATUS_SE1:
		1285	case SRBM_STATUS:
		1286	case SRBM_STATUS2:
		1287	case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
		1288	case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
		1289	case UVD_STATUS:
		1290	*val = RREG32(reg);
		1291	return 0;
		1292	default:
		1293	return -EINVAL;
		1294	}
		1295	}
		1296
3764	Serge	1297	#define PCIE_BUS_CLK 10000
		1298	#define TCLK (PCIE_BUS_CLK / 10)
		1299
		1300	/**
		1301	* si_get_xclk - get the xclk
		1302	*
		1303	* @rdev: radeon_device pointer
		1304	*
		1305	* Returns the reference clock used by the gfx engine
		1306	* (SI).
		1307	*/
		1308	u32 si_get_xclk(struct radeon_device *rdev)
		1309	{
		1310	u32 reference_clock = rdev->clock.spll.reference_freq;
		1311	u32 tmp;
		1312
		1313	tmp = RREG32(CG_CLKPIN_CNTL_2);
		1314	if (tmp & MUX_TCLK_TO_XCLK)
		1315	return TCLK;
		1316
		1317	tmp = RREG32(CG_CLKPIN_CNTL);
		1318	if (tmp & XTALIN_DIVIDE)
		1319	return reference_clock / 4;
		1320
		1321	return reference_clock;
		1322	}
		1323
2997	Serge	1324	/* get temperature in millidegrees */
		1325	int si_get_temp(struct radeon_device *rdev)
		1326	{
		1327	u32 temp;
		1328	int actual_temp = 0;
		1329
		1330	temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
		1331	CTF_TEMP_SHIFT;
		1332
		1333	if (temp & 0x200)
		1334	actual_temp = 255;
		1335	else
		1336	actual_temp = temp & 0x1ff;
		1337
		1338	actual_temp = (actual_temp * 1000);
		1339
		1340	return actual_temp;
		1341	}
		1342
		1343	#define TAHITI_IO_MC_REGS_SIZE 36
		1344
		1345	static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1346	{0x0000006f, 0x03044000},
		1347	{0x00000070, 0x0480c018},
		1348	{0x00000071, 0x00000040},
		1349	{0x00000072, 0x01000000},
		1350	{0x00000074, 0x000000ff},
		1351	{0x00000075, 0x00143400},
		1352	{0x00000076, 0x08ec0800},
		1353	{0x00000077, 0x040000cc},
		1354	{0x00000079, 0x00000000},
		1355	{0x0000007a, 0x21000409},
		1356	{0x0000007c, 0x00000000},
		1357	{0x0000007d, 0xe8000000},
		1358	{0x0000007e, 0x044408a8},
		1359	{0x0000007f, 0x00000003},
		1360	{0x00000080, 0x00000000},
		1361	{0x00000081, 0x01000000},
		1362	{0x00000082, 0x02000000},
		1363	{0x00000083, 0x00000000},
		1364	{0x00000084, 0xe3f3e4f4},
		1365	{0x00000085, 0x00052024},
		1366	{0x00000087, 0x00000000},
		1367	{0x00000088, 0x66036603},
		1368	{0x00000089, 0x01000000},
		1369	{0x0000008b, 0x1c0a0000},
		1370	{0x0000008c, 0xff010000},
		1371	{0x0000008e, 0xffffefff},
		1372	{0x0000008f, 0xfff3efff},
		1373	{0x00000090, 0xfff3efbf},
		1374	{0x00000094, 0x00101101},
		1375	{0x00000095, 0x00000fff},
		1376	{0x00000096, 0x00116fff},
		1377	{0x00000097, 0x60010000},
		1378	{0x00000098, 0x10010000},
		1379	{0x00000099, 0x00006000},
		1380	{0x0000009a, 0x00001000},
		1381	{0x0000009f, 0x00a77400}
		1382	};
		1383
		1384	static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1385	{0x0000006f, 0x03044000},
		1386	{0x00000070, 0x0480c018},
		1387	{0x00000071, 0x00000040},
		1388	{0x00000072, 0x01000000},
		1389	{0x00000074, 0x000000ff},
		1390	{0x00000075, 0x00143400},
		1391	{0x00000076, 0x08ec0800},
		1392	{0x00000077, 0x040000cc},
		1393	{0x00000079, 0x00000000},
		1394	{0x0000007a, 0x21000409},
		1395	{0x0000007c, 0x00000000},
		1396	{0x0000007d, 0xe8000000},
		1397	{0x0000007e, 0x044408a8},
		1398	{0x0000007f, 0x00000003},
		1399	{0x00000080, 0x00000000},
		1400	{0x00000081, 0x01000000},
		1401	{0x00000082, 0x02000000},
		1402	{0x00000083, 0x00000000},
		1403	{0x00000084, 0xe3f3e4f4},
		1404	{0x00000085, 0x00052024},
		1405	{0x00000087, 0x00000000},
		1406	{0x00000088, 0x66036603},
		1407	{0x00000089, 0x01000000},
		1408	{0x0000008b, 0x1c0a0000},
		1409	{0x0000008c, 0xff010000},
		1410	{0x0000008e, 0xffffefff},
		1411	{0x0000008f, 0xfff3efff},
		1412	{0x00000090, 0xfff3efbf},
		1413	{0x00000094, 0x00101101},
		1414	{0x00000095, 0x00000fff},
		1415	{0x00000096, 0x00116fff},
		1416	{0x00000097, 0x60010000},
		1417	{0x00000098, 0x10010000},
		1418	{0x00000099, 0x00006000},
		1419	{0x0000009a, 0x00001000},
		1420	{0x0000009f, 0x00a47400}
		1421	};
		1422
		1423	static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1424	{0x0000006f, 0x03044000},
		1425	{0x00000070, 0x0480c018},
		1426	{0x00000071, 0x00000040},
		1427	{0x00000072, 0x01000000},
		1428	{0x00000074, 0x000000ff},
		1429	{0x00000075, 0x00143400},
		1430	{0x00000076, 0x08ec0800},
		1431	{0x00000077, 0x040000cc},
		1432	{0x00000079, 0x00000000},
		1433	{0x0000007a, 0x21000409},
		1434	{0x0000007c, 0x00000000},
		1435	{0x0000007d, 0xe8000000},
		1436	{0x0000007e, 0x044408a8},
		1437	{0x0000007f, 0x00000003},
		1438	{0x00000080, 0x00000000},
		1439	{0x00000081, 0x01000000},
		1440	{0x00000082, 0x02000000},
		1441	{0x00000083, 0x00000000},
		1442	{0x00000084, 0xe3f3e4f4},
		1443	{0x00000085, 0x00052024},
		1444	{0x00000087, 0x00000000},
		1445	{0x00000088, 0x66036603},
		1446	{0x00000089, 0x01000000},
		1447	{0x0000008b, 0x1c0a0000},
		1448	{0x0000008c, 0xff010000},
		1449	{0x0000008e, 0xffffefff},
		1450	{0x0000008f, 0xfff3efff},
		1451	{0x00000090, 0xfff3efbf},
		1452	{0x00000094, 0x00101101},
		1453	{0x00000095, 0x00000fff},
		1454	{0x00000096, 0x00116fff},
		1455	{0x00000097, 0x60010000},
		1456	{0x00000098, 0x10010000},
		1457	{0x00000099, 0x00006000},
		1458	{0x0000009a, 0x00001000},
		1459	{0x0000009f, 0x00a37400}
		1460	};
		1461
3764	Serge	1462	static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1463	{0x0000006f, 0x03044000},
		1464	{0x00000070, 0x0480c018},
		1465	{0x00000071, 0x00000040},
		1466	{0x00000072, 0x01000000},
		1467	{0x00000074, 0x000000ff},
		1468	{0x00000075, 0x00143400},
		1469	{0x00000076, 0x08ec0800},
		1470	{0x00000077, 0x040000cc},
		1471	{0x00000079, 0x00000000},
		1472	{0x0000007a, 0x21000409},
		1473	{0x0000007c, 0x00000000},
		1474	{0x0000007d, 0xe8000000},
		1475	{0x0000007e, 0x044408a8},
		1476	{0x0000007f, 0x00000003},
		1477	{0x00000080, 0x00000000},
		1478	{0x00000081, 0x01000000},
		1479	{0x00000082, 0x02000000},
		1480	{0x00000083, 0x00000000},
		1481	{0x00000084, 0xe3f3e4f4},
		1482	{0x00000085, 0x00052024},
		1483	{0x00000087, 0x00000000},
		1484	{0x00000088, 0x66036603},
		1485	{0x00000089, 0x01000000},
		1486	{0x0000008b, 0x1c0a0000},
		1487	{0x0000008c, 0xff010000},
		1488	{0x0000008e, 0xffffefff},
		1489	{0x0000008f, 0xfff3efff},
		1490	{0x00000090, 0xfff3efbf},
		1491	{0x00000094, 0x00101101},
		1492	{0x00000095, 0x00000fff},
		1493	{0x00000096, 0x00116fff},
		1494	{0x00000097, 0x60010000},
		1495	{0x00000098, 0x10010000},
		1496	{0x00000099, 0x00006000},
		1497	{0x0000009a, 0x00001000},
		1498	{0x0000009f, 0x00a17730}
		1499	};
		1500
		1501	static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1502	{0x0000006f, 0x03044000},
		1503	{0x00000070, 0x0480c018},
		1504	{0x00000071, 0x00000040},
		1505	{0x00000072, 0x01000000},
		1506	{0x00000074, 0x000000ff},
		1507	{0x00000075, 0x00143400},
		1508	{0x00000076, 0x08ec0800},
		1509	{0x00000077, 0x040000cc},
		1510	{0x00000079, 0x00000000},
		1511	{0x0000007a, 0x21000409},
		1512	{0x0000007c, 0x00000000},
		1513	{0x0000007d, 0xe8000000},
		1514	{0x0000007e, 0x044408a8},
		1515	{0x0000007f, 0x00000003},
		1516	{0x00000080, 0x00000000},
		1517	{0x00000081, 0x01000000},
		1518	{0x00000082, 0x02000000},
		1519	{0x00000083, 0x00000000},
		1520	{0x00000084, 0xe3f3e4f4},
		1521	{0x00000085, 0x00052024},
		1522	{0x00000087, 0x00000000},
		1523	{0x00000088, 0x66036603},
		1524	{0x00000089, 0x01000000},
		1525	{0x0000008b, 0x1c0a0000},
		1526	{0x0000008c, 0xff010000},
		1527	{0x0000008e, 0xffffefff},
		1528	{0x0000008f, 0xfff3efff},
		1529	{0x00000090, 0xfff3efbf},
		1530	{0x00000094, 0x00101101},
		1531	{0x00000095, 0x00000fff},
		1532	{0x00000096, 0x00116fff},
		1533	{0x00000097, 0x60010000},
		1534	{0x00000098, 0x10010000},
		1535	{0x00000099, 0x00006000},
		1536	{0x0000009a, 0x00001000},
		1537	{0x0000009f, 0x00a07730}
		1538	};
		1539
2997	Serge	1540	/* ucode loading */
5078	serge	1541	int si_mc_load_microcode(struct radeon_device *rdev)
2997	Serge	1542	{
5078	serge	1543	const __be32 *fw_data = NULL;
		1544	const __le32 *new_fw_data = NULL;
2997	Serge	1545	u32 running, blackout = 0;
5078	serge	1546	u32 *io_mc_regs = NULL;
		1547	const __le32 *new_io_mc_regs = NULL;
		1548	int i, regs_size, ucode_size;
2997	Serge	1549
		1550	if (!rdev->mc_fw)
		1551	return -EINVAL;
		1552
5078	serge	1553	if (rdev->new_fw) {
		1554	const struct mc_firmware_header_v1_0 *hdr =
		1555	(const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
		1556
		1557	radeon_ucode_print_mc_hdr(&hdr->header);
		1558	regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
		1559	new_io_mc_regs = (const __le32 *)
		1560	(rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
		1561	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
		1562	new_fw_data = (const __le32 *)
		1563	(rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
		1564	} else {
6104	serge	1565	ucode_size = rdev->mc_fw->size / 4;
5078	serge	1566
6104	serge	1567	switch (rdev->family) {
		1568	case CHIP_TAHITI:
		1569	io_mc_regs = (u32 *)&tahiti_io_mc_regs;
		1570	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1571	break;
		1572	case CHIP_PITCAIRN:
		1573	io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
		1574	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1575	break;
		1576	case CHIP_VERDE:
		1577	default:
		1578	io_mc_regs = (u32 *)&verde_io_mc_regs;
		1579	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1580	break;
		1581	case CHIP_OLAND:
		1582	io_mc_regs = (u32 *)&oland_io_mc_regs;
		1583	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1584	break;
		1585	case CHIP_HAINAN:
		1586	io_mc_regs = (u32 *)&hainan_io_mc_regs;
		1587	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1588	break;
		1589	}
5078	serge	1590	fw_data = (const __be32 *)rdev->mc_fw->data;
		1591	}
2997	Serge	1592
		1593	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
		1594
		1595	if (running == 0) {
		1596	if (running) {
		1597	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
		1598	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout \| 1);
		1599	}
		1600
		1601	/* reset the engine and set to writable */
		1602	WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
		1603	WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
		1604
		1605	/* load mc io regs */
		1606	for (i = 0; i < regs_size; i++) {
5078	serge	1607	if (rdev->new_fw) {
		1608	WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
		1609	WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
		1610	} else {
6104	serge	1611	WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
		1612	WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
		1613	}
2997	Serge	1614	}
		1615	/* load the MC ucode */
5078	serge	1616	for (i = 0; i < ucode_size; i++) {
		1617	if (rdev->new_fw)
		1618	WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
		1619	else
6104	serge	1620	WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
5078	serge	1621	}
2997	Serge	1622
		1623	/* put the engine back into the active state */
		1624	WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
		1625	WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
		1626	WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
		1627
		1628	/* wait for training to complete */
		1629	for (i = 0; i < rdev->usec_timeout; i++) {
		1630	if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
		1631	break;
		1632	udelay(1);
		1633	}
		1634	for (i = 0; i < rdev->usec_timeout; i++) {
		1635	if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
		1636	break;
		1637	udelay(1);
		1638	}
		1639
		1640	if (running)
		1641	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
		1642	}
		1643
		1644	return 0;
		1645	}
		1646
		1647	static int si_init_microcode(struct radeon_device *rdev)
		1648	{
		1649	const char *chip_name;
5078	serge	1650	const char *new_chip_name;
2997	Serge	1651	size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
5078	serge	1652	size_t smc_req_size, mc2_req_size;
2997	Serge	1653	char fw_name[30];
		1654	int err;
5078	serge	1655	int new_fw = 0;
2997	Serge	1656
		1657	DRM_DEBUG("\n");
		1658
		1659	switch (rdev->family) {
		1660	case CHIP_TAHITI:
		1661	chip_name = "TAHITI";
5078	serge	1662	new_chip_name = "tahiti";
2997	Serge	1663	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1664	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1665	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1666	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1667	mc_req_size = SI_MC_UCODE_SIZE * 4;
5078	serge	1668	mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
		1669	smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
2997	Serge	1670	break;
		1671	case CHIP_PITCAIRN:
		1672	chip_name = "PITCAIRN";
5078	serge	1673	new_chip_name = "pitcairn";
2997	Serge	1674	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1675	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1676	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1677	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1678	mc_req_size = SI_MC_UCODE_SIZE * 4;
5078	serge	1679	mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
		1680	smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
2997	Serge	1681	break;
		1682	case CHIP_VERDE:
		1683	chip_name = "VERDE";
5078	serge	1684	new_chip_name = "verde";
2997	Serge	1685	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1686	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1687	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1688	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1689	mc_req_size = SI_MC_UCODE_SIZE * 4;
5078	serge	1690	mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
		1691	smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
2997	Serge	1692	break;
3764	Serge	1693	case CHIP_OLAND:
		1694	chip_name = "OLAND";
5078	serge	1695	new_chip_name = "oland";
3764	Serge	1696	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1697	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1698	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1699	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
5078	serge	1700	mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
		1701	smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
3764	Serge	1702	break;
		1703	case CHIP_HAINAN:
		1704	chip_name = "HAINAN";
5078	serge	1705	new_chip_name = "hainan";
3764	Serge	1706	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1707	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1708	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1709	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
5078	serge	1710	mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
		1711	smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
3764	Serge	1712	break;
2997	Serge	1713	default: BUG();
		1714	}
		1715
5078	serge	1716	DRM_INFO("Loading %s Microcode\n", new_chip_name);
2997	Serge	1717
5078	serge	1718	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
		1719	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
		1720	if (err) {
6104	serge	1721	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
		1722	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
		1723	if (err)
		1724	goto out;
		1725	if (rdev->pfp_fw->size != pfp_req_size) {
		1726	printk(KERN_ERR
		1727	"si_cp: Bogus length %zu in firmware \"%s\"\n",
		1728	rdev->pfp_fw->size, fw_name);
		1729	err = -EINVAL;
		1730	goto out;
		1731	}
5078	serge	1732	} else {
		1733	err = radeon_ucode_validate(rdev->pfp_fw);
		1734	if (err) {
		1735	printk(KERN_ERR
		1736	"si_cp: validation failed for firmware \"%s\"\n",
		1737	fw_name);
		1738	goto out;
		1739	} else {
		1740	new_fw++;
		1741	}
		1742	}
2997	Serge	1743
5078	serge	1744	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
		1745	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
		1746	if (err) {
6104	serge	1747	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
		1748	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
		1749	if (err)
		1750	goto out;
		1751	if (rdev->me_fw->size != me_req_size) {
		1752	printk(KERN_ERR
		1753	"si_cp: Bogus length %zu in firmware \"%s\"\n",
		1754	rdev->me_fw->size, fw_name);
		1755	err = -EINVAL;
		1756	}
5078	serge	1757	} else {
		1758	err = radeon_ucode_validate(rdev->me_fw);
		1759	if (err) {
		1760	printk(KERN_ERR
		1761	"si_cp: validation failed for firmware \"%s\"\n",
		1762	fw_name);
		1763	goto out;
		1764	} else {
		1765	new_fw++;
		1766	}
		1767	}
2997	Serge	1768
5078	serge	1769	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
		1770	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
		1771	if (err) {
6104	serge	1772	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
		1773	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
		1774	if (err)
		1775	goto out;
		1776	if (rdev->ce_fw->size != ce_req_size) {
		1777	printk(KERN_ERR
		1778	"si_cp: Bogus length %zu in firmware \"%s\"\n",
		1779	rdev->ce_fw->size, fw_name);
		1780	err = -EINVAL;
		1781	}
5078	serge	1782	} else {
		1783	err = radeon_ucode_validate(rdev->ce_fw);
		1784	if (err) {
		1785	printk(KERN_ERR
		1786	"si_cp: validation failed for firmware \"%s\"\n",
		1787	fw_name);
		1788	goto out;
		1789	} else {
		1790	new_fw++;
		1791	}
		1792	}
2997	Serge	1793
5078	serge	1794	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
		1795	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
		1796	if (err) {
		1797	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
6104	serge	1798	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
		1799	if (err)
		1800	goto out;
		1801	if (rdev->rlc_fw->size != rlc_req_size) {
		1802	printk(KERN_ERR
		1803	"si_rlc: Bogus length %zu in firmware \"%s\"\n",
		1804	rdev->rlc_fw->size, fw_name);
		1805	err = -EINVAL;
		1806	}
5078	serge	1807	} else {
		1808	err = radeon_ucode_validate(rdev->rlc_fw);
		1809	if (err) {
		1810	printk(KERN_ERR
		1811	"si_cp: validation failed for firmware \"%s\"\n",
		1812	fw_name);
		1813	goto out;
		1814	} else {
		1815	new_fw++;
		1816	}
		1817	}
2997	Serge	1818
5078	serge	1819	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
		1820	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
		1821	if (err) {
6104	serge	1822	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
		1823	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
		1824	if (err) {
		1825	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
		1826	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
		1827	if (err)
		1828	goto out;
		1829	}
		1830	if ((rdev->mc_fw->size != mc_req_size) &&
		1831	(rdev->mc_fw->size != mc2_req_size)) {
		1832	printk(KERN_ERR
		1833	"si_mc: Bogus length %zu in firmware \"%s\"\n",
		1834	rdev->mc_fw->size, fw_name);
		1835	err = -EINVAL;
		1836	}
		1837	DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
5078	serge	1838	} else {
		1839	err = radeon_ucode_validate(rdev->mc_fw);
		1840	if (err) {
		1841	printk(KERN_ERR
		1842	"si_cp: validation failed for firmware \"%s\"\n",
		1843	fw_name);
		1844	goto out;
		1845	} else {
		1846	new_fw++;
		1847	}
		1848	}
2997	Serge	1849
5078	serge	1850	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
		1851	err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
		1852	if (err) {
6104	serge	1853	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
		1854	err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
		1855	if (err) {
		1856	printk(KERN_ERR
		1857	"smc: error loading firmware \"%s\"\n",
		1858	fw_name);
		1859	release_firmware(rdev->smc_fw);
		1860	rdev->smc_fw = NULL;
		1861	err = 0;
		1862	} else if (rdev->smc_fw->size != smc_req_size) {
		1863	printk(KERN_ERR
		1864	"si_smc: Bogus length %zu in firmware \"%s\"\n",
		1865	rdev->smc_fw->size, fw_name);
		1866	err = -EINVAL;
		1867	}
5078	serge	1868	} else {
		1869	err = radeon_ucode_validate(rdev->smc_fw);
		1870	if (err) {
		1871	printk(KERN_ERR
		1872	"si_cp: validation failed for firmware \"%s\"\n",
		1873	fw_name);
		1874	goto out;
		1875	} else {
		1876	new_fw++;
		1877	}
		1878	}
		1879
		1880	if (new_fw == 0) {
		1881	rdev->new_fw = false;
		1882	} else if (new_fw < 6) {
		1883	printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
		1884	err = -EINVAL;
		1885	} else {
		1886	rdev->new_fw = true;
		1887	}
2997	Serge	1888	out:
		1889	if (err) {
		1890	if (err != -EINVAL)
		1891	printk(KERN_ERR
		1892	"si_cp: Failed to load firmware \"%s\"\n",
		1893	fw_name);
		1894	release_firmware(rdev->pfp_fw);
		1895	rdev->pfp_fw = NULL;
		1896	release_firmware(rdev->me_fw);
		1897	rdev->me_fw = NULL;
		1898	release_firmware(rdev->ce_fw);
		1899	rdev->ce_fw = NULL;
		1900	release_firmware(rdev->rlc_fw);
		1901	rdev->rlc_fw = NULL;
		1902	release_firmware(rdev->mc_fw);
		1903	rdev->mc_fw = NULL;
5078	serge	1904	release_firmware(rdev->smc_fw);
		1905	rdev->smc_fw = NULL;
2997	Serge	1906	}
		1907	return err;
		1908	}
		1909
		1910	/* watermark setup */
		1911	static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
		1912	struct radeon_crtc *radeon_crtc,
		1913	struct drm_display_mode *mode,
		1914	struct drm_display_mode *other_mode)
		1915	{
5078	serge	1916	u32 tmp, buffer_alloc, i;
		1917	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
2997	Serge	1918	/*
		1919	* Line Buffer Setup
		1920	* There are 3 line buffers, each one shared by 2 display controllers.
		1921	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
		1922	* the display controllers. The paritioning is done via one of four
		1923	* preset allocations specified in bits 21:20:
		1924	* 0 - half lb
		1925	* 2 - whole lb, other crtc must be disabled
		1926	*/
		1927	/* this can get tricky if we have two large displays on a paired group
		1928	* of crtcs. Ideally for multiple large displays we'd assign them to
		1929	* non-linked crtcs for maximum line buffer allocation.
		1930	*/
		1931	if (radeon_crtc->base.enabled && mode) {
5078	serge	1932	if (other_mode) {
2997	Serge	1933	tmp = 0; /* 1/2 */
5078	serge	1934	buffer_alloc = 1;
		1935	} else {
2997	Serge	1936	tmp = 2; /* whole */
5078	serge	1937	buffer_alloc = 2;
		1938	}
		1939	} else {
2997	Serge	1940	tmp = 0;
5078	serge	1941	buffer_alloc = 0;
		1942	}
2997	Serge	1943
		1944	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
		1945	DC_LB_MEMORY_CONFIG(tmp));
		1946
5078	serge	1947	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
		1948	DMIF_BUFFERS_ALLOCATED(buffer_alloc));
		1949	for (i = 0; i < rdev->usec_timeout; i++) {
		1950	if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
		1951	DMIF_BUFFERS_ALLOCATED_COMPLETED)
		1952	break;
		1953	udelay(1);
		1954	}
		1955
2997	Serge	1956	if (radeon_crtc->base.enabled && mode) {
		1957	switch (tmp) {
		1958	case 0:
		1959	default:
		1960	return 4096 * 2;
		1961	case 2:
		1962	return 8192 * 2;
		1963	}
		1964	}
		1965
		1966	/* controller not enabled, so no lb used */
		1967	return 0;
		1968	}
		1969
		1970	static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
		1971	{
		1972	u32 tmp = RREG32(MC_SHARED_CHMAP);
		1973
		1974	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		1975	case 0:
		1976	default:
		1977	return 1;
		1978	case 1:
		1979	return 2;
		1980	case 2:
		1981	return 4;
		1982	case 3:
		1983	return 8;
		1984	case 4:
		1985	return 3;
		1986	case 5:
		1987	return 6;
		1988	case 6:
		1989	return 10;
		1990	case 7:
		1991	return 12;
		1992	case 8:
		1993	return 16;
		1994	}
		1995	}
		1996
		1997	struct dce6_wm_params {
		1998	u32 dram_channels; /* number of dram channels */
		1999	u32 yclk; /* bandwidth per dram data pin in kHz */
		2000	u32 sclk; /* engine clock in kHz */
		2001	u32 disp_clk; /* display clock in kHz */
		2002	u32 src_width; /* viewport width */
		2003	u32 active_time; /* active display time in ns */
		2004	u32 blank_time; /* blank time in ns */
		2005	bool interlaced; /* mode is interlaced */
		2006	fixed20_12 vsc; /* vertical scale ratio */
		2007	u32 num_heads; /* number of active crtcs */
		2008	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
		2009	u32 lb_size; /* line buffer allocated to pipe */
		2010	u32 vtaps; /* vertical scaler taps */
		2011	};
		2012
		2013	static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
		2014	{
		2015	/* Calculate raw DRAM Bandwidth */
		2016	fixed20_12 dram_efficiency; /* 0.7 */
		2017	fixed20_12 yclk, dram_channels, bandwidth;
		2018	fixed20_12 a;
		2019
		2020	a.full = dfixed_const(1000);
		2021	yclk.full = dfixed_const(wm->yclk);
		2022	yclk.full = dfixed_div(yclk, a);
		2023	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		2024	a.full = dfixed_const(10);
		2025	dram_efficiency.full = dfixed_const(7);
		2026	dram_efficiency.full = dfixed_div(dram_efficiency, a);
		2027	bandwidth.full = dfixed_mul(dram_channels, yclk);
		2028	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
		2029
		2030	return dfixed_trunc(bandwidth);
		2031	}
		2032
		2033	static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
		2034	{
		2035	/* Calculate DRAM Bandwidth and the part allocated to display. */
		2036	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
		2037	fixed20_12 yclk, dram_channels, bandwidth;
		2038	fixed20_12 a;
		2039
		2040	a.full = dfixed_const(1000);
		2041	yclk.full = dfixed_const(wm->yclk);
		2042	yclk.full = dfixed_div(yclk, a);
		2043	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		2044	a.full = dfixed_const(10);
		2045	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
		2046	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
		2047	bandwidth.full = dfixed_mul(dram_channels, yclk);
		2048	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
		2049
		2050	return dfixed_trunc(bandwidth);
		2051	}
		2052
		2053	static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
		2054	{
		2055	/* Calculate the display Data return Bandwidth */
		2056	fixed20_12 return_efficiency; /* 0.8 */
		2057	fixed20_12 sclk, bandwidth;
		2058	fixed20_12 a;
		2059
		2060	a.full = dfixed_const(1000);
		2061	sclk.full = dfixed_const(wm->sclk);
		2062	sclk.full = dfixed_div(sclk, a);
		2063	a.full = dfixed_const(10);
		2064	return_efficiency.full = dfixed_const(8);
		2065	return_efficiency.full = dfixed_div(return_efficiency, a);
		2066	a.full = dfixed_const(32);
		2067	bandwidth.full = dfixed_mul(a, sclk);
		2068	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
		2069
		2070	return dfixed_trunc(bandwidth);
		2071	}
		2072
		2073	static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
		2074	{
		2075	return 32;
		2076	}
		2077
		2078	static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
		2079	{
		2080	/* Calculate the DMIF Request Bandwidth */
		2081	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
		2082	fixed20_12 disp_clk, sclk, bandwidth;
		2083	fixed20_12 a, b1, b2;
		2084	u32 min_bandwidth;
		2085
		2086	a.full = dfixed_const(1000);
		2087	disp_clk.full = dfixed_const(wm->disp_clk);
		2088	disp_clk.full = dfixed_div(disp_clk, a);
		2089	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
		2090	b1.full = dfixed_mul(a, disp_clk);
		2091
		2092	a.full = dfixed_const(1000);
		2093	sclk.full = dfixed_const(wm->sclk);
		2094	sclk.full = dfixed_div(sclk, a);
		2095	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
		2096	b2.full = dfixed_mul(a, sclk);
		2097
		2098	a.full = dfixed_const(10);
		2099	disp_clk_request_efficiency.full = dfixed_const(8);
		2100	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
		2101
		2102	min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
		2103
		2104	a.full = dfixed_const(min_bandwidth);
		2105	bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
		2106
		2107	return dfixed_trunc(bandwidth);
		2108	}
		2109
		2110	static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
		2111	{
		2112	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
		2113	u32 dram_bandwidth = dce6_dram_bandwidth(wm);
		2114	u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
		2115	u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
		2116
		2117	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
		2118	}
		2119
		2120	static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
		2121	{
		2122	/* Calculate the display mode Average Bandwidth
		2123	* DisplayMode should contain the source and destination dimensions,
		2124	* timing, etc.
		2125	*/
		2126	fixed20_12 bpp;
		2127	fixed20_12 line_time;
		2128	fixed20_12 src_width;
		2129	fixed20_12 bandwidth;
		2130	fixed20_12 a;
		2131
		2132	a.full = dfixed_const(1000);
		2133	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
		2134	line_time.full = dfixed_div(line_time, a);
		2135	bpp.full = dfixed_const(wm->bytes_per_pixel);
		2136	src_width.full = dfixed_const(wm->src_width);
		2137	bandwidth.full = dfixed_mul(src_width, bpp);
		2138	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
		2139	bandwidth.full = dfixed_div(bandwidth, line_time);
		2140
		2141	return dfixed_trunc(bandwidth);
		2142	}
		2143
		2144	static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
		2145	{
		2146	/* First calcualte the latency in ns */
		2147	u32 mc_latency = 2000; /* 2000 ns. */
		2148	u32 available_bandwidth = dce6_available_bandwidth(wm);
		2149	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
		2150	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
		2151	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
		2152	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
		2153	(wm->num_heads * cursor_line_pair_return_time);
		2154	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
		2155	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
		2156	u32 tmp, dmif_size = 12288;
		2157	fixed20_12 a, b, c;
		2158
		2159	if (wm->num_heads == 0)
		2160	return 0;
		2161
		2162	a.full = dfixed_const(2);
		2163	b.full = dfixed_const(1);
		2164	if ((wm->vsc.full > a.full) \|\|
		2165	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) \|\|
		2166	(wm->vtaps >= 5) \|\|
		2167	((wm->vsc.full >= a.full) && wm->interlaced))
		2168	max_src_lines_per_dst_line = 4;
		2169	else
		2170	max_src_lines_per_dst_line = 2;
		2171
		2172	a.full = dfixed_const(available_bandwidth);
		2173	b.full = dfixed_const(wm->num_heads);
		2174	a.full = dfixed_div(a, b);
		2175
		2176	b.full = dfixed_const(mc_latency + 512);
		2177	c.full = dfixed_const(wm->disp_clk);
		2178	b.full = dfixed_div(b, c);
		2179
		2180	c.full = dfixed_const(dmif_size);
		2181	b.full = dfixed_div(c, b);
		2182
		2183	tmp = min(dfixed_trunc(a), dfixed_trunc(b));
		2184
		2185	b.full = dfixed_const(1000);
		2186	c.full = dfixed_const(wm->disp_clk);
		2187	b.full = dfixed_div(c, b);
		2188	c.full = dfixed_const(wm->bytes_per_pixel);
		2189	b.full = dfixed_mul(b, c);
		2190
		2191	lb_fill_bw = min(tmp, dfixed_trunc(b));
		2192
		2193	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
		2194	b.full = dfixed_const(1000);
		2195	c.full = dfixed_const(lb_fill_bw);
		2196	b.full = dfixed_div(c, b);
		2197	a.full = dfixed_div(a, b);
		2198	line_fill_time = dfixed_trunc(a);
		2199
		2200	if (line_fill_time < wm->active_time)
		2201	return latency;
		2202	else
		2203	return latency + (line_fill_time - wm->active_time);
		2204
		2205	}
		2206
		2207	static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
		2208	{
		2209	if (dce6_average_bandwidth(wm) <=
		2210	(dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
		2211	return true;
		2212	else
		2213	return false;
		2214	};
		2215
		2216	static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
		2217	{
		2218	if (dce6_average_bandwidth(wm) <=
		2219	(dce6_available_bandwidth(wm) / wm->num_heads))
		2220	return true;
		2221	else
		2222	return false;
		2223	};
		2224
		2225	static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
		2226	{
		2227	u32 lb_partitions = wm->lb_size / wm->src_width;
		2228	u32 line_time = wm->active_time + wm->blank_time;
		2229	u32 latency_tolerant_lines;
		2230	u32 latency_hiding;
		2231	fixed20_12 a;
		2232
		2233	a.full = dfixed_const(1);
		2234	if (wm->vsc.full > a.full)
		2235	latency_tolerant_lines = 1;
		2236	else {
		2237	if (lb_partitions <= (wm->vtaps + 1))
		2238	latency_tolerant_lines = 1;
		2239	else
		2240	latency_tolerant_lines = 2;
		2241	}
		2242
		2243	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
		2244
		2245	if (dce6_latency_watermark(wm) <= latency_hiding)
		2246	return true;
		2247	else
		2248	return false;
		2249	}
		2250
		2251	static void dce6_program_watermarks(struct radeon_device *rdev,
		2252	struct radeon_crtc *radeon_crtc,
		2253	u32 lb_size, u32 num_heads)
		2254	{
		2255	struct drm_display_mode *mode = &radeon_crtc->base.mode;
5078	serge	2256	struct dce6_wm_params wm_low, wm_high;
		2257	u32 dram_channels;
2997	Serge	2258	u32 pixel_period;
		2259	u32 line_time = 0;
		2260	u32 latency_watermark_a = 0, latency_watermark_b = 0;
		2261	u32 priority_a_mark = 0, priority_b_mark = 0;
		2262	u32 priority_a_cnt = PRIORITY_OFF;
		2263	u32 priority_b_cnt = PRIORITY_OFF;
		2264	u32 tmp, arb_control3;
		2265	fixed20_12 a, b, c;
		2266
		2267	if (radeon_crtc->base.enabled && num_heads && mode) {
		2268	pixel_period = 1000000 / (u32)mode->clock;
		2269	line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
		2270	priority_a_cnt = 0;
		2271	priority_b_cnt = 0;
		2272
		2273	if (rdev->family == CHIP_ARUBA)
5078	serge	2274	dram_channels = evergreen_get_number_of_dram_channels(rdev);
2997	Serge	2275	else
5078	serge	2276	dram_channels = si_get_number_of_dram_channels(rdev);
2997	Serge	2277
5078	serge	2278	/* watermark for high clocks */
		2279	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
		2280	wm_high.yclk =
		2281	radeon_dpm_get_mclk(rdev, false) * 10;
		2282	wm_high.sclk =
		2283	radeon_dpm_get_sclk(rdev, false) * 10;
		2284	} else {
		2285	wm_high.yclk = rdev->pm.current_mclk * 10;
		2286	wm_high.sclk = rdev->pm.current_sclk * 10;
		2287	}
		2288
		2289	wm_high.disp_clk = mode->clock;
		2290	wm_high.src_width = mode->crtc_hdisplay;
		2291	wm_high.active_time = mode->crtc_hdisplay * pixel_period;
		2292	wm_high.blank_time = line_time - wm_high.active_time;
		2293	wm_high.interlaced = false;
		2294	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		2295	wm_high.interlaced = true;
		2296	wm_high.vsc = radeon_crtc->vsc;
		2297	wm_high.vtaps = 1;
		2298	if (radeon_crtc->rmx_type != RMX_OFF)
		2299	wm_high.vtaps = 2;
		2300	wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
		2301	wm_high.lb_size = lb_size;
		2302	wm_high.dram_channels = dram_channels;
		2303	wm_high.num_heads = num_heads;
		2304
		2305	/* watermark for low clocks */
		2306	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
		2307	wm_low.yclk =
		2308	radeon_dpm_get_mclk(rdev, true) * 10;
		2309	wm_low.sclk =
		2310	radeon_dpm_get_sclk(rdev, true) * 10;
		2311	} else {
		2312	wm_low.yclk = rdev->pm.current_mclk * 10;
		2313	wm_low.sclk = rdev->pm.current_sclk * 10;
		2314	}
		2315
		2316	wm_low.disp_clk = mode->clock;
		2317	wm_low.src_width = mode->crtc_hdisplay;
		2318	wm_low.active_time = mode->crtc_hdisplay * pixel_period;
		2319	wm_low.blank_time = line_time - wm_low.active_time;
		2320	wm_low.interlaced = false;
		2321	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		2322	wm_low.interlaced = true;
		2323	wm_low.vsc = radeon_crtc->vsc;
		2324	wm_low.vtaps = 1;
		2325	if (radeon_crtc->rmx_type != RMX_OFF)
		2326	wm_low.vtaps = 2;
		2327	wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
		2328	wm_low.lb_size = lb_size;
		2329	wm_low.dram_channels = dram_channels;
		2330	wm_low.num_heads = num_heads;
		2331
2997	Serge	2332	/* set for high clocks */
5078	serge	2333	latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2997	Serge	2334	/* set for low clocks */
5078	serge	2335	latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2997	Serge	2336
		2337	/* possibly force display priority to high */
		2338	/* should really do this at mode validation time... */
5078	serge	2339	if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) \|\|
		2340	!dce6_average_bandwidth_vs_available_bandwidth(&wm_high) \|\|
		2341	!dce6_check_latency_hiding(&wm_high) \|\|
2997	Serge	2342	(rdev->disp_priority == 2)) {
		2343	DRM_DEBUG_KMS("force priority to high\n");
		2344	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		2345	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		2346	}
5078	serge	2347	if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) \|\|
		2348	!dce6_average_bandwidth_vs_available_bandwidth(&wm_low) \|\|
		2349	!dce6_check_latency_hiding(&wm_low) \|\|
		2350	(rdev->disp_priority == 2)) {
		2351	DRM_DEBUG_KMS("force priority to high\n");
		2352	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		2353	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		2354	}
2997	Serge	2355
		2356	a.full = dfixed_const(1000);
		2357	b.full = dfixed_const(mode->clock);
		2358	b.full = dfixed_div(b, a);
		2359	c.full = dfixed_const(latency_watermark_a);
		2360	c.full = dfixed_mul(c, b);
		2361	c.full = dfixed_mul(c, radeon_crtc->hsc);
		2362	c.full = dfixed_div(c, a);
		2363	a.full = dfixed_const(16);
		2364	c.full = dfixed_div(c, a);
		2365	priority_a_mark = dfixed_trunc(c);
		2366	priority_a_cnt \|= priority_a_mark & PRIORITY_MARK_MASK;
		2367
		2368	a.full = dfixed_const(1000);
		2369	b.full = dfixed_const(mode->clock);
		2370	b.full = dfixed_div(b, a);
		2371	c.full = dfixed_const(latency_watermark_b);
		2372	c.full = dfixed_mul(c, b);
		2373	c.full = dfixed_mul(c, radeon_crtc->hsc);
		2374	c.full = dfixed_div(c, a);
		2375	a.full = dfixed_const(16);
		2376	c.full = dfixed_div(c, a);
		2377	priority_b_mark = dfixed_trunc(c);
		2378	priority_b_cnt \|= priority_b_mark & PRIORITY_MARK_MASK;
6104	serge	2379
		2380	/* Save number of lines the linebuffer leads before the scanout */
		2381	radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2997	Serge	2382	}
		2383
		2384	/* select wm A */
		2385	arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
		2386	tmp = arb_control3;
		2387	tmp &= ~LATENCY_WATERMARK_MASK(3);
		2388	tmp \|= LATENCY_WATERMARK_MASK(1);
		2389	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
		2390	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
		2391	(LATENCY_LOW_WATERMARK(latency_watermark_a) \|
		2392	LATENCY_HIGH_WATERMARK(line_time)));
		2393	/* select wm B */
		2394	tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
		2395	tmp &= ~LATENCY_WATERMARK_MASK(3);
		2396	tmp \|= LATENCY_WATERMARK_MASK(2);
		2397	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
		2398	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
		2399	(LATENCY_LOW_WATERMARK(latency_watermark_b) \|
		2400	LATENCY_HIGH_WATERMARK(line_time)));
		2401	/* restore original selection */
		2402	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
		2403
		2404	/* write the priority marks */
		2405	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
		2406	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
		2407
5078	serge	2408	/* save values for DPM */
		2409	radeon_crtc->line_time = line_time;
		2410	radeon_crtc->wm_high = latency_watermark_a;
		2411	radeon_crtc->wm_low = latency_watermark_b;
2997	Serge	2412	}
		2413
		2414	void dce6_bandwidth_update(struct radeon_device *rdev)
		2415	{
		2416	struct drm_display_mode *mode0 = NULL;
		2417	struct drm_display_mode *mode1 = NULL;
		2418	u32 num_heads = 0, lb_size;
		2419	int i;
		2420
5271	serge	2421	if (!rdev->mode_info.mode_config_initialized)
		2422	return;
		2423
2997	Serge	2424	radeon_update_display_priority(rdev);
		2425
		2426	for (i = 0; i < rdev->num_crtc; i++) {
		2427	if (rdev->mode_info.crtcs[i]->base.enabled)
		2428	num_heads++;
		2429	}
		2430	for (i = 0; i < rdev->num_crtc; i += 2) {
		2431	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
		2432	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
		2433	lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
		2434	dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
		2435	lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
		2436	dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
		2437	}
		2438	}
		2439
		2440	/*
		2441	* Core functions
		2442	*/
		2443	static void si_tiling_mode_table_init(struct radeon_device *rdev)
		2444	{
		2445	const u32 num_tile_mode_states = 32;
		2446	u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
		2447
		2448	switch (rdev->config.si.mem_row_size_in_kb) {
		2449	case 1:
		2450	split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
		2451	break;
		2452	case 2:
		2453	default:
		2454	split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
		2455	break;
		2456	case 4:
		2457	split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
		2458	break;
		2459	}
		2460
		2461	if ((rdev->family == CHIP_TAHITI) \|\|
		2462	(rdev->family == CHIP_PITCAIRN)) {
		2463	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
		2464	switch (reg_offset) {
		2465	case 0: /* non-AA compressed depth or any compressed stencil */
		2466	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2467	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2468	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2469	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2470	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2471	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2472	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2473	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2474	break;
		2475	case 1: /* 2xAA/4xAA compressed depth only */
		2476	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2477	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2478	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2479	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		2480	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2481	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2482	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2483	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2484	break;
		2485	case 2: /* 8xAA compressed depth only */
		2486	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2487	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2488	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2489	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2490	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2491	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2492	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2493	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2494	break;
		2495	case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
		2496	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2497	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2498	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2499	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		2500	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2501	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2502	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2503	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2504	break;
		2505	case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
		2506	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2507	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2508	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2509	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2510	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2511	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2512	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2513	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2514	break;
		2515	case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
		2516	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2517	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2518	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2519	TILE_SPLIT(split_equal_to_row_size) \|
		2520	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2521	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2522	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2523	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2524	break;
		2525	case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
		2526	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2527	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2528	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2529	TILE_SPLIT(split_equal_to_row_size) \|
		2530	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2531	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2532	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2533	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2534	break;
		2535	case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
		2536	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2537	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2538	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2539	TILE_SPLIT(split_equal_to_row_size) \|
		2540	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2541	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2542	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2543	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2544	break;
		2545	case 8: /* 1D and 1D Array Surfaces */
		2546	gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) \|
		2547	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2548	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2549	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2550	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2551	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2552	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2553	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2554	break;
		2555	case 9: /* Displayable maps. */
		2556	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2557	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2558	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2559	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2560	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2561	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2562	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2563	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2564	break;
		2565	case 10: /* Display 8bpp. */
		2566	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2567	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2568	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2569	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2570	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2571	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2572	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2573	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2574	break;
		2575	case 11: /* Display 16bpp. */
		2576	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2577	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2578	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2579	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2580	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2581	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2582	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2583	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2584	break;
		2585	case 12: /* Display 32bpp. */
		2586	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2587	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2588	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2589	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2590	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2591	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2592	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2593	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2594	break;
		2595	case 13: /* Thin. */
		2596	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2597	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2598	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2599	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2600	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2601	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2602	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2603	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2604	break;
		2605	case 14: /* Thin 8 bpp. */
		2606	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2607	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2608	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2609	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2610	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2611	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2612	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2613	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2614	break;
		2615	case 15: /* Thin 16 bpp. */
		2616	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2617	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2618	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2619	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2620	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2621	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2622	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2623	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2624	break;
		2625	case 16: /* Thin 32 bpp. */
		2626	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2627	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2628	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2629	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2630	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2631	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2632	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2633	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2634	break;
		2635	case 17: /* Thin 64 bpp. */
		2636	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2637	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2638	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2639	TILE_SPLIT(split_equal_to_row_size) \|
		2640	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2641	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2642	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2643	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2644	break;
		2645	case 21: /* 8 bpp PRT. */
		2646	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2647	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2648	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2649	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2650	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2651	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) \|
		2652	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2653	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2654	break;
		2655	case 22: /* 16 bpp PRT */
		2656	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2657	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2658	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2659	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2660	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2661	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2662	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2663	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2664	break;
		2665	case 23: /* 32 bpp PRT */
		2666	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2667	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2668	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2669	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2670	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2671	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2672	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2673	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2674	break;
		2675	case 24: /* 64 bpp PRT */
		2676	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2677	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2678	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2679	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2680	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2681	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2682	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2683	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2684	break;
		2685	case 25: /* 128 bpp PRT */
		2686	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2687	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2688	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2689	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) \|
		2690	NUM_BANKS(ADDR_SURF_8_BANK) \|
		2691	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2692	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2693	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2694	break;
		2695	default:
		2696	gb_tile_moden = 0;
		2697	break;
		2698	}
3764	Serge	2699	rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2997	Serge	2700	WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
		2701	}
3764	Serge	2702	} else if ((rdev->family == CHIP_VERDE) \|\|
		2703	(rdev->family == CHIP_OLAND) \|\|
		2704	(rdev->family == CHIP_HAINAN)) {
2997	Serge	2705	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
		2706	switch (reg_offset) {
		2707	case 0: /* non-AA compressed depth or any compressed stencil */
		2708	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2709	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2710	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2711	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2712	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2713	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2714	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2715	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2716	break;
		2717	case 1: /* 2xAA/4xAA compressed depth only */
		2718	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2719	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2720	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2721	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		2722	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2723	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2724	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2725	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2726	break;
		2727	case 2: /* 8xAA compressed depth only */
		2728	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2729	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2730	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2731	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2732	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2733	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2734	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2735	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2736	break;
		2737	case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
		2738	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2739	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2740	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2741	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		2742	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2743	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2744	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2745	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2746	break;
		2747	case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
		2748	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2749	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2750	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2751	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2752	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2753	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2754	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2755	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2756	break;
		2757	case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
		2758	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2759	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2760	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2761	TILE_SPLIT(split_equal_to_row_size) \|
		2762	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2763	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2764	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2765	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2766	break;
		2767	case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
		2768	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2769	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2770	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2771	TILE_SPLIT(split_equal_to_row_size) \|
		2772	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2773	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2774	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2775	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2776	break;
		2777	case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
		2778	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2779	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2780	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2781	TILE_SPLIT(split_equal_to_row_size) \|
		2782	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2783	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2784	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2785	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2786	break;
		2787	case 8: /* 1D and 1D Array Surfaces */
		2788	gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) \|
		2789	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2790	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2791	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2792	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2793	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2794	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2795	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2796	break;
		2797	case 9: /* Displayable maps. */
		2798	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2799	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2800	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2801	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2802	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2803	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2804	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2805	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2806	break;
		2807	case 10: /* Display 8bpp. */
		2808	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2809	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2810	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2811	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2812	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2813	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2814	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2815	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2816	break;
		2817	case 11: /* Display 16bpp. */
		2818	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2819	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2820	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2821	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2822	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2823	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2824	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2825	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2826	break;
		2827	case 12: /* Display 32bpp. */
		2828	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2829	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2830	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2831	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2832	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2833	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2834	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2835	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2836	break;
		2837	case 13: /* Thin. */
		2838	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2839	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2840	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2841	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2842	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2843	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2844	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2845	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2846	break;
		2847	case 14: /* Thin 8 bpp. */
		2848	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2849	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2850	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2851	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2852	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2853	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2854	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2855	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2856	break;
		2857	case 15: /* Thin 16 bpp. */
		2858	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2859	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2860	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2861	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2862	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2863	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2864	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2865	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2866	break;
		2867	case 16: /* Thin 32 bpp. */
		2868	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2869	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2870	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2871	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2872	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2873	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2874	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2875	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2876	break;
		2877	case 17: /* Thin 64 bpp. */
		2878	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2879	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2880	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2881	TILE_SPLIT(split_equal_to_row_size) \|
		2882	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2883	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2884	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2885	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2886	break;
		2887	case 21: /* 8 bpp PRT. */
		2888	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2889	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2890	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2891	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2892	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2893	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) \|
		2894	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2895	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2896	break;
		2897	case 22: /* 16 bpp PRT */
		2898	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2899	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2900	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2901	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2902	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2903	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2904	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2905	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2906	break;
		2907	case 23: /* 32 bpp PRT */
		2908	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2909	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2910	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2911	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2912	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2913	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2914	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2915	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2916	break;
		2917	case 24: /* 64 bpp PRT */
		2918	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2919	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2920	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2921	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2922	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2923	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2924	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2925	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2926	break;
		2927	case 25: /* 128 bpp PRT */
		2928	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2929	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2930	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2931	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) \|
		2932	NUM_BANKS(ADDR_SURF_8_BANK) \|
		2933	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2934	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2935	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2936	break;
		2937	default:
		2938	gb_tile_moden = 0;
		2939	break;
		2940	}
3764	Serge	2941	rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2997	Serge	2942	WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
		2943	}
		2944	} else
		2945	DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
		2946	}
		2947
		2948	static void si_select_se_sh(struct radeon_device *rdev,
		2949	u32 se_num, u32 sh_num)
		2950	{
		2951	u32 data = INSTANCE_BROADCAST_WRITES;
		2952
		2953	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3764	Serge	2954	data \|= SH_BROADCAST_WRITES \| SE_BROADCAST_WRITES;
2997	Serge	2955	else if (se_num == 0xffffffff)
		2956	data \|= SE_BROADCAST_WRITES \| SH_INDEX(sh_num);
		2957	else if (sh_num == 0xffffffff)
		2958	data \|= SH_BROADCAST_WRITES \| SE_INDEX(se_num);
		2959	else
		2960	data \|= SH_INDEX(sh_num) \| SE_INDEX(se_num);
		2961	WREG32(GRBM_GFX_INDEX, data);
		2962	}
		2963
		2964	static u32 si_create_bitmask(u32 bit_width)
		2965	{
		2966	u32 i, mask = 0;
		2967
		2968	for (i = 0; i < bit_width; i++) {
		2969	mask <<= 1;
		2970	mask \|= 1;
		2971	}
		2972	return mask;
		2973	}
		2974
		2975	static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
		2976	{
		2977	u32 data, mask;
		2978
		2979	data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
		2980	if (data & 1)
		2981	data &= INACTIVE_CUS_MASK;
		2982	else
		2983	data = 0;
		2984	data \|= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
		2985
		2986	data >>= INACTIVE_CUS_SHIFT;
		2987
		2988	mask = si_create_bitmask(cu_per_sh);
		2989
		2990	return ~data & mask;
		2991	}
		2992
		2993	static void si_setup_spi(struct radeon_device *rdev,
		2994	u32 se_num, u32 sh_per_se,
		2995	u32 cu_per_sh)
		2996	{
		2997	int i, j, k;
		2998	u32 data, mask, active_cu;
		2999
		3000	for (i = 0; i < se_num; i++) {
		3001	for (j = 0; j < sh_per_se; j++) {
		3002	si_select_se_sh(rdev, i, j);
		3003	data = RREG32(SPI_STATIC_THREAD_MGMT_3);
		3004	active_cu = si_get_cu_enabled(rdev, cu_per_sh);
		3005
		3006	mask = 1;
		3007	for (k = 0; k < 16; k++) {
		3008	mask <<= k;
		3009	if (active_cu & mask) {
		3010	data &= ~mask;
		3011	WREG32(SPI_STATIC_THREAD_MGMT_3, data);
		3012	break;
		3013	}
		3014	}
		3015	}
		3016	}
		3017	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		3018	}
		3019
		3020	static u32 si_get_rb_disabled(struct radeon_device *rdev,
5078	serge	3021	u32 max_rb_num_per_se,
2997	Serge	3022	u32 sh_per_se)
		3023	{
		3024	u32 data, mask;
		3025
		3026	data = RREG32(CC_RB_BACKEND_DISABLE);
		3027	if (data & 1)
		3028	data &= BACKEND_DISABLE_MASK;
		3029	else
		3030	data = 0;
		3031	data \|= RREG32(GC_USER_RB_BACKEND_DISABLE);
		3032
		3033	data >>= BACKEND_DISABLE_SHIFT;
		3034
5078	serge	3035	mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2997	Serge	3036
		3037	return data & mask;
		3038	}
		3039
		3040	static void si_setup_rb(struct radeon_device *rdev,
		3041	u32 se_num, u32 sh_per_se,
5078	serge	3042	u32 max_rb_num_per_se)
2997	Serge	3043	{
		3044	int i, j;
		3045	u32 data, mask;
		3046	u32 disabled_rbs = 0;
		3047	u32 enabled_rbs = 0;
		3048
		3049	for (i = 0; i < se_num; i++) {
		3050	for (j = 0; j < sh_per_se; j++) {
		3051	si_select_se_sh(rdev, i, j);
5078	serge	3052	data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
2997	Serge	3053	disabled_rbs \|= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
		3054	}
		3055	}
		3056	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		3057
		3058	mask = 1;
5078	serge	3059	for (i = 0; i < max_rb_num_per_se * se_num; i++) {
2997	Serge	3060	if (!(disabled_rbs & mask))
		3061	enabled_rbs \|= mask;
		3062	mask <<= 1;
		3063	}
		3064
5078	serge	3065	rdev->config.si.backend_enable_mask = enabled_rbs;
		3066
2997	Serge	3067	for (i = 0; i < se_num; i++) {
		3068	si_select_se_sh(rdev, i, 0xffffffff);
		3069	data = 0;
		3070	for (j = 0; j < sh_per_se; j++) {
		3071	switch (enabled_rbs & 3) {
		3072	case 1:
		3073	data \|= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
		3074	break;
		3075	case 2:
		3076	data \|= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
		3077	break;
		3078	case 3:
		3079	default:
		3080	data \|= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
		3081	break;
		3082	}
		3083	enabled_rbs >>= 2;
		3084	}
		3085	WREG32(PA_SC_RASTER_CONFIG, data);
		3086	}
		3087	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		3088	}
		3089
		3090	static void si_gpu_init(struct radeon_device *rdev)
		3091	{
		3092	u32 gb_addr_config = 0;
		3093	u32 mc_shared_chmap, mc_arb_ramcfg;
		3094	u32 sx_debug_1;
		3095	u32 hdp_host_path_cntl;
		3096	u32 tmp;
		3097	int i, j;
		3098
		3099	switch (rdev->family) {
		3100	case CHIP_TAHITI:
		3101	rdev->config.si.max_shader_engines = 2;
		3102	rdev->config.si.max_tile_pipes = 12;
		3103	rdev->config.si.max_cu_per_sh = 8;
		3104	rdev->config.si.max_sh_per_se = 2;
		3105	rdev->config.si.max_backends_per_se = 4;
		3106	rdev->config.si.max_texture_channel_caches = 12;
		3107	rdev->config.si.max_gprs = 256;
		3108	rdev->config.si.max_gs_threads = 32;
		3109	rdev->config.si.max_hw_contexts = 8;
		3110
		3111	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		3112	rdev->config.si.sc_prim_fifo_size_backend = 0x100;
		3113	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		3114	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		3115	gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
		3116	break;
		3117	case CHIP_PITCAIRN:
		3118	rdev->config.si.max_shader_engines = 2;
		3119	rdev->config.si.max_tile_pipes = 8;
		3120	rdev->config.si.max_cu_per_sh = 5;
		3121	rdev->config.si.max_sh_per_se = 2;
		3122	rdev->config.si.max_backends_per_se = 4;
		3123	rdev->config.si.max_texture_channel_caches = 8;
		3124	rdev->config.si.max_gprs = 256;
		3125	rdev->config.si.max_gs_threads = 32;
		3126	rdev->config.si.max_hw_contexts = 8;
		3127
		3128	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		3129	rdev->config.si.sc_prim_fifo_size_backend = 0x100;
		3130	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		3131	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		3132	gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
		3133	break;
		3134	case CHIP_VERDE:
		3135	default:
		3136	rdev->config.si.max_shader_engines = 1;
		3137	rdev->config.si.max_tile_pipes = 4;
3764	Serge	3138	rdev->config.si.max_cu_per_sh = 5;
2997	Serge	3139	rdev->config.si.max_sh_per_se = 2;
		3140	rdev->config.si.max_backends_per_se = 4;
		3141	rdev->config.si.max_texture_channel_caches = 4;
		3142	rdev->config.si.max_gprs = 256;
		3143	rdev->config.si.max_gs_threads = 32;
		3144	rdev->config.si.max_hw_contexts = 8;
		3145
		3146	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		3147	rdev->config.si.sc_prim_fifo_size_backend = 0x40;
		3148	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		3149	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		3150	gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
		3151	break;
3764	Serge	3152	case CHIP_OLAND:
		3153	rdev->config.si.max_shader_engines = 1;
		3154	rdev->config.si.max_tile_pipes = 4;
		3155	rdev->config.si.max_cu_per_sh = 6;
		3156	rdev->config.si.max_sh_per_se = 1;
		3157	rdev->config.si.max_backends_per_se = 2;
		3158	rdev->config.si.max_texture_channel_caches = 4;
		3159	rdev->config.si.max_gprs = 256;
		3160	rdev->config.si.max_gs_threads = 16;
		3161	rdev->config.si.max_hw_contexts = 8;
		3162
		3163	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		3164	rdev->config.si.sc_prim_fifo_size_backend = 0x40;
		3165	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		3166	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		3167	gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
		3168	break;
		3169	case CHIP_HAINAN:
		3170	rdev->config.si.max_shader_engines = 1;
		3171	rdev->config.si.max_tile_pipes = 4;
		3172	rdev->config.si.max_cu_per_sh = 5;
		3173	rdev->config.si.max_sh_per_se = 1;
		3174	rdev->config.si.max_backends_per_se = 1;
		3175	rdev->config.si.max_texture_channel_caches = 2;
		3176	rdev->config.si.max_gprs = 256;
		3177	rdev->config.si.max_gs_threads = 16;
		3178	rdev->config.si.max_hw_contexts = 8;
		3179
		3180	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		3181	rdev->config.si.sc_prim_fifo_size_backend = 0x40;
		3182	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		3183	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		3184	gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
		3185	break;
2997	Serge	3186	}
		3187
		3188	/* Initialize HDP */
		3189	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		3190	WREG32((0x2c14 + j), 0x00000000);
		3191	WREG32((0x2c18 + j), 0x00000000);
		3192	WREG32((0x2c1c + j), 0x00000000);
		3193	WREG32((0x2c20 + j), 0x00000000);
		3194	WREG32((0x2c24 + j), 0x00000000);
		3195	}
		3196
		3197	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
6104	serge	3198	WREG32(SRBM_INT_CNTL, 1);
		3199	WREG32(SRBM_INT_ACK, 1);
2997	Serge	3200
		3201	evergreen_fix_pci_max_read_req_size(rdev);
		3202
		3203	WREG32(BIF_FB_EN, FB_READ_EN \| FB_WRITE_EN);
		3204
		3205	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
		3206	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
		3207
		3208	rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
		3209	rdev->config.si.mem_max_burst_length_bytes = 256;
		3210	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
		3211	rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
		3212	if (rdev->config.si.mem_row_size_in_kb > 4)
		3213	rdev->config.si.mem_row_size_in_kb = 4;
		3214	/* XXX use MC settings? */
		3215	rdev->config.si.shader_engine_tile_size = 32;
		3216	rdev->config.si.num_gpus = 1;
		3217	rdev->config.si.multi_gpu_tile_size = 64;
		3218
		3219	/* fix up row size */
		3220	gb_addr_config &= ~ROW_SIZE_MASK;
		3221	switch (rdev->config.si.mem_row_size_in_kb) {
		3222	case 1:
		3223	default:
		3224	gb_addr_config \|= ROW_SIZE(0);
		3225	break;
		3226	case 2:
		3227	gb_addr_config \|= ROW_SIZE(1);
		3228	break;
		3229	case 4:
		3230	gb_addr_config \|= ROW_SIZE(2);
		3231	break;
		3232	}
		3233
		3234	/* setup tiling info dword. gb_addr_config is not adequate since it does
		3235	* not have bank info, so create a custom tiling dword.
		3236	* bits 3:0 num_pipes
		3237	* bits 7:4 num_banks
		3238	* bits 11:8 group_size
		3239	* bits 15:12 row_size
		3240	*/
		3241	rdev->config.si.tile_config = 0;
		3242	switch (rdev->config.si.num_tile_pipes) {
		3243	case 1:
		3244	rdev->config.si.tile_config \|= (0 << 0);
		3245	break;
		3246	case 2:
		3247	rdev->config.si.tile_config \|= (1 << 0);
		3248	break;
		3249	case 4:
		3250	rdev->config.si.tile_config \|= (2 << 0);
		3251	break;
		3252	case 8:
		3253	default:
		3254	/* XXX what about 12? */
		3255	rdev->config.si.tile_config \|= (3 << 0);
		3256	break;
6104	serge	3257	}
2997	Serge	3258	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
		3259	case 0: /* four banks */
		3260	rdev->config.si.tile_config \|= 0 << 4;
		3261	break;
		3262	case 1: /* eight banks */
		3263	rdev->config.si.tile_config \|= 1 << 4;
		3264	break;
		3265	case 2: /* sixteen banks */
		3266	default:
		3267	rdev->config.si.tile_config \|= 2 << 4;
		3268	break;
		3269	}
		3270	rdev->config.si.tile_config \|=
		3271	((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
		3272	rdev->config.si.tile_config \|=
		3273	((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
		3274
		3275	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		3276	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3764	Serge	3277	WREG32(DMIF_ADDR_CALC, gb_addr_config);
2997	Serge	3278	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3192	Serge	3279	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
		3280	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3764	Serge	3281	if (rdev->has_uvd) {
		3282	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
		3283	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
		3284	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
		3285	}
2997	Serge	3286
		3287	si_tiling_mode_table_init(rdev);
		3288
		3289	si_setup_rb(rdev, rdev->config.si.max_shader_engines,
		3290	rdev->config.si.max_sh_per_se,
		3291	rdev->config.si.max_backends_per_se);
		3292
		3293	si_setup_spi(rdev, rdev->config.si.max_shader_engines,
		3294	rdev->config.si.max_sh_per_se,
		3295	rdev->config.si.max_cu_per_sh);
		3296
5078	serge	3297	rdev->config.si.active_cus = 0;
		3298	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
		3299	for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
6104	serge	3300	rdev->config.si.active_cus +=
		3301	hweight32(si_get_cu_active_bitmap(rdev, i, j));
5078	serge	3302	}
6104	serge	3303	}
2997	Serge	3304
		3305	/* set HW defaults for 3D engine */
		3306	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \|
		3307	ROQ_IB2_START(0x2b)));
		3308	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) \| MEQ2_START(0x60));
		3309
		3310	sx_debug_1 = RREG32(SX_DEBUG_1);
		3311	WREG32(SX_DEBUG_1, sx_debug_1);
		3312
		3313	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
		3314
		3315	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) \|
		3316	SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) \|
		3317	SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) \|
		3318	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
		3319
		3320	WREG32(VGT_NUM_INSTANCES, 1);
		3321
		3322	WREG32(CP_PERFMON_CNTL, 0);
		3323
		3324	WREG32(SQ_CONFIG, 0);
		3325
		3326	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		3327	FORCE_EOV_MAX_REZ_CNT(255)));
		3328
		3329	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) \|
		3330	AUTO_INVLD_EN(ES_AND_GS_AUTO));
		3331
		3332	WREG32(VGT_GS_VERTEX_REUSE, 16);
		3333	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		3334
		3335	WREG32(CB_PERFCOUNTER0_SELECT0, 0);
		3336	WREG32(CB_PERFCOUNTER0_SELECT1, 0);
		3337	WREG32(CB_PERFCOUNTER1_SELECT0, 0);
		3338	WREG32(CB_PERFCOUNTER1_SELECT1, 0);
		3339	WREG32(CB_PERFCOUNTER2_SELECT0, 0);
		3340	WREG32(CB_PERFCOUNTER2_SELECT1, 0);
		3341	WREG32(CB_PERFCOUNTER3_SELECT0, 0);
		3342	WREG32(CB_PERFCOUNTER3_SELECT1, 0);
		3343
		3344	tmp = RREG32(HDP_MISC_CNTL);
		3345	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		3346	WREG32(HDP_MISC_CNTL, tmp);
		3347
		3348	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		3349	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		3350
		3351	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		3352
		3353	udelay(50);
		3354	}
		3355
		3356	/*
		3357	* GPU scratch registers helpers function.
		3358	*/
		3359	static void si_scratch_init(struct radeon_device *rdev)
		3360	{
		3361	int i;
		3362
		3363	rdev->scratch.num_reg = 7;
		3364	rdev->scratch.reg_base = SCRATCH_REG0;
		3365	for (i = 0; i < rdev->scratch.num_reg; i++) {
		3366	rdev->scratch.free[i] = true;
		3367	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
		3368	}
		3369	}
		3370
		3371	void si_fence_ring_emit(struct radeon_device *rdev,
		3372	struct radeon_fence *fence)
		3373	{
		3374	struct radeon_ring *ring = &rdev->ring[fence->ring];
		3375	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		3376
		3377	/* flush read cache over gart */
		3378	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		3379	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
		3380	radeon_ring_write(ring, 0);
		3381	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		3382	radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA \|
		3383	PACKET3_TC_ACTION_ENA \|
		3384	PACKET3_SH_KCACHE_ACTION_ENA \|
		3385	PACKET3_SH_ICACHE_ACTION_ENA);
		3386	radeon_ring_write(ring, 0xFFFFFFFF);
		3387	radeon_ring_write(ring, 0);
		3388	radeon_ring_write(ring, 10); /* poll interval */
		3389	/* EVENT_WRITE_EOP - flush caches, send int */
		3390	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		3391	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) \| EVENT_INDEX(5));
5078	serge	3392	radeon_ring_write(ring, lower_32_bits(addr));
2997	Serge	3393	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		3394	radeon_ring_write(ring, fence->seq);
		3395	radeon_ring_write(ring, 0);
		3396	}
		3397
		3398	/*
		3399	* IB stuff
		3400	*/
		3401	void si_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		3402	{
		3403	struct radeon_ring *ring = &rdev->ring[ib->ring];
5271	serge	3404	unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2997	Serge	3405	u32 header;
		3406
		3407	if (ib->is_const_ib) {
		3408	/* set switch buffer packet before const IB */
		3409	radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
		3410	radeon_ring_write(ring, 0);
		3411
		3412	header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
		3413	} else {
		3414	u32 next_rptr;
		3415	if (ring->rptr_save_reg) {
		3416	next_rptr = ring->wptr + 3 + 4 + 8;
		3417	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		3418	radeon_ring_write(ring, ((ring->rptr_save_reg -
		3419	PACKET3_SET_CONFIG_REG_START) >> 2));
		3420	radeon_ring_write(ring, next_rptr);
		3421	} else if (rdev->wb.enabled) {
		3422	next_rptr = ring->wptr + 5 + 4 + 8;
		3423	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
		3424	radeon_ring_write(ring, (1 << 8));
		3425	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5078	serge	3426	radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2997	Serge	3427	radeon_ring_write(ring, next_rptr);
		3428	}
		3429
		3430	header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
		3431	}
		3432
		3433	radeon_ring_write(ring, header);
		3434	radeon_ring_write(ring,
		3435	#ifdef __BIG_ENDIAN
		3436	(2 << 0) \|
		3437	#endif
		3438	(ib->gpu_addr & 0xFFFFFFFC));
		3439	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
5271	serge	3440	radeon_ring_write(ring, ib->length_dw \| (vm_id << 24));
2997	Serge	3441
		3442	if (!ib->is_const_ib) {
		3443	/* flush read cache over gart for this vmid */
		3444	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		3445	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
5271	serge	3446	radeon_ring_write(ring, vm_id);
2997	Serge	3447	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		3448	radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA \|
		3449	PACKET3_TC_ACTION_ENA \|
		3450	PACKET3_SH_KCACHE_ACTION_ENA \|
		3451	PACKET3_SH_ICACHE_ACTION_ENA);
		3452	radeon_ring_write(ring, 0xFFFFFFFF);
		3453	radeon_ring_write(ring, 0);
		3454	radeon_ring_write(ring, 10); /* poll interval */
		3455	}
		3456	}
		3457
		3458	/*
		3459	* CP.
		3460	*/
		3461	static void si_cp_enable(struct radeon_device *rdev, bool enable)
		3462	{
		3463	if (enable)
		3464	WREG32(CP_ME_CNTL, 0);
		3465	else {
5078	serge	3466	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
6104	serge	3467	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2997	Serge	3468	WREG32(CP_ME_CNTL, (CP_ME_HALT \| CP_PFP_HALT \| CP_CE_HALT));
		3469	WREG32(SCRATCH_UMSK, 0);
3192	Serge	3470	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
		3471	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		3472	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2997	Serge	3473	}
		3474	udelay(50);
		3475	}
		3476
		3477	static int si_cp_load_microcode(struct radeon_device *rdev)
		3478	{
		3479	int i;
		3480
5078	serge	3481	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->ce_fw)
2997	Serge	3482	return -EINVAL;
		3483
		3484	si_cp_enable(rdev, false);
		3485
5078	serge	3486	if (rdev->new_fw) {
		3487	const struct gfx_firmware_header_v1_0 *pfp_hdr =
		3488	(const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
		3489	const struct gfx_firmware_header_v1_0 *ce_hdr =
		3490	(const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
		3491	const struct gfx_firmware_header_v1_0 *me_hdr =
		3492	(const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
		3493	const __le32 *fw_data;
		3494	u32 fw_size;
		3495
		3496	radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
		3497	radeon_ucode_print_gfx_hdr(&ce_hdr->header);
		3498	radeon_ucode_print_gfx_hdr(&me_hdr->header);
		3499
		3500	/* PFP */
		3501	fw_data = (const __le32 *)
		3502	(rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
		3503	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
		3504	WREG32(CP_PFP_UCODE_ADDR, 0);
		3505	for (i = 0; i < fw_size; i++)
		3506	WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
		3507	WREG32(CP_PFP_UCODE_ADDR, 0);
		3508
		3509	/* CE */
		3510	fw_data = (const __le32 *)
		3511	(rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
		3512	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
		3513	WREG32(CP_CE_UCODE_ADDR, 0);
		3514	for (i = 0; i < fw_size; i++)
		3515	WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
		3516	WREG32(CP_CE_UCODE_ADDR, 0);
		3517
		3518	/* ME */
		3519	fw_data = (const __be32 *)
		3520	(rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
		3521	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
		3522	WREG32(CP_ME_RAM_WADDR, 0);
		3523	for (i = 0; i < fw_size; i++)
		3524	WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
		3525	WREG32(CP_ME_RAM_WADDR, 0);
		3526	} else {
		3527	const __be32 *fw_data;
		3528
6104	serge	3529	/* PFP */
		3530	fw_data = (const __be32 *)rdev->pfp_fw->data;
		3531	WREG32(CP_PFP_UCODE_ADDR, 0);
		3532	for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
		3533	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		3534	WREG32(CP_PFP_UCODE_ADDR, 0);
2997	Serge	3535
6104	serge	3536	/* CE */
		3537	fw_data = (const __be32 *)rdev->ce_fw->data;
		3538	WREG32(CP_CE_UCODE_ADDR, 0);
		3539	for (i = 0; i < SI_CE_UCODE_SIZE; i++)
		3540	WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
		3541	WREG32(CP_CE_UCODE_ADDR, 0);
2997	Serge	3542
6104	serge	3543	/* ME */
		3544	fw_data = (const __be32 *)rdev->me_fw->data;
		3545	WREG32(CP_ME_RAM_WADDR, 0);
		3546	for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
		3547	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		3548	WREG32(CP_ME_RAM_WADDR, 0);
5078	serge	3549	}
2997	Serge	3550
		3551	WREG32(CP_PFP_UCODE_ADDR, 0);
		3552	WREG32(CP_CE_UCODE_ADDR, 0);
		3553	WREG32(CP_ME_RAM_WADDR, 0);
		3554	WREG32(CP_ME_RAM_RADDR, 0);
		3555	return 0;
		3556	}
		3557
		3558	static int si_cp_start(struct radeon_device *rdev)
		3559	{
		3560	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		3561	int r, i;
		3562
		3563	r = radeon_ring_lock(rdev, ring, 7 + 4);
		3564	if (r) {
		3565	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		3566	return r;
		3567	}
		3568	/* init the CP */
		3569	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		3570	radeon_ring_write(ring, 0x1);
		3571	radeon_ring_write(ring, 0x0);
		3572	radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
		3573	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		3574	radeon_ring_write(ring, 0);
		3575	radeon_ring_write(ring, 0);
		3576
		3577	/* init the CE partitions */
		3578	radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
		3579	radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
		3580	radeon_ring_write(ring, 0xc000);
		3581	radeon_ring_write(ring, 0xe000);
5078	serge	3582	radeon_ring_unlock_commit(rdev, ring, false);
2997	Serge	3583
		3584	si_cp_enable(rdev, true);
		3585
		3586	r = radeon_ring_lock(rdev, ring, si_default_size + 10);
		3587	if (r) {
		3588	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		3589	return r;
		3590	}
		3591
		3592	/* setup clear context state */
		3593	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		3594	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
		3595
		3596	for (i = 0; i < si_default_size; i++)
		3597	radeon_ring_write(ring, si_default_state[i]);
		3598
		3599	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		3600	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
		3601
		3602	/* set clear context state */
		3603	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
		3604	radeon_ring_write(ring, 0);
		3605
		3606	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
		3607	radeon_ring_write(ring, 0x00000316);
		3608	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		3609	radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
		3610
5078	serge	3611	radeon_ring_unlock_commit(rdev, ring, false);
2997	Serge	3612
		3613	for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
		3614	ring = &rdev->ring[i];
		3615	r = radeon_ring_lock(rdev, ring, 2);
		3616
		3617	/* clear the compute context state */
		3618	radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
		3619	radeon_ring_write(ring, 0);
		3620
5078	serge	3621	radeon_ring_unlock_commit(rdev, ring, false);
2997	Serge	3622	}
		3623
		3624	return 0;
		3625	}
		3626
		3627	static void si_cp_fini(struct radeon_device *rdev)
		3628	{
		3629	struct radeon_ring *ring;
		3630	si_cp_enable(rdev, false);
		3631
5078	serge	3632	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		3633	radeon_ring_fini(rdev, ring);
		3634	radeon_scratch_free(rdev, ring->rptr_save_reg);
2997	Serge	3635
5078	serge	3636	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
		3637	radeon_ring_fini(rdev, ring);
		3638	radeon_scratch_free(rdev, ring->rptr_save_reg);
2997	Serge	3639
5078	serge	3640	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
		3641	radeon_ring_fini(rdev, ring);
		3642	radeon_scratch_free(rdev, ring->rptr_save_reg);
2997	Serge	3643	}
		3644
		3645	static int si_cp_resume(struct radeon_device *rdev)
		3646	{
		3647	struct radeon_ring *ring;
		3648	u32 tmp;
		3649	u32 rb_bufsz;
		3650	int r;
		3651
5078	serge	3652	si_enable_gui_idle_interrupt(rdev, false);
2997	Serge	3653
		3654	WREG32(CP_SEM_WAIT_TIMER, 0x0);
		3655	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
		3656
		3657	/* Set the write pointer delay */
		3658	WREG32(CP_RB_WPTR_DELAY, 0);
		3659
		3660	WREG32(CP_DEBUG, 0);
		3661	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		3662
		3663	/* ring 0 - compute and gfx */
		3664	/* Set ring buffer size */
		3665	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5078	serge	3666	rb_bufsz = order_base_2(ring->ring_size / 8);
		3667	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
2997	Serge	3668	#ifdef __BIG_ENDIAN
		3669	tmp \|= BUF_SWAP_32BIT;
		3670	#endif
		3671	WREG32(CP_RB0_CNTL, tmp);
		3672
		3673	/* Initialize the ring buffer's read and write pointers */
		3674	WREG32(CP_RB0_CNTL, tmp \| RB_RPTR_WR_ENA);
		3675	ring->wptr = 0;
		3676	WREG32(CP_RB0_WPTR, ring->wptr);
		3677
3120	serge	3678	/* set the wb address whether it's enabled or not */
2997	Serge	3679	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
		3680	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		3681
		3682	if (rdev->wb.enabled)
		3683	WREG32(SCRATCH_UMSK, 0xff);
		3684	else {
		3685	tmp \|= RB_NO_UPDATE;
		3686	WREG32(SCRATCH_UMSK, 0);
		3687	}
		3688
		3689	mdelay(1);
		3690	WREG32(CP_RB0_CNTL, tmp);
		3691
		3692	WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
		3693
		3694	/* ring1 - compute only */
		3695	/* Set ring buffer size */
		3696	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5078	serge	3697	rb_bufsz = order_base_2(ring->ring_size / 8);
		3698	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
2997	Serge	3699	#ifdef __BIG_ENDIAN
		3700	tmp \|= BUF_SWAP_32BIT;
		3701	#endif
		3702	WREG32(CP_RB1_CNTL, tmp);
		3703
		3704	/* Initialize the ring buffer's read and write pointers */
		3705	WREG32(CP_RB1_CNTL, tmp \| RB_RPTR_WR_ENA);
		3706	ring->wptr = 0;
		3707	WREG32(CP_RB1_WPTR, ring->wptr);
		3708
3120	serge	3709	/* set the wb address whether it's enabled or not */
2997	Serge	3710	WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
		3711	WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
		3712
		3713	mdelay(1);
		3714	WREG32(CP_RB1_CNTL, tmp);
		3715
		3716	WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
		3717
		3718	/* ring2 - compute only */
		3719	/* Set ring buffer size */
		3720	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5078	serge	3721	rb_bufsz = order_base_2(ring->ring_size / 8);
		3722	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
2997	Serge	3723	#ifdef __BIG_ENDIAN
		3724	tmp \|= BUF_SWAP_32BIT;
		3725	#endif
		3726	WREG32(CP_RB2_CNTL, tmp);
		3727
		3728	/* Initialize the ring buffer's read and write pointers */
		3729	WREG32(CP_RB2_CNTL, tmp \| RB_RPTR_WR_ENA);
		3730	ring->wptr = 0;
		3731	WREG32(CP_RB2_WPTR, ring->wptr);
		3732
3120	serge	3733	/* set the wb address whether it's enabled or not */
2997	Serge	3734	WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
		3735	WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
		3736
		3737	mdelay(1);
		3738	WREG32(CP_RB2_CNTL, tmp);
		3739
		3740	WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
		3741
		3742	/* start the rings */
		3743	si_cp_start(rdev);
		3744	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
		3745	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
		3746	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
		3747	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
		3748	if (r) {
		3749	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
		3750	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		3751	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
		3752	return r;
		3753	}
		3754	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
		3755	if (r) {
		3756	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		3757	}
		3758	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
		3759	if (r) {
		3760	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
		3761	}
		3762
5078	serge	3763	si_enable_gui_idle_interrupt(rdev, true);
		3764
		3765	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
		3766	radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
		3767
2997	Serge	3768	return 0;
		3769	}
		3770
5078	serge	3771	u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
2997	Serge	3772	{
3764	Serge	3773	u32 reset_mask = 0;
		3774	u32 tmp;
2997	Serge	3775
3764	Serge	3776	/* GRBM_STATUS */
		3777	tmp = RREG32(GRBM_STATUS);
		3778	if (tmp & (PA_BUSY \| SC_BUSY \|
		3779	BCI_BUSY \| SX_BUSY \|
		3780	TA_BUSY \| VGT_BUSY \|
		3781	DB_BUSY \| CB_BUSY \|
		3782	GDS_BUSY \| SPI_BUSY \|
		3783	IA_BUSY \| IA_BUSY_NO_DMA))
		3784	reset_mask \|= RADEON_RESET_GFX;
		3785
		3786	if (tmp & (CF_RQ_PENDING \| PF_RQ_PENDING \|
		3787	CP_BUSY \| CP_COHERENCY_BUSY))
		3788	reset_mask \|= RADEON_RESET_CP;
		3789
		3790	if (tmp & GRBM_EE_BUSY)
		3791	reset_mask \|= RADEON_RESET_GRBM \| RADEON_RESET_GFX \| RADEON_RESET_CP;
		3792
		3793	/* GRBM_STATUS2 */
		3794	tmp = RREG32(GRBM_STATUS2);
		3795	if (tmp & (RLC_RQ_PENDING \| RLC_BUSY))
		3796	reset_mask \|= RADEON_RESET_RLC;
		3797
		3798	/* DMA_STATUS_REG 0 */
		3799	tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
		3800	if (!(tmp & DMA_IDLE))
		3801	reset_mask \|= RADEON_RESET_DMA;
		3802
		3803	/* DMA_STATUS_REG 1 */
		3804	tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
		3805	if (!(tmp & DMA_IDLE))
		3806	reset_mask \|= RADEON_RESET_DMA1;
		3807
		3808	/* SRBM_STATUS2 */
		3809	tmp = RREG32(SRBM_STATUS2);
		3810	if (tmp & DMA_BUSY)
		3811	reset_mask \|= RADEON_RESET_DMA;
		3812
		3813	if (tmp & DMA1_BUSY)
		3814	reset_mask \|= RADEON_RESET_DMA1;
		3815
		3816	/* SRBM_STATUS */
		3817	tmp = RREG32(SRBM_STATUS);
		3818
		3819	if (tmp & IH_BUSY)
		3820	reset_mask \|= RADEON_RESET_IH;
		3821
		3822	if (tmp & SEM_BUSY)
		3823	reset_mask \|= RADEON_RESET_SEM;
		3824
		3825	if (tmp & GRBM_RQ_PENDING)
		3826	reset_mask \|= RADEON_RESET_GRBM;
		3827
		3828	if (tmp & VMC_BUSY)
		3829	reset_mask \|= RADEON_RESET_VMC;
		3830
		3831	if (tmp & (MCB_BUSY \| MCB_NON_DISPLAY_BUSY \|
		3832	MCC_BUSY \| MCD_BUSY))
		3833	reset_mask \|= RADEON_RESET_MC;
		3834
		3835	if (evergreen_is_display_hung(rdev))
		3836	reset_mask \|= RADEON_RESET_DISPLAY;
		3837
		3838	/* VM_L2_STATUS */
		3839	tmp = RREG32(VM_L2_STATUS);
		3840	if (tmp & L2_BUSY)
		3841	reset_mask \|= RADEON_RESET_VMC;
		3842
		3843	/* Skip MC reset as it's mostly likely not hung, just busy */
		3844	if (reset_mask & RADEON_RESET_MC) {
		3845	DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
		3846	reset_mask &= ~RADEON_RESET_MC;
2997	Serge	3847	}
3764	Serge	3848
		3849	return reset_mask;
2997	Serge	3850	}
		3851
3764	Serge	3852	static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
2997	Serge	3853	{
3764	Serge	3854	struct evergreen_mc_save save;
		3855	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
		3856	u32 tmp;
2997	Serge	3857
3764	Serge	3858	if (reset_mask == 0)
3192	Serge	3859	return;
2997	Serge	3860
3764	Serge	3861	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3192	Serge	3862
3764	Serge	3863	evergreen_print_gpu_status_regs(rdev);
		3864	dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
		3865	RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
		3866	dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
		3867	RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
		3868
5078	serge	3869	/* disable PG/CG */
		3870	si_fini_pg(rdev);
		3871	si_fini_cg(rdev);
		3872
		3873	/* stop the rlc */
		3874	si_rlc_stop(rdev);
		3875
2997	Serge	3876	/* Disable CP parsing/prefetching */
		3877	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT \| CP_CE_HALT);
		3878
3764	Serge	3879	if (reset_mask & RADEON_RESET_DMA) {
		3880	/* dma0 */
		3881	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
		3882	tmp &= ~DMA_RB_ENABLE;
		3883	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
		3884	}
		3885	if (reset_mask & RADEON_RESET_DMA1) {
		3886	/* dma1 */
		3887	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
		3888	tmp &= ~DMA_RB_ENABLE;
		3889	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
		3890	}
		3891
		3892	udelay(50);
		3893
		3894	evergreen_mc_stop(rdev, &save);
		3895	if (evergreen_mc_wait_for_idle(rdev)) {
		3896	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		3897	}
		3898
		3899	if (reset_mask & (RADEON_RESET_GFX \| RADEON_RESET_COMPUTE \| RADEON_RESET_CP)) {
		3900	grbm_soft_reset = SOFT_RESET_CB \|
6104	serge	3901	SOFT_RESET_DB \|
		3902	SOFT_RESET_GDS \|
		3903	SOFT_RESET_PA \|
		3904	SOFT_RESET_SC \|
		3905	SOFT_RESET_BCI \|
		3906	SOFT_RESET_SPI \|
		3907	SOFT_RESET_SX \|
		3908	SOFT_RESET_TC \|
		3909	SOFT_RESET_TA \|
		3910	SOFT_RESET_VGT \|
3764	Serge	3911	SOFT_RESET_IA;
		3912	}
2997	Serge	3913
3764	Serge	3914	if (reset_mask & RADEON_RESET_CP) {
		3915	grbm_soft_reset \|= SOFT_RESET_CP \| SOFT_RESET_VGT;
3192	Serge	3916
3764	Serge	3917	srbm_soft_reset \|= SOFT_RESET_GRBM;
		3918	}
3192	Serge	3919
3764	Serge	3920	if (reset_mask & RADEON_RESET_DMA)
		3921	srbm_soft_reset \|= SOFT_RESET_DMA;
3192	Serge	3922
3764	Serge	3923	if (reset_mask & RADEON_RESET_DMA1)
		3924	srbm_soft_reset \|= SOFT_RESET_DMA1;
3192	Serge	3925
3764	Serge	3926	if (reset_mask & RADEON_RESET_DISPLAY)
		3927	srbm_soft_reset \|= SOFT_RESET_DC;
3192	Serge	3928
3764	Serge	3929	if (reset_mask & RADEON_RESET_RLC)
		3930	grbm_soft_reset \|= SOFT_RESET_RLC;
3192	Serge	3931
3764	Serge	3932	if (reset_mask & RADEON_RESET_SEM)
		3933	srbm_soft_reset \|= SOFT_RESET_SEM;
3192	Serge	3934
3764	Serge	3935	if (reset_mask & RADEON_RESET_IH)
		3936	srbm_soft_reset \|= SOFT_RESET_IH;
3192	Serge	3937
3764	Serge	3938	if (reset_mask & RADEON_RESET_GRBM)
		3939	srbm_soft_reset \|= SOFT_RESET_GRBM;
3192	Serge	3940
3764	Serge	3941	if (reset_mask & RADEON_RESET_VMC)
		3942	srbm_soft_reset \|= SOFT_RESET_VMC;
3192	Serge	3943
3764	Serge	3944	if (reset_mask & RADEON_RESET_MC)
		3945	srbm_soft_reset \|= SOFT_RESET_MC;
3192	Serge	3946
3764	Serge	3947	if (grbm_soft_reset) {
		3948	tmp = RREG32(GRBM_SOFT_RESET);
		3949	tmp \|= grbm_soft_reset;
		3950	dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
		3951	WREG32(GRBM_SOFT_RESET, tmp);
		3952	tmp = RREG32(GRBM_SOFT_RESET);
3192	Serge	3953
3764	Serge	3954	udelay(50);
3192	Serge	3955
3764	Serge	3956	tmp &= ~grbm_soft_reset;
		3957	WREG32(GRBM_SOFT_RESET, tmp);
		3958	tmp = RREG32(GRBM_SOFT_RESET);
3192	Serge	3959	}
		3960
3764	Serge	3961	if (srbm_soft_reset) {
		3962	tmp = RREG32(SRBM_SOFT_RESET);
		3963	tmp \|= srbm_soft_reset;
		3964	dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
		3965	WREG32(SRBM_SOFT_RESET, tmp);
		3966	tmp = RREG32(SRBM_SOFT_RESET);
3192	Serge	3967
3764	Serge	3968	udelay(50);
3192	Serge	3969
3764	Serge	3970	tmp &= ~srbm_soft_reset;
		3971	WREG32(SRBM_SOFT_RESET, tmp);
		3972	tmp = RREG32(SRBM_SOFT_RESET);
		3973	}
		3974
3192	Serge	3975	/* Wait a little for things to settle down */
		3976	udelay(50);
		3977
2997	Serge	3978	evergreen_mc_resume(rdev, &save);
3764	Serge	3979	udelay(50);
		3980
		3981	evergreen_print_gpu_status_regs(rdev);
2997	Serge	3982	}
		3983
5078	serge	3984	static void si_set_clk_bypass_mode(struct radeon_device *rdev)
		3985	{
		3986	u32 tmp, i;
		3987
		3988	tmp = RREG32(CG_SPLL_FUNC_CNTL);
		3989	tmp \|= SPLL_BYPASS_EN;
		3990	WREG32(CG_SPLL_FUNC_CNTL, tmp);
		3991
		3992	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
		3993	tmp \|= SPLL_CTLREQ_CHG;
		3994	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
		3995
		3996	for (i = 0; i < rdev->usec_timeout; i++) {
		3997	if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
		3998	break;
		3999	udelay(1);
		4000	}
		4001
		4002	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
		4003	tmp &= ~(SPLL_CTLREQ_CHG \| SCLK_MUX_UPDATE);
		4004	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
		4005
		4006	tmp = RREG32(MPLL_CNTL_MODE);
		4007	tmp &= ~MPLL_MCLK_SEL;
		4008	WREG32(MPLL_CNTL_MODE, tmp);
		4009	}
		4010
		4011	static void si_spll_powerdown(struct radeon_device *rdev)
		4012	{
		4013	u32 tmp;
		4014
		4015	tmp = RREG32(SPLL_CNTL_MODE);
		4016	tmp \|= SPLL_SW_DIR_CONTROL;
		4017	WREG32(SPLL_CNTL_MODE, tmp);
		4018
		4019	tmp = RREG32(CG_SPLL_FUNC_CNTL);
		4020	tmp \|= SPLL_RESET;
		4021	WREG32(CG_SPLL_FUNC_CNTL, tmp);
		4022
		4023	tmp = RREG32(CG_SPLL_FUNC_CNTL);
		4024	tmp \|= SPLL_SLEEP;
		4025	WREG32(CG_SPLL_FUNC_CNTL, tmp);
		4026
		4027	tmp = RREG32(SPLL_CNTL_MODE);
		4028	tmp &= ~SPLL_SW_DIR_CONTROL;
		4029	WREG32(SPLL_CNTL_MODE, tmp);
		4030	}
		4031
		4032	static void si_gpu_pci_config_reset(struct radeon_device *rdev)
		4033	{
		4034	struct evergreen_mc_save save;
		4035	u32 tmp, i;
		4036
		4037	dev_info(rdev->dev, "GPU pci config reset\n");
		4038
		4039	/* disable dpm? */
		4040
		4041	/* disable cg/pg */
		4042	si_fini_pg(rdev);
		4043	si_fini_cg(rdev);
		4044
		4045	/* Disable CP parsing/prefetching */
		4046	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT \| CP_CE_HALT);
		4047	/* dma0 */
		4048	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
		4049	tmp &= ~DMA_RB_ENABLE;
		4050	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
		4051	/* dma1 */
		4052	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
		4053	tmp &= ~DMA_RB_ENABLE;
		4054	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
		4055	/* XXX other engines? */
		4056
		4057	/* halt the rlc, disable cp internal ints */
		4058	si_rlc_stop(rdev);
		4059
		4060	udelay(50);
		4061
		4062	/* disable mem access */
		4063	evergreen_mc_stop(rdev, &save);
		4064	if (evergreen_mc_wait_for_idle(rdev)) {
		4065	dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
		4066	}
		4067
		4068	/* set mclk/sclk to bypass */
		4069	si_set_clk_bypass_mode(rdev);
		4070	/* powerdown spll */
		4071	si_spll_powerdown(rdev);
		4072	/* disable BM */
		4073	pci_clear_master(rdev->pdev);
		4074	/* reset */
		4075	radeon_pci_config_reset(rdev);
		4076	/* wait for asic to come out of reset */
		4077	for (i = 0; i < rdev->usec_timeout; i++) {
		4078	if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
		4079	break;
		4080	udelay(1);
		4081	}
		4082	}
		4083
2997	Serge	4084	int si_asic_reset(struct radeon_device *rdev)
		4085	{
3764	Serge	4086	u32 reset_mask;
		4087
		4088	reset_mask = si_gpu_check_soft_reset(rdev);
		4089
		4090	if (reset_mask)
		4091	r600_set_bios_scratch_engine_hung(rdev, true);
		4092
5078	serge	4093	/* try soft reset */
3764	Serge	4094	si_gpu_soft_reset(rdev, reset_mask);
		4095
		4096	reset_mask = si_gpu_check_soft_reset(rdev);
		4097
5078	serge	4098	/* try pci config reset */
		4099	if (reset_mask && radeon_hard_reset)
		4100	si_gpu_pci_config_reset(rdev);
		4101
		4102	reset_mask = si_gpu_check_soft_reset(rdev);
		4103
3764	Serge	4104	if (!reset_mask)
		4105	r600_set_bios_scratch_engine_hung(rdev, false);
		4106
		4107	return 0;
		4108	}
		4109
		4110	/**
		4111	* si_gfx_is_lockup - Check if the GFX engine is locked up
		4112	*
		4113	* @rdev: radeon_device pointer
		4114	* @ring: radeon_ring structure holding ring information
		4115	*
		4116	* Check if the GFX engine is locked up.
		4117	* Returns true if the engine appears to be locked up, false if not.
		4118	*/
		4119	bool si_gfx_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
		4120	{
		4121	u32 reset_mask = si_gpu_check_soft_reset(rdev);
		4122
		4123	if (!(reset_mask & (RADEON_RESET_GFX \|
6104	serge	4124	RADEON_RESET_COMPUTE \|
3764	Serge	4125	RADEON_RESET_CP))) {
5078	serge	4126	radeon_ring_lockup_update(rdev, ring);
3764	Serge	4127	return false;
		4128	}
		4129	return radeon_ring_test_lockup(rdev, ring);
2997	Serge	4130	}
		4131
		4132	/* MC */
		4133	static void si_mc_program(struct radeon_device *rdev)
		4134	{
		4135	struct evergreen_mc_save save;
		4136	u32 tmp;
		4137	int i, j;
		4138
		4139	/* Initialize HDP */
		4140	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		4141	WREG32((0x2c14 + j), 0x00000000);
		4142	WREG32((0x2c18 + j), 0x00000000);
		4143	WREG32((0x2c1c + j), 0x00000000);
		4144	WREG32((0x2c20 + j), 0x00000000);
		4145	WREG32((0x2c24 + j), 0x00000000);
		4146	}
		4147	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		4148
		4149	evergreen_mc_stop(rdev, &save);
		4150	if (radeon_mc_wait_for_idle(rdev)) {
		4151	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		4152	}
3764	Serge	4153	if (!ASIC_IS_NODCE(rdev))
6104	serge	4154	/* Lockout access through VGA aperture*/
		4155	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2997	Serge	4156	/* Update configuration */
		4157	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		4158	rdev->mc.vram_start >> 12);
		4159	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		4160	rdev->mc.vram_end >> 12);
		4161	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
		4162	rdev->vram_scratch.gpu_addr >> 12);
		4163	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		4164	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		4165	WREG32(MC_VM_FB_LOCATION, tmp);
		4166	/* XXX double check these! */
		4167	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
		4168	WREG32(HDP_NONSURFACE_INFO, (2 << 7) \| (1 << 30));
		4169	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
		4170	WREG32(MC_VM_AGP_BASE, 0);
		4171	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		4172	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		4173	if (radeon_mc_wait_for_idle(rdev)) {
		4174	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		4175	}
		4176	evergreen_mc_resume(rdev, &save);
3764	Serge	4177	if (!ASIC_IS_NODCE(rdev)) {
6104	serge	4178	/* we need to own VRAM, so turn off the VGA renderer here
		4179	* to stop it overwriting our objects */
		4180	rv515_vga_render_disable(rdev);
		4181	}
2997	Serge	4182	}
		4183
5078	serge	4184	void si_vram_gtt_location(struct radeon_device *rdev,
6104	serge	4185	struct radeon_mc *mc)
2997	Serge	4186	{
		4187	if (mc->mc_vram_size > 0xFFC0000000ULL) {
		4188	/* leave room for at least 1024M GTT */
		4189	dev_warn(rdev->dev, "limiting VRAM\n");
		4190	mc->real_vram_size = 0xFFC0000000ULL;
		4191	mc->mc_vram_size = 0xFFC0000000ULL;
		4192	}
3764	Serge	4193	radeon_vram_location(rdev, &rdev->mc, 0);
2997	Serge	4194	rdev->mc.gtt_base_align = 0;
3764	Serge	4195	radeon_gtt_location(rdev, mc);
2997	Serge	4196	}
		4197
		4198	static int si_mc_init(struct radeon_device *rdev)
		4199	{
		4200	u32 tmp;
		4201	int chansize, numchan;
		4202
		4203	/* Get VRAM informations */
		4204	rdev->mc.vram_is_ddr = true;
		4205	tmp = RREG32(MC_ARB_RAMCFG);
		4206	if (tmp & CHANSIZE_OVERRIDE) {
		4207	chansize = 16;
		4208	} else if (tmp & CHANSIZE_MASK) {
		4209	chansize = 64;
		4210	} else {
		4211	chansize = 32;
		4212	}
		4213	tmp = RREG32(MC_SHARED_CHMAP);
		4214	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		4215	case 0:
		4216	default:
		4217	numchan = 1;
		4218	break;
		4219	case 1:
		4220	numchan = 2;
		4221	break;
		4222	case 2:
		4223	numchan = 4;
		4224	break;
		4225	case 3:
		4226	numchan = 8;
		4227	break;
		4228	case 4:
		4229	numchan = 3;
		4230	break;
		4231	case 5:
		4232	numchan = 6;
		4233	break;
		4234	case 6:
		4235	numchan = 10;
		4236	break;
		4237	case 7:
		4238	numchan = 12;
		4239	break;
		4240	case 8:
		4241	numchan = 16;
		4242	break;
		4243	}
		4244	rdev->mc.vram_width = numchan * chansize;
		4245	/* Could aper size report 0 ? */
		4246	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		4247	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
		4248	/* size in MB on si */
5078	serge	4249	tmp = RREG32(CONFIG_MEMSIZE);
		4250	/* some boards may have garbage in the upper 16 bits */
		4251	if (tmp & 0xffff0000) {
		4252	DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
		4253	if (tmp & 0xffff)
		4254	tmp &= 0xffff;
		4255	}
		4256	rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
		4257	rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
2997	Serge	4258	rdev->mc.visible_vram_size = rdev->mc.aper_size;
		4259	si_vram_gtt_location(rdev, &rdev->mc);
		4260	radeon_update_bandwidth_info(rdev);
		4261
		4262	return 0;
		4263	}
		4264
		4265	/*
		4266	* GART
		4267	*/
		4268	void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
		4269	{
		4270	/* flush hdp cache */
		4271	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		4272
		4273	/* bits 0-15 are the VM contexts0-15 */
		4274	WREG32(VM_INVALIDATE_REQUEST, 1);
		4275	}
		4276
		4277	static int si_pcie_gart_enable(struct radeon_device *rdev)
		4278	{
		4279	int r, i;
		4280
		4281	if (rdev->gart.robj == NULL) {
		4282	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		4283	return -EINVAL;
		4284	}
		4285	r = radeon_gart_table_vram_pin(rdev);
		4286	if (r)
		4287	return r;
		4288	/* Setup TLB control */
		4289	WREG32(MC_VM_MX_L1_TLB_CNTL,
		4290	(0xA << 7) \|
		4291	ENABLE_L1_TLB \|
5078	serge	4292	ENABLE_L1_FRAGMENT_PROCESSING \|
2997	Serge	4293	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		4294	ENABLE_ADVANCED_DRIVER_MODEL \|
		4295	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
		4296	/* Setup L2 cache */
		4297	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \|
5078	serge	4298	ENABLE_L2_FRAGMENT_PROCESSING \|
2997	Serge	4299	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		4300	ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE \|
		4301	EFFECTIVE_L2_QUEUE_SIZE(7) \|
		4302	CONTEXT1_IDENTITY_ACCESS_MODE(1));
		4303	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS \| INVALIDATE_L2_CACHE);
		4304	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY \|
5078	serge	4305	BANK_SELECT(4) \|
		4306	L2_CACHE_BIGK_FRAGMENT_SIZE(4));
2997	Serge	4307	/* setup context0 */
		4308	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		4309	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		4310	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		4311	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		4312	(u32)(rdev->dummy_page.addr >> 12));
		4313	WREG32(VM_CONTEXT0_CNTL2, 0);
		4314	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		4315	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
		4316
		4317	WREG32(0x15D4, 0);
		4318	WREG32(0x15D8, 0);
		4319	WREG32(0x15DC, 0);
		4320
		4321	/* empty context1-15 */
		4322	/* set vm size, must be a multiple of 4 */
		4323	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
6104	serge	4324	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
2997	Serge	4325	/* Assign the pt base to something valid for now; the pts used for
		4326	* the VMs are determined by the application and setup and assigned
		4327	* on the fly in the vm part of radeon_gart.c
		4328	*/
		4329	for (i = 1; i < 16; i++) {
		4330	if (i < 8)
		4331	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5078	serge	4332	rdev->vm_manager.saved_table_addr[i]);
2997	Serge	4333	else
		4334	WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5078	serge	4335	rdev->vm_manager.saved_table_addr[i]);
2997	Serge	4336	}
		4337
		4338	/* enable context1-15 */
		4339	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
		4340	(u32)(rdev->dummy_page.addr >> 12));
3192	Serge	4341	WREG32(VM_CONTEXT1_CNTL2, 4);
2997	Serge	4342	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(1) \|
5078	serge	4343	PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) \|
3192	Serge	4344	RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4345	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT \|
		4346	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4347	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT \|
		4348	PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4349	PDE0_PROTECTION_FAULT_ENABLE_DEFAULT \|
		4350	VALID_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4351	VALID_PROTECTION_FAULT_ENABLE_DEFAULT \|
		4352	READ_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4353	READ_PROTECTION_FAULT_ENABLE_DEFAULT \|
		4354	WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4355	WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
2997	Serge	4356
		4357	si_pcie_gart_tlb_flush(rdev);
		4358	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		4359	(unsigned)(rdev->mc.gtt_size >> 20),
		4360	(unsigned long long)rdev->gart.table_addr);
		4361	rdev->gart.ready = true;
		4362	return 0;
		4363	}
		4364
		4365	static void si_pcie_gart_disable(struct radeon_device *rdev)
		4366	{
5078	serge	4367	unsigned i;
		4368
		4369	for (i = 1; i < 16; ++i) {
		4370	uint32_t reg;
		4371	if (i < 8)
		4372	reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
		4373	else
		4374	reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
		4375	rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
		4376	}
		4377
2997	Serge	4378	/* Disable all tables */
		4379	WREG32(VM_CONTEXT0_CNTL, 0);
		4380	WREG32(VM_CONTEXT1_CNTL, 0);
		4381	/* Setup TLB control */
		4382	WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		4383	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
		4384	/* Setup L2 cache */
		4385	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		4386	ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE \|
		4387	EFFECTIVE_L2_QUEUE_SIZE(7) \|
		4388	CONTEXT1_IDENTITY_ACCESS_MODE(1));
		4389	WREG32(VM_L2_CNTL2, 0);
		4390	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY \|
		4391	L2_CACHE_BIGK_FRAGMENT_SIZE(0));
		4392	radeon_gart_table_vram_unpin(rdev);
		4393	}
		4394
		4395	static void si_pcie_gart_fini(struct radeon_device *rdev)
		4396	{
		4397	si_pcie_gart_disable(rdev);
		4398	radeon_gart_table_vram_free(rdev);
6104	serge	4399	radeon_gart_fini(rdev);
2997	Serge	4400	}
		4401
		4402	/* vm parser */
		4403	static bool si_vm_reg_valid(u32 reg)
		4404	{
		4405	/* context regs are fine */
		4406	if (reg >= 0x28000)
		4407	return true;
		4408
		4409	/* check config regs */
		4410	switch (reg) {
		4411	case GRBM_GFX_INDEX:
3031	serge	4412	case CP_STRMOUT_CNTL:
2997	Serge	4413	case VGT_VTX_VECT_EJECT_REG:
		4414	case VGT_CACHE_INVALIDATION:
		4415	case VGT_ESGS_RING_SIZE:
		4416	case VGT_GSVS_RING_SIZE:
		4417	case VGT_GS_VERTEX_REUSE:
		4418	case VGT_PRIMITIVE_TYPE:
		4419	case VGT_INDEX_TYPE:
		4420	case VGT_NUM_INDICES:
		4421	case VGT_NUM_INSTANCES:
		4422	case VGT_TF_RING_SIZE:
		4423	case VGT_HS_OFFCHIP_PARAM:
		4424	case VGT_TF_MEMORY_BASE:
		4425	case PA_CL_ENHANCE:
		4426	case PA_SU_LINE_STIPPLE_VALUE:
		4427	case PA_SC_LINE_STIPPLE_STATE:
		4428	case PA_SC_ENHANCE:
		4429	case SQC_CACHES:
		4430	case SPI_STATIC_THREAD_MGMT_1:
		4431	case SPI_STATIC_THREAD_MGMT_2:
		4432	case SPI_STATIC_THREAD_MGMT_3:
		4433	case SPI_PS_MAX_WAVE_ID:
		4434	case SPI_CONFIG_CNTL:
		4435	case SPI_CONFIG_CNTL_1:
		4436	case TA_CNTL_AUX:
		4437	return true;
		4438	default:
		4439	DRM_ERROR("Invalid register 0x%x in CS\n", reg);
		4440	return false;
		4441	}
		4442	}
		4443
		4444	static int si_vm_packet3_ce_check(struct radeon_device *rdev,
		4445	u32 ib, struct radeon_cs_packet pkt)
		4446	{
		4447	switch (pkt->opcode) {
		4448	case PACKET3_NOP:
		4449	case PACKET3_SET_BASE:
		4450	case PACKET3_SET_CE_DE_COUNTERS:
		4451	case PACKET3_LOAD_CONST_RAM:
		4452	case PACKET3_WRITE_CONST_RAM:
		4453	case PACKET3_WRITE_CONST_RAM_OFFSET:
		4454	case PACKET3_DUMP_CONST_RAM:
		4455	case PACKET3_INCREMENT_CE_COUNTER:
		4456	case PACKET3_WAIT_ON_DE_COUNTER:
		4457	case PACKET3_CE_WRITE:
		4458	break;
		4459	default:
		4460	DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
		4461	return -EINVAL;
		4462	}
		4463	return 0;
		4464	}
		4465
5078	serge	4466	static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
		4467	{
		4468	u32 start_reg, reg, i;
		4469	u32 command = ib[idx + 4];
		4470	u32 info = ib[idx + 1];
		4471	u32 idx_value = ib[idx];
		4472	if (command & PACKET3_CP_DMA_CMD_SAS) {
		4473	/* src address space is register */
		4474	if (((info & 0x60000000) >> 29) == 0) {
		4475	start_reg = idx_value << 2;
		4476	if (command & PACKET3_CP_DMA_CMD_SAIC) {
		4477	reg = start_reg;
		4478	if (!si_vm_reg_valid(reg)) {
		4479	DRM_ERROR("CP DMA Bad SRC register\n");
		4480	return -EINVAL;
		4481	}
		4482	} else {
		4483	for (i = 0; i < (command & 0x1fffff); i++) {
		4484	reg = start_reg + (4 * i);
		4485	if (!si_vm_reg_valid(reg)) {
		4486	DRM_ERROR("CP DMA Bad SRC register\n");
		4487	return -EINVAL;
		4488	}
		4489	}
		4490	}
		4491	}
		4492	}
		4493	if (command & PACKET3_CP_DMA_CMD_DAS) {
		4494	/* dst address space is register */
		4495	if (((info & 0x00300000) >> 20) == 0) {
		4496	start_reg = ib[idx + 2];
		4497	if (command & PACKET3_CP_DMA_CMD_DAIC) {
		4498	reg = start_reg;
		4499	if (!si_vm_reg_valid(reg)) {
		4500	DRM_ERROR("CP DMA Bad DST register\n");
		4501	return -EINVAL;
		4502	}
		4503	} else {
		4504	for (i = 0; i < (command & 0x1fffff); i++) {
		4505	reg = start_reg + (4 * i);
		4506	if (!si_vm_reg_valid(reg)) {
		4507	DRM_ERROR("CP DMA Bad DST register\n");
		4508	return -EINVAL;
		4509	}
		4510	}
		4511	}
		4512	}
		4513	}
		4514	return 0;
		4515	}
		4516
2997	Serge	4517	static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
		4518	u32 ib, struct radeon_cs_packet pkt)
		4519	{
5078	serge	4520	int r;
2997	Serge	4521	u32 idx = pkt->idx + 1;
		4522	u32 idx_value = ib[idx];
		4523	u32 start_reg, end_reg, reg, i;
		4524
		4525	switch (pkt->opcode) {
		4526	case PACKET3_NOP:
		4527	case PACKET3_SET_BASE:
		4528	case PACKET3_CLEAR_STATE:
		4529	case PACKET3_INDEX_BUFFER_SIZE:
		4530	case PACKET3_DISPATCH_DIRECT:
		4531	case PACKET3_DISPATCH_INDIRECT:
		4532	case PACKET3_ALLOC_GDS:
		4533	case PACKET3_WRITE_GDS_RAM:
		4534	case PACKET3_ATOMIC_GDS:
		4535	case PACKET3_ATOMIC:
		4536	case PACKET3_OCCLUSION_QUERY:
		4537	case PACKET3_SET_PREDICATION:
		4538	case PACKET3_COND_EXEC:
		4539	case PACKET3_PRED_EXEC:
		4540	case PACKET3_DRAW_INDIRECT:
		4541	case PACKET3_DRAW_INDEX_INDIRECT:
		4542	case PACKET3_INDEX_BASE:
		4543	case PACKET3_DRAW_INDEX_2:
		4544	case PACKET3_CONTEXT_CONTROL:
		4545	case PACKET3_INDEX_TYPE:
		4546	case PACKET3_DRAW_INDIRECT_MULTI:
		4547	case PACKET3_DRAW_INDEX_AUTO:
		4548	case PACKET3_DRAW_INDEX_IMMD:
		4549	case PACKET3_NUM_INSTANCES:
		4550	case PACKET3_DRAW_INDEX_MULTI_AUTO:
		4551	case PACKET3_STRMOUT_BUFFER_UPDATE:
		4552	case PACKET3_DRAW_INDEX_OFFSET_2:
		4553	case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
		4554	case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
		4555	case PACKET3_MPEG_INDEX:
		4556	case PACKET3_WAIT_REG_MEM:
		4557	case PACKET3_MEM_WRITE:
		4558	case PACKET3_PFP_SYNC_ME:
		4559	case PACKET3_SURFACE_SYNC:
		4560	case PACKET3_EVENT_WRITE:
		4561	case PACKET3_EVENT_WRITE_EOP:
		4562	case PACKET3_EVENT_WRITE_EOS:
		4563	case PACKET3_SET_CONTEXT_REG:
		4564	case PACKET3_SET_CONTEXT_REG_INDIRECT:
		4565	case PACKET3_SET_SH_REG:
		4566	case PACKET3_SET_SH_REG_OFFSET:
		4567	case PACKET3_INCREMENT_DE_COUNTER:
		4568	case PACKET3_WAIT_ON_CE_COUNTER:
		4569	case PACKET3_WAIT_ON_AVAIL_BUFFER:
		4570	case PACKET3_ME_WRITE:
		4571	break;
		4572	case PACKET3_COPY_DATA:
		4573	if ((idx_value & 0xf00) == 0) {
		4574	reg = ib[idx + 3] * 4;
		4575	if (!si_vm_reg_valid(reg))
		4576	return -EINVAL;
		4577	}
		4578	break;
		4579	case PACKET3_WRITE_DATA:
		4580	if ((idx_value & 0xf00) == 0) {
		4581	start_reg = ib[idx + 1] * 4;
		4582	if (idx_value & 0x10000) {
		4583	if (!si_vm_reg_valid(start_reg))
		4584	return -EINVAL;
		4585	} else {
		4586	for (i = 0; i < (pkt->count - 2); i++) {
		4587	reg = start_reg + (4 * i);
		4588	if (!si_vm_reg_valid(reg))
		4589	return -EINVAL;
		4590	}
		4591	}
		4592	}
		4593	break;
		4594	case PACKET3_COND_WRITE:
		4595	if (idx_value & 0x100) {
		4596	reg = ib[idx + 5] * 4;
		4597	if (!si_vm_reg_valid(reg))
		4598	return -EINVAL;
		4599	}
		4600	break;
		4601	case PACKET3_COPY_DW:
		4602	if (idx_value & 0x2) {
		4603	reg = ib[idx + 3] * 4;
		4604	if (!si_vm_reg_valid(reg))
		4605	return -EINVAL;
		4606	}
		4607	break;
		4608	case PACKET3_SET_CONFIG_REG:
		4609	start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
		4610	end_reg = 4 * pkt->count + start_reg - 4;
		4611	if ((start_reg < PACKET3_SET_CONFIG_REG_START) \|\|
		4612	(start_reg >= PACKET3_SET_CONFIG_REG_END) \|\|
		4613	(end_reg >= PACKET3_SET_CONFIG_REG_END)) {
		4614	DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
		4615	return -EINVAL;
		4616	}
		4617	for (i = 0; i < pkt->count; i++) {
		4618	reg = start_reg + (4 * i);
		4619	if (!si_vm_reg_valid(reg))
		4620	return -EINVAL;
		4621	}
		4622	break;
3192	Serge	4623	case PACKET3_CP_DMA:
5078	serge	4624	r = si_vm_packet3_cp_dma_check(ib, idx);
		4625	if (r)
		4626	return r;
3192	Serge	4627	break;
2997	Serge	4628	default:
		4629	DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
		4630	return -EINVAL;
		4631	}
		4632	return 0;
		4633	}
		4634
		4635	static int si_vm_packet3_compute_check(struct radeon_device *rdev,
		4636	u32 ib, struct radeon_cs_packet pkt)
		4637	{
5078	serge	4638	int r;
2997	Serge	4639	u32 idx = pkt->idx + 1;
		4640	u32 idx_value = ib[idx];
		4641	u32 start_reg, reg, i;
		4642
		4643	switch (pkt->opcode) {
		4644	case PACKET3_NOP:
		4645	case PACKET3_SET_BASE:
		4646	case PACKET3_CLEAR_STATE:
		4647	case PACKET3_DISPATCH_DIRECT:
		4648	case PACKET3_DISPATCH_INDIRECT:
		4649	case PACKET3_ALLOC_GDS:
		4650	case PACKET3_WRITE_GDS_RAM:
		4651	case PACKET3_ATOMIC_GDS:
		4652	case PACKET3_ATOMIC:
		4653	case PACKET3_OCCLUSION_QUERY:
		4654	case PACKET3_SET_PREDICATION:
		4655	case PACKET3_COND_EXEC:
		4656	case PACKET3_PRED_EXEC:
		4657	case PACKET3_CONTEXT_CONTROL:
		4658	case PACKET3_STRMOUT_BUFFER_UPDATE:
		4659	case PACKET3_WAIT_REG_MEM:
		4660	case PACKET3_MEM_WRITE:
		4661	case PACKET3_PFP_SYNC_ME:
		4662	case PACKET3_SURFACE_SYNC:
		4663	case PACKET3_EVENT_WRITE:
		4664	case PACKET3_EVENT_WRITE_EOP:
		4665	case PACKET3_EVENT_WRITE_EOS:
		4666	case PACKET3_SET_CONTEXT_REG:
		4667	case PACKET3_SET_CONTEXT_REG_INDIRECT:
		4668	case PACKET3_SET_SH_REG:
		4669	case PACKET3_SET_SH_REG_OFFSET:
		4670	case PACKET3_INCREMENT_DE_COUNTER:
		4671	case PACKET3_WAIT_ON_CE_COUNTER:
		4672	case PACKET3_WAIT_ON_AVAIL_BUFFER:
		4673	case PACKET3_ME_WRITE:
		4674	break;
		4675	case PACKET3_COPY_DATA:
		4676	if ((idx_value & 0xf00) == 0) {
		4677	reg = ib[idx + 3] * 4;
		4678	if (!si_vm_reg_valid(reg))
		4679	return -EINVAL;
		4680	}
		4681	break;
		4682	case PACKET3_WRITE_DATA:
		4683	if ((idx_value & 0xf00) == 0) {
		4684	start_reg = ib[idx + 1] * 4;
		4685	if (idx_value & 0x10000) {
		4686	if (!si_vm_reg_valid(start_reg))
		4687	return -EINVAL;
		4688	} else {
		4689	for (i = 0; i < (pkt->count - 2); i++) {
		4690	reg = start_reg + (4 * i);
		4691	if (!si_vm_reg_valid(reg))
		4692	return -EINVAL;
		4693	}
		4694	}
		4695	}
		4696	break;
		4697	case PACKET3_COND_WRITE:
		4698	if (idx_value & 0x100) {
		4699	reg = ib[idx + 5] * 4;
		4700	if (!si_vm_reg_valid(reg))
		4701	return -EINVAL;
		4702	}
		4703	break;
		4704	case PACKET3_COPY_DW:
		4705	if (idx_value & 0x2) {
		4706	reg = ib[idx + 3] * 4;
		4707	if (!si_vm_reg_valid(reg))
		4708	return -EINVAL;
		4709	}
		4710	break;
5078	serge	4711	case PACKET3_CP_DMA:
		4712	r = si_vm_packet3_cp_dma_check(ib, idx);
		4713	if (r)
		4714	return r;
		4715	break;
2997	Serge	4716	default:
		4717	DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
		4718	return -EINVAL;
		4719	}
		4720	return 0;
		4721	}
		4722
		4723	int si_ib_parse(struct radeon_device rdev, struct radeon_ib ib)
		4724	{
		4725	int ret = 0;
5271	serge	4726	u32 idx = 0, i;
2997	Serge	4727	struct radeon_cs_packet pkt;
		4728
		4729	do {
		4730	pkt.idx = idx;
3764	Serge	4731	pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
		4732	pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
2997	Serge	4733	pkt.one_reg_wr = 0;
		4734	switch (pkt.type) {
3764	Serge	4735	case RADEON_PACKET_TYPE0:
2997	Serge	4736	dev_err(rdev->dev, "Packet0 not allowed!\n");
		4737	ret = -EINVAL;
		4738	break;
3764	Serge	4739	case RADEON_PACKET_TYPE2:
2997	Serge	4740	idx += 1;
		4741	break;
3764	Serge	4742	case RADEON_PACKET_TYPE3:
		4743	pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
2997	Serge	4744	if (ib->is_const_ib)
		4745	ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
		4746	else {
		4747	switch (ib->ring) {
		4748	case RADEON_RING_TYPE_GFX_INDEX:
		4749	ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
		4750	break;
		4751	case CAYMAN_RING_TYPE_CP1_INDEX:
		4752	case CAYMAN_RING_TYPE_CP2_INDEX:
		4753	ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
		4754	break;
		4755	default:
		4756	dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
		4757	ret = -EINVAL;
		4758	break;
		4759	}
		4760	}
		4761	idx += pkt.count + 2;
		4762	break;
		4763	default:
		4764	dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
		4765	ret = -EINVAL;
		4766	break;
		4767	}
6104	serge	4768	if (ret) {
		4769	for (i = 0; i < ib->length_dw; i++) {
		4770	if (i == idx)
		4771	printk("\t0x%08x <---\n", ib->ptr[i]);
		4772	else
		4773	printk("\t0x%08x\n", ib->ptr[i]);
		4774	}
2997	Serge	4775	break;
6104	serge	4776	}
2997	Serge	4777	} while (idx < ib->length_dw);
		4778
		4779	return ret;
		4780	}
		4781
		4782	/*
		4783	* vm
		4784	*/
		4785	int si_vm_init(struct radeon_device *rdev)
		4786	{
		4787	/* number of VMs */
		4788	rdev->vm_manager.nvm = 16;
		4789	/* base offset of vram pages */
		4790	rdev->vm_manager.vram_base_offset = 0;
		4791
		4792	return 0;
		4793	}
		4794
		4795	void si_vm_fini(struct radeon_device *rdev)
		4796	{
		4797	}
		4798
		4799	/**
5078	serge	4800	* si_vm_decode_fault - print human readable fault info
2997	Serge	4801	*
		4802	* @rdev: radeon_device pointer
5078	serge	4803	* @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
		4804	* @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2997	Serge	4805	*
5078	serge	4806	* Print human readable fault information (SI).
2997	Serge	4807	*/
5078	serge	4808	static void si_vm_decode_fault(struct radeon_device *rdev,
		4809	u32 status, u32 addr)
2997	Serge	4810	{
5078	serge	4811	u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
		4812	u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
		4813	u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
		4814	char *block;
2997	Serge	4815
5078	serge	4816	if (rdev->family == CHIP_TAHITI) {
		4817	switch (mc_id) {
		4818	case 160:
		4819	case 144:
		4820	case 96:
		4821	case 80:
		4822	case 224:
		4823	case 208:
		4824	case 32:
		4825	case 16:
		4826	block = "CB";
		4827	break;
		4828	case 161:
		4829	case 145:
		4830	case 97:
		4831	case 81:
		4832	case 225:
		4833	case 209:
		4834	case 33:
		4835	case 17:
		4836	block = "CB_FMASK";
		4837	break;
		4838	case 162:
		4839	case 146:
		4840	case 98:
		4841	case 82:
		4842	case 226:
		4843	case 210:
		4844	case 34:
		4845	case 18:
		4846	block = "CB_CMASK";
		4847	break;
		4848	case 163:
		4849	case 147:
		4850	case 99:
		4851	case 83:
		4852	case 227:
		4853	case 211:
		4854	case 35:
		4855	case 19:
		4856	block = "CB_IMMED";
		4857	break;
		4858	case 164:
		4859	case 148:
		4860	case 100:
		4861	case 84:
		4862	case 228:
		4863	case 212:
		4864	case 36:
		4865	case 20:
		4866	block = "DB";
		4867	break;
		4868	case 165:
		4869	case 149:
		4870	case 101:
		4871	case 85:
		4872	case 229:
		4873	case 213:
		4874	case 37:
		4875	case 21:
		4876	block = "DB_HTILE";
		4877	break;
		4878	case 167:
		4879	case 151:
		4880	case 103:
		4881	case 87:
		4882	case 231:
		4883	case 215:
		4884	case 39:
		4885	case 23:
		4886	block = "DB_STEN";
		4887	break;
		4888	case 72:
		4889	case 68:
		4890	case 64:
		4891	case 8:
		4892	case 4:
		4893	case 0:
		4894	case 136:
		4895	case 132:
		4896	case 128:
		4897	case 200:
		4898	case 196:
		4899	case 192:
		4900	block = "TC";
		4901	break;
		4902	case 112:
		4903	case 48:
		4904	block = "CP";
		4905	break;
		4906	case 49:
		4907	case 177:
		4908	case 50:
		4909	case 178:
		4910	block = "SH";
		4911	break;
		4912	case 53:
		4913	case 190:
		4914	block = "VGT";
		4915	break;
		4916	case 117:
		4917	block = "IH";
		4918	break;
		4919	case 51:
		4920	case 115:
		4921	block = "RLC";
		4922	break;
		4923	case 119:
		4924	case 183:
		4925	block = "DMA0";
		4926	break;
		4927	case 61:
		4928	block = "DMA1";
		4929	break;
		4930	case 248:
		4931	case 120:
		4932	block = "HDP";
		4933	break;
		4934	default:
		4935	block = "unknown";
		4936	break;
2997	Serge	4937	}
3192	Serge	4938	} else {
5078	serge	4939	switch (mc_id) {
		4940	case 32:
		4941	case 16:
		4942	case 96:
		4943	case 80:
		4944	case 160:
		4945	case 144:
		4946	case 224:
		4947	case 208:
		4948	block = "CB";
		4949	break;
		4950	case 33:
		4951	case 17:
		4952	case 97:
		4953	case 81:
		4954	case 161:
		4955	case 145:
		4956	case 225:
		4957	case 209:
		4958	block = "CB_FMASK";
		4959	break;
		4960	case 34:
		4961	case 18:
		4962	case 98:
		4963	case 82:
		4964	case 162:
		4965	case 146:
		4966	case 226:
		4967	case 210:
		4968	block = "CB_CMASK";
		4969	break;
		4970	case 35:
		4971	case 19:
		4972	case 99:
		4973	case 83:
		4974	case 163:
		4975	case 147:
		4976	case 227:
		4977	case 211:
		4978	block = "CB_IMMED";
		4979	break;
		4980	case 36:
		4981	case 20:
		4982	case 100:
		4983	case 84:
		4984	case 164:
		4985	case 148:
		4986	case 228:
		4987	case 212:
		4988	block = "DB";
		4989	break;
		4990	case 37:
		4991	case 21:
		4992	case 101:
		4993	case 85:
		4994	case 165:
		4995	case 149:
		4996	case 229:
		4997	case 213:
		4998	block = "DB_HTILE";
		4999	break;
		5000	case 39:
		5001	case 23:
		5002	case 103:
		5003	case 87:
		5004	case 167:
		5005	case 151:
		5006	case 231:
		5007	case 215:
		5008	block = "DB_STEN";
		5009	break;
		5010	case 72:
		5011	case 68:
		5012	case 8:
		5013	case 4:
		5014	case 136:
		5015	case 132:
		5016	case 200:
		5017	case 196:
		5018	block = "TC";
		5019	break;
		5020	case 112:
		5021	case 48:
		5022	block = "CP";
		5023	break;
		5024	case 49:
		5025	case 177:
		5026	case 50:
		5027	case 178:
		5028	block = "SH";
		5029	break;
		5030	case 53:
		5031	block = "VGT";
		5032	break;
		5033	case 117:
		5034	block = "IH";
		5035	break;
		5036	case 51:
		5037	case 115:
		5038	block = "RLC";
		5039	break;
		5040	case 119:
		5041	case 183:
		5042	block = "DMA0";
		5043	break;
		5044	case 61:
		5045	block = "DMA1";
		5046	break;
		5047	case 248:
		5048	case 120:
		5049	block = "HDP";
		5050	break;
		5051	default:
		5052	block = "unknown";
		5053	break;
3192	Serge	5054	}
		5055	}
		5056
5078	serge	5057	printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
		5058	protections, vmid, addr,
		5059	(status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
		5060	block, mc_id);
2997	Serge	5061	}
		5062
5271	serge	5063	void si_vm_flush(struct radeon_device rdev, struct radeon_ring ring,
		5064	unsigned vm_id, uint64_t pd_addr)
2997	Serge	5065	{
		5066	/* write new base address */
		5067	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5078	serge	5068	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) \|
2997	Serge	5069	WRITE_DATA_DST_SEL(0)));
		5070
5271	serge	5071	if (vm_id < 8) {
2997	Serge	5072	radeon_ring_write(ring,
5271	serge	5073	(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
2997	Serge	5074	} else {
		5075	radeon_ring_write(ring,
5271	serge	5076	(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
2997	Serge	5077	}
		5078	radeon_ring_write(ring, 0);
5271	serge	5079	radeon_ring_write(ring, pd_addr >> 12);
2997	Serge	5080
		5081	/* flush hdp cache */
		5082	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5078	serge	5083	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) \|
2997	Serge	5084	WRITE_DATA_DST_SEL(0)));
		5085	radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
		5086	radeon_ring_write(ring, 0);
		5087	radeon_ring_write(ring, 0x1);
		5088
		5089	/* bits 0-15 are the VM contexts0-15 */
		5090	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5078	serge	5091	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) \|
2997	Serge	5092	WRITE_DATA_DST_SEL(0)));
		5093	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
		5094	radeon_ring_write(ring, 0);
5271	serge	5095	radeon_ring_write(ring, 1 << vm_id);
2997	Serge	5096
6104	serge	5097	/* wait for the invalidate to complete */
		5098	radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
		5099	radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) \| /* always */
		5100	WAIT_REG_MEM_ENGINE(0))); /* me */
		5101	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
		5102	radeon_ring_write(ring, 0);
		5103	radeon_ring_write(ring, 0); /* ref */
		5104	radeon_ring_write(ring, 0); /* mask */
		5105	radeon_ring_write(ring, 0x20); /* poll interval */
		5106
2997	Serge	5107	/* sync PFP to ME, otherwise we might get invalid PFP reads */
		5108	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
		5109	radeon_ring_write(ring, 0x0);
		5110	}
		5111
5078	serge	5112	/*
		5113	* Power and clock gating
		5114	*/
		5115	static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
3192	Serge	5116	{
5078	serge	5117	int i;
3192	Serge	5118
5078	serge	5119	for (i = 0; i < rdev->usec_timeout; i++) {
		5120	if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
		5121	break;
		5122	udelay(1);
		5123	}
3192	Serge	5124
5078	serge	5125	for (i = 0; i < rdev->usec_timeout; i++) {
		5126	if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
		5127	break;
		5128	udelay(1);
		5129	}
		5130	}
		5131
		5132	static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
		5133	bool enable)
		5134	{
		5135	u32 tmp = RREG32(CP_INT_CNTL_RING0);
		5136	u32 mask;
		5137	int i;
		5138
		5139	if (enable)
		5140	tmp \|= (CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		5141	else
		5142	tmp &= ~(CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		5143	WREG32(CP_INT_CNTL_RING0, tmp);
		5144
		5145	if (!enable) {
		5146	/* read a gfx register */
		5147	tmp = RREG32(DB_DEPTH_INFO);
		5148
		5149	mask = RLC_BUSY_STATUS \| GFX_POWER_STATUS \| GFX_CLOCK_STATUS \| GFX_LS_STATUS;
		5150	for (i = 0; i < rdev->usec_timeout; i++) {
		5151	if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS \| GFX_POWER_STATUS))
		5152	break;
		5153	udelay(1);
		5154	}
		5155	}
		5156	}
		5157
		5158	static void si_set_uvd_dcm(struct radeon_device *rdev,
		5159	bool sw_mode)
		5160	{
		5161	u32 tmp, tmp2;
		5162
		5163	tmp = RREG32(UVD_CGC_CTRL);
		5164	tmp &= ~(CLK_OD_MASK \| CG_DT_MASK);
		5165	tmp \|= DCM \| CG_DT(1) \| CLK_OD(4);
		5166
		5167	if (sw_mode) {
		5168	tmp &= ~0x7ffff800;
		5169	tmp2 = DYN_OR_EN \| DYN_RR_EN \| G_DIV_ID(7);
3192	Serge	5170	} else {
5078	serge	5171	tmp \|= 0x7ffff800;
		5172	tmp2 = 0;
3192	Serge	5173	}
		5174
5078	serge	5175	WREG32(UVD_CGC_CTRL, tmp);
		5176	WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
		5177	}
3192	Serge	5178
5078	serge	5179	void si_init_uvd_internal_cg(struct radeon_device *rdev)
		5180	{
		5181	bool hw_mode = true;
		5182
		5183	if (hw_mode) {
		5184	si_set_uvd_dcm(rdev, false);
		5185	} else {
		5186	u32 tmp = RREG32(UVD_CGC_CTRL);
		5187	tmp &= ~DCM;
		5188	WREG32(UVD_CGC_CTRL, tmp);
		5189	}
3192	Serge	5190	}
		5191
5078	serge	5192	static u32 si_halt_rlc(struct radeon_device *rdev)
2997	Serge	5193	{
5078	serge	5194	u32 data, orig;
2997	Serge	5195
5078	serge	5196	orig = data = RREG32(RLC_CNTL);
2997	Serge	5197
5078	serge	5198	if (data & RLC_ENABLE) {
		5199	data &= ~RLC_ENABLE;
		5200	WREG32(RLC_CNTL, data);
		5201
		5202	si_wait_for_rlc_serdes(rdev);
2997	Serge	5203	}
		5204
5078	serge	5205	return orig;
		5206	}
2997	Serge	5207
5078	serge	5208	static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
		5209	{
		5210	u32 tmp;
		5211
		5212	tmp = RREG32(RLC_CNTL);
		5213	if (tmp != rlc)
		5214	WREG32(RLC_CNTL, rlc);
		5215	}
		5216
		5217	static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
		5218	{
		5219	u32 data, orig;
		5220
		5221	orig = data = RREG32(DMA_PG);
		5222	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
		5223	data \|= PG_CNTL_ENABLE;
		5224	else
		5225	data &= ~PG_CNTL_ENABLE;
		5226	if (orig != data)
		5227	WREG32(DMA_PG, data);
		5228	}
		5229
		5230	static void si_init_dma_pg(struct radeon_device *rdev)
		5231	{
		5232	u32 tmp;
		5233
		5234	WREG32(DMA_PGFSM_WRITE, 0x00002000);
		5235	WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
		5236
		5237	for (tmp = 0; tmp < 5; tmp++)
		5238	WREG32(DMA_PGFSM_WRITE, 0);
		5239	}
		5240
		5241	static void si_enable_gfx_cgpg(struct radeon_device *rdev,
		5242	bool enable)
		5243	{
		5244	u32 tmp;
		5245
		5246	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
		5247	tmp = RLC_PUD(0x10) \| RLC_PDD(0x10) \| RLC_TTPD(0x10) \| RLC_MSD(0x10);
		5248	WREG32(RLC_TTOP_D, tmp);
		5249
		5250	tmp = RREG32(RLC_PG_CNTL);
		5251	tmp \|= GFX_PG_ENABLE;
		5252	WREG32(RLC_PG_CNTL, tmp);
		5253
		5254	tmp = RREG32(RLC_AUTO_PG_CTRL);
		5255	tmp \|= AUTO_PG_EN;
		5256	WREG32(RLC_AUTO_PG_CTRL, tmp);
		5257	} else {
		5258	tmp = RREG32(RLC_AUTO_PG_CTRL);
		5259	tmp &= ~AUTO_PG_EN;
		5260	WREG32(RLC_AUTO_PG_CTRL, tmp);
		5261
		5262	tmp = RREG32(DB_RENDER_CONTROL);
2997	Serge	5263	}
		5264	}
		5265
5078	serge	5266	static void si_init_gfx_cgpg(struct radeon_device *rdev)
2997	Serge	5267	{
5078	serge	5268	u32 tmp;
2997	Serge	5269
5078	serge	5270	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		5271
		5272	tmp = RREG32(RLC_PG_CNTL);
		5273	tmp \|= GFX_PG_SRC;
		5274	WREG32(RLC_PG_CNTL, tmp);
		5275
		5276	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		5277
		5278	tmp = RREG32(RLC_AUTO_PG_CTRL);
		5279
		5280	tmp &= ~GRBM_REG_SGIT_MASK;
		5281	tmp \|= GRBM_REG_SGIT(0x700);
		5282	tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
		5283	WREG32(RLC_AUTO_PG_CTRL, tmp);
		5284	}
		5285
		5286	static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
		5287	{
		5288	u32 mask = 0, tmp, tmp1;
		5289	int i;
		5290
		5291	si_select_se_sh(rdev, se, sh);
		5292	tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
		5293	tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
		5294	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		5295
		5296	tmp &= 0xffff0000;
		5297
		5298	tmp \|= tmp1;
		5299	tmp >>= 16;
		5300
		5301	for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
		5302	mask <<= 1;
		5303	mask \|= 1;
		5304	}
		5305
		5306	return (~tmp) & mask;
		5307	}
		5308
		5309	static void si_init_ao_cu_mask(struct radeon_device *rdev)
		5310	{
		5311	u32 i, j, k, active_cu_number = 0;
		5312	u32 mask, counter, cu_bitmap;
		5313	u32 tmp = 0;
		5314
		5315	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
		5316	for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
		5317	mask = 1;
		5318	cu_bitmap = 0;
		5319	counter = 0;
		5320	for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
		5321	if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
		5322	if (counter < 2)
		5323	cu_bitmap \|= mask;
		5324	counter++;
		5325	}
		5326	mask <<= 1;
		5327	}
		5328
		5329	active_cu_number += counter;
		5330	tmp \|= (cu_bitmap << (i * 16 + j * 8));
2997	Serge	5331	}
		5332	}
		5333
5078	serge	5334	WREG32(RLC_PG_AO_CU_MASK, tmp);
		5335
		5336	tmp = RREG32(RLC_MAX_PG_CU);
		5337	tmp &= ~MAX_PU_CU_MASK;
		5338	tmp \|= MAX_PU_CU(active_cu_number);
		5339	WREG32(RLC_MAX_PG_CU, tmp);
		5340	}
		5341
		5342	static void si_enable_cgcg(struct radeon_device *rdev,
		5343	bool enable)
		5344	{
		5345	u32 data, orig, tmp;
		5346
		5347	orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
		5348
		5349	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
		5350	si_enable_gui_idle_interrupt(rdev, true);
		5351
		5352	WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
		5353
		5354	tmp = si_halt_rlc(rdev);
		5355
		5356	WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
		5357	WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
		5358	WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
		5359
		5360	si_wait_for_rlc_serdes(rdev);
		5361
		5362	si_update_rlc(rdev, tmp);
		5363
		5364	WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
		5365
		5366	data \|= CGCG_EN \| CGLS_EN;
		5367	} else {
		5368	si_enable_gui_idle_interrupt(rdev, false);
		5369
		5370	RREG32(CB_CGTT_SCLK_CTRL);
		5371	RREG32(CB_CGTT_SCLK_CTRL);
		5372	RREG32(CB_CGTT_SCLK_CTRL);
		5373	RREG32(CB_CGTT_SCLK_CTRL);
		5374
		5375	data &= ~(CGCG_EN \| CGLS_EN);
2997	Serge	5376	}
5078	serge	5377
		5378	if (orig != data)
		5379	WREG32(RLC_CGCG_CGLS_CTRL, data);
		5380	}
		5381
		5382	static void si_enable_mgcg(struct radeon_device *rdev,
		5383	bool enable)
		5384	{
		5385	u32 data, orig, tmp = 0;
		5386
		5387	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
		5388	orig = data = RREG32(CGTS_SM_CTRL_REG);
		5389	data = 0x96940200;
		5390	if (orig != data)
		5391	WREG32(CGTS_SM_CTRL_REG, data);
		5392
		5393	if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
		5394	orig = data = RREG32(CP_MEM_SLP_CNTL);
		5395	data \|= CP_MEM_LS_EN;
		5396	if (orig != data)
		5397	WREG32(CP_MEM_SLP_CNTL, data);
		5398	}
		5399
		5400	orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
		5401	data &= 0xffffffc0;
		5402	if (orig != data)
		5403	WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
		5404
		5405	tmp = si_halt_rlc(rdev);
		5406
		5407	WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
		5408	WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
		5409	WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
		5410
		5411	si_update_rlc(rdev, tmp);
		5412	} else {
		5413	orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
		5414	data \|= 0x00000003;
		5415	if (orig != data)
		5416	WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
		5417
		5418	data = RREG32(CP_MEM_SLP_CNTL);
		5419	if (data & CP_MEM_LS_EN) {
		5420	data &= ~CP_MEM_LS_EN;
		5421	WREG32(CP_MEM_SLP_CNTL, data);
		5422	}
		5423	orig = data = RREG32(CGTS_SM_CTRL_REG);
		5424	data \|= LS_OVERRIDE \| OVERRIDE;
		5425	if (orig != data)
		5426	WREG32(CGTS_SM_CTRL_REG, data);
		5427
		5428	tmp = si_halt_rlc(rdev);
		5429
		5430	WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
		5431	WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
		5432	WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
		5433
		5434	si_update_rlc(rdev, tmp);
2997	Serge	5435	}
5078	serge	5436	}
2997	Serge	5437
5078	serge	5438	static void si_enable_uvd_mgcg(struct radeon_device *rdev,
		5439	bool enable)
		5440	{
		5441	u32 orig, data, tmp;
		5442
		5443	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
		5444	tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
		5445	tmp \|= 0x3fff;
		5446	WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
		5447
		5448	orig = data = RREG32(UVD_CGC_CTRL);
		5449	data \|= DCM;
		5450	if (orig != data)
		5451	WREG32(UVD_CGC_CTRL, data);
		5452
		5453	WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
		5454	WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
		5455	} else {
		5456	tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
		5457	tmp &= ~0x3fff;
		5458	WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
		5459
		5460	orig = data = RREG32(UVD_CGC_CTRL);
		5461	data &= ~DCM;
		5462	if (orig != data)
		5463	WREG32(UVD_CGC_CTRL, data);
		5464
		5465	WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
		5466	WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
		5467	}
		5468	}
		5469
		5470	static const u32 mc_cg_registers[] =
		5471	{
		5472	MC_HUB_MISC_HUB_CG,
		5473	MC_HUB_MISC_SIP_CG,
		5474	MC_HUB_MISC_VM_CG,
		5475	MC_XPB_CLK_GAT,
		5476	ATC_MISC_CG,
		5477	MC_CITF_MISC_WR_CG,
		5478	MC_CITF_MISC_RD_CG,
		5479	MC_CITF_MISC_VM_CG,
		5480	VM_L2_CG,
		5481	};
		5482
		5483	static void si_enable_mc_ls(struct radeon_device *rdev,
		5484	bool enable)
		5485	{
		5486	int i;
		5487	u32 orig, data;
		5488
		5489	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
		5490	orig = data = RREG32(mc_cg_registers[i]);
		5491	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
		5492	data \|= MC_LS_ENABLE;
		5493	else
		5494	data &= ~MC_LS_ENABLE;
		5495	if (data != orig)
		5496	WREG32(mc_cg_registers[i], data);
		5497	}
		5498	}
		5499
		5500	static void si_enable_mc_mgcg(struct radeon_device *rdev,
		5501	bool enable)
		5502	{
		5503	int i;
		5504	u32 orig, data;
		5505
		5506	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
		5507	orig = data = RREG32(mc_cg_registers[i]);
		5508	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
		5509	data \|= MC_CG_ENABLE;
		5510	else
		5511	data &= ~MC_CG_ENABLE;
		5512	if (data != orig)
		5513	WREG32(mc_cg_registers[i], data);
		5514	}
		5515	}
		5516
		5517	static void si_enable_dma_mgcg(struct radeon_device *rdev,
		5518	bool enable)
		5519	{
		5520	u32 orig, data, offset;
		5521	int i;
		5522
		5523	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
		5524	for (i = 0; i < 2; i++) {
		5525	if (i == 0)
		5526	offset = DMA0_REGISTER_OFFSET;
		5527	else
		5528	offset = DMA1_REGISTER_OFFSET;
		5529	orig = data = RREG32(DMA_POWER_CNTL + offset);
		5530	data &= ~MEM_POWER_OVERRIDE;
		5531	if (data != orig)
		5532	WREG32(DMA_POWER_CNTL + offset, data);
		5533	WREG32(DMA_CLK_CTRL + offset, 0x00000100);
2997	Serge	5534	}
5078	serge	5535	} else {
		5536	for (i = 0; i < 2; i++) {
		5537	if (i == 0)
		5538	offset = DMA0_REGISTER_OFFSET;
		5539	else
		5540	offset = DMA1_REGISTER_OFFSET;
		5541	orig = data = RREG32(DMA_POWER_CNTL + offset);
		5542	data \|= MEM_POWER_OVERRIDE;
		5543	if (data != orig)
		5544	WREG32(DMA_POWER_CNTL + offset, data);
		5545
		5546	orig = data = RREG32(DMA_CLK_CTRL + offset);
		5547	data = 0xff000000;
		5548	if (data != orig)
		5549	WREG32(DMA_CLK_CTRL + offset, data);
		5550	}
2997	Serge	5551	}
5078	serge	5552	}
		5553
		5554	static void si_enable_bif_mgls(struct radeon_device *rdev,
		5555	bool enable)
		5556	{
		5557	u32 orig, data;
		5558
		5559	orig = data = RREG32_PCIE(PCIE_CNTL2);
		5560
		5561	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
		5562	data \|= SLV_MEM_LS_EN \| MST_MEM_LS_EN \|
		5563	REPLAY_MEM_LS_EN \| SLV_MEM_AGGRESSIVE_LS_EN;
		5564	else
		5565	data &= ~(SLV_MEM_LS_EN \| MST_MEM_LS_EN \|
		5566	REPLAY_MEM_LS_EN \| SLV_MEM_AGGRESSIVE_LS_EN);
		5567
		5568	if (orig != data)
		5569	WREG32_PCIE(PCIE_CNTL2, data);
		5570	}
		5571
		5572	static void si_enable_hdp_mgcg(struct radeon_device *rdev,
		5573	bool enable)
		5574	{
		5575	u32 orig, data;
		5576
		5577	orig = data = RREG32(HDP_HOST_PATH_CNTL);
		5578
		5579	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
		5580	data &= ~CLOCK_GATING_DIS;
		5581	else
		5582	data \|= CLOCK_GATING_DIS;
		5583
		5584	if (orig != data)
		5585	WREG32(HDP_HOST_PATH_CNTL, data);
		5586	}
		5587
		5588	static void si_enable_hdp_ls(struct radeon_device *rdev,
		5589	bool enable)
		5590	{
		5591	u32 orig, data;
		5592
		5593	orig = data = RREG32(HDP_MEM_POWER_LS);
		5594
		5595	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
		5596	data \|= HDP_LS_ENABLE;
		5597	else
		5598	data &= ~HDP_LS_ENABLE;
		5599
		5600	if (orig != data)
		5601	WREG32(HDP_MEM_POWER_LS, data);
		5602	}
		5603
		5604	static void si_update_cg(struct radeon_device *rdev,
6104	serge	5605	u32 block, bool enable)
5078	serge	5606	{
		5607	if (block & RADEON_CG_BLOCK_GFX) {
		5608	si_enable_gui_idle_interrupt(rdev, false);
		5609	/* order matters! */
		5610	if (enable) {
		5611	si_enable_mgcg(rdev, true);
		5612	si_enable_cgcg(rdev, true);
		5613	} else {
		5614	si_enable_cgcg(rdev, false);
		5615	si_enable_mgcg(rdev, false);
		5616	}
		5617	si_enable_gui_idle_interrupt(rdev, true);
2997	Serge	5618	}
5078	serge	5619
		5620	if (block & RADEON_CG_BLOCK_MC) {
		5621	si_enable_mc_mgcg(rdev, enable);
		5622	si_enable_mc_ls(rdev, enable);
2997	Serge	5623	}
		5624
5078	serge	5625	if (block & RADEON_CG_BLOCK_SDMA) {
		5626	si_enable_dma_mgcg(rdev, enable);
		5627	}
		5628
		5629	if (block & RADEON_CG_BLOCK_BIF) {
		5630	si_enable_bif_mgls(rdev, enable);
		5631	}
		5632
		5633	if (block & RADEON_CG_BLOCK_UVD) {
		5634	if (rdev->has_uvd) {
		5635	si_enable_uvd_mgcg(rdev, enable);
		5636	}
		5637	}
		5638
		5639	if (block & RADEON_CG_BLOCK_HDP) {
		5640	si_enable_hdp_mgcg(rdev, enable);
		5641	si_enable_hdp_ls(rdev, enable);
		5642	}
		5643	}
		5644
		5645	static void si_init_cg(struct radeon_device *rdev)
		5646	{
		5647	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX \|
		5648	RADEON_CG_BLOCK_MC \|
		5649	RADEON_CG_BLOCK_SDMA \|
		5650	RADEON_CG_BLOCK_BIF \|
		5651	RADEON_CG_BLOCK_HDP), true);
		5652	if (rdev->has_uvd) {
		5653	si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
		5654	si_init_uvd_internal_cg(rdev);
6104	serge	5655	}
5078	serge	5656	}
		5657
		5658	static void si_fini_cg(struct radeon_device *rdev)
		5659	{
		5660	if (rdev->has_uvd) {
		5661	si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
		5662	}
		5663	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX \|
		5664	RADEON_CG_BLOCK_MC \|
		5665	RADEON_CG_BLOCK_SDMA \|
		5666	RADEON_CG_BLOCK_BIF \|
		5667	RADEON_CG_BLOCK_HDP), false);
		5668	}
		5669
		5670	u32 si_get_csb_size(struct radeon_device *rdev)
		5671	{
		5672	u32 count = 0;
		5673	const struct cs_section_def *sect = NULL;
		5674	const struct cs_extent_def *ext = NULL;
		5675
		5676	if (rdev->rlc.cs_data == NULL)
6104	serge	5677	return 0;
5078	serge	5678
		5679	/* begin clear state */
		5680	count += 2;
		5681	/* context control state */
		5682	count += 3;
		5683
		5684	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
		5685	for (ext = sect->section; ext->extent != NULL; ++ext) {
		5686	if (sect->id == SECT_CONTEXT)
		5687	count += 2 + ext->reg_count;
		5688	else
		5689	return 0;
		5690	}
		5691	}
		5692	/* pa_sc_raster_config */
		5693	count += 3;
		5694	/* end clear state */
		5695	count += 2;
		5696	/* clear state */
		5697	count += 2;
		5698
		5699	return count;
2997	Serge	5700	}
		5701
5078	serge	5702	void si_get_csb_buffer(struct radeon_device rdev, volatile u32 buffer)
		5703	{
		5704	u32 count = 0, i;
		5705	const struct cs_section_def *sect = NULL;
		5706	const struct cs_extent_def *ext = NULL;
		5707
		5708	if (rdev->rlc.cs_data == NULL)
		5709	return;
		5710	if (buffer == NULL)
		5711	return;
		5712
		5713	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		5714	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
		5715
		5716	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
		5717	buffer[count++] = cpu_to_le32(0x80000000);
		5718	buffer[count++] = cpu_to_le32(0x80000000);
		5719
		5720	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
		5721	for (ext = sect->section; ext->extent != NULL; ++ext) {
		5722	if (sect->id == SECT_CONTEXT) {
		5723	buffer[count++] =
		5724	cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
		5725	buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
		5726	for (i = 0; i < ext->reg_count; i++)
		5727	buffer[count++] = cpu_to_le32(ext->extent[i]);
		5728	} else {
		5729	return;
		5730	}
		5731	}
		5732	}
		5733
		5734	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
		5735	buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
		5736	switch (rdev->family) {
		5737	case CHIP_TAHITI:
		5738	case CHIP_PITCAIRN:
		5739	buffer[count++] = cpu_to_le32(0x2a00126a);
		5740	break;
		5741	case CHIP_VERDE:
		5742	buffer[count++] = cpu_to_le32(0x0000124a);
		5743	break;
		5744	case CHIP_OLAND:
		5745	buffer[count++] = cpu_to_le32(0x00000082);
		5746	break;
		5747	case CHIP_HAINAN:
		5748	buffer[count++] = cpu_to_le32(0x00000000);
		5749	break;
		5750	default:
		5751	buffer[count++] = cpu_to_le32(0x00000000);
		5752	break;
		5753	}
		5754
		5755	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		5756	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
		5757
		5758	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
		5759	buffer[count++] = cpu_to_le32(0);
		5760	}
		5761
		5762	static void si_init_pg(struct radeon_device *rdev)
		5763	{
		5764	if (rdev->pg_flags) {
		5765	if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
		5766	si_init_dma_pg(rdev);
		5767	}
		5768	si_init_ao_cu_mask(rdev);
		5769	if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
		5770	si_init_gfx_cgpg(rdev);
		5771	} else {
		5772	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		5773	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		5774	}
		5775	si_enable_dma_pg(rdev, true);
		5776	si_enable_gfx_cgpg(rdev, true);
		5777	} else {
		5778	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		5779	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		5780	}
		5781	}
		5782
		5783	static void si_fini_pg(struct radeon_device *rdev)
		5784	{
		5785	if (rdev->pg_flags) {
		5786	si_enable_dma_pg(rdev, false);
		5787	si_enable_gfx_cgpg(rdev, false);
		5788	}
		5789	}
		5790
		5791	/*
		5792	* RLC
		5793	*/
		5794	void si_rlc_reset(struct radeon_device *rdev)
		5795	{
		5796	u32 tmp = RREG32(GRBM_SOFT_RESET);
		5797
		5798	tmp \|= SOFT_RESET_RLC;
		5799	WREG32(GRBM_SOFT_RESET, tmp);
		5800	udelay(50);
		5801	tmp &= ~SOFT_RESET_RLC;
		5802	WREG32(GRBM_SOFT_RESET, tmp);
		5803	udelay(50);
		5804	}
		5805
2997	Serge	5806	static void si_rlc_stop(struct radeon_device *rdev)
		5807	{
		5808	WREG32(RLC_CNTL, 0);
5078	serge	5809
		5810	si_enable_gui_idle_interrupt(rdev, false);
		5811
		5812	si_wait_for_rlc_serdes(rdev);
2997	Serge	5813	}
		5814
		5815	static void si_rlc_start(struct radeon_device *rdev)
		5816	{
		5817	WREG32(RLC_CNTL, RLC_ENABLE);
5078	serge	5818
		5819	si_enable_gui_idle_interrupt(rdev, true);
		5820
		5821	udelay(50);
2997	Serge	5822	}
		5823
5078	serge	5824	static bool si_lbpw_supported(struct radeon_device *rdev)
		5825	{
		5826	u32 tmp;
		5827
		5828	/* Enable LBPW only for DDR3 */
		5829	tmp = RREG32(MC_SEQ_MISC0);
		5830	if ((tmp & 0xF0000000) == 0xB0000000)
		5831	return true;
		5832	return false;
		5833	}
		5834
		5835	static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
		5836	{
		5837	u32 tmp;
		5838
		5839	tmp = RREG32(RLC_LB_CNTL);
		5840	if (enable)
		5841	tmp \|= LOAD_BALANCE_ENABLE;
		5842	else
		5843	tmp &= ~LOAD_BALANCE_ENABLE;
		5844	WREG32(RLC_LB_CNTL, tmp);
		5845
		5846	if (!enable) {
		5847	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		5848	WREG32(SPI_LB_CU_MASK, 0x00ff);
		5849	}
		5850	}
		5851
2997	Serge	5852	static int si_rlc_resume(struct radeon_device *rdev)
		5853	{
		5854	u32 i;
		5855
		5856	if (!rdev->rlc_fw)
		5857	return -EINVAL;
		5858
		5859	si_rlc_stop(rdev);
		5860
5078	serge	5861	si_rlc_reset(rdev);
		5862
		5863	si_init_pg(rdev);
		5864
		5865	si_init_cg(rdev);
		5866
2997	Serge	5867	WREG32(RLC_RL_BASE, 0);
		5868	WREG32(RLC_RL_SIZE, 0);
		5869	WREG32(RLC_LB_CNTL, 0);
		5870	WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
		5871	WREG32(RLC_LB_CNTR_INIT, 0);
5078	serge	5872	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
2997	Serge	5873
		5874	WREG32(RLC_MC_CNTL, 0);
		5875	WREG32(RLC_UCODE_CNTL, 0);
		5876
5078	serge	5877	if (rdev->new_fw) {
		5878	const struct rlc_firmware_header_v1_0 *hdr =
		5879	(const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
		5880	u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
		5881	const __le32 fw_data = (const __le32 )
		5882	(rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
		5883
		5884	radeon_ucode_print_rlc_hdr(&hdr->header);
		5885
		5886	for (i = 0; i < fw_size; i++) {
		5887	WREG32(RLC_UCODE_ADDR, i);
		5888	WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
		5889	}
		5890	} else {
		5891	const __be32 *fw_data =
		5892	(const __be32 *)rdev->rlc_fw->data;
6104	serge	5893	for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
		5894	WREG32(RLC_UCODE_ADDR, i);
		5895	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		5896	}
2997	Serge	5897	}
		5898	WREG32(RLC_UCODE_ADDR, 0);
		5899
5078	serge	5900	si_enable_lbpw(rdev, si_lbpw_supported(rdev));
		5901
2997	Serge	5902	si_rlc_start(rdev);
		5903
		5904	return 0;
		5905	}
		5906
		5907	static void si_enable_interrupts(struct radeon_device *rdev)
		5908	{
		5909	u32 ih_cntl = RREG32(IH_CNTL);
		5910	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		5911
		5912	ih_cntl \|= ENABLE_INTR;
		5913	ih_rb_cntl \|= IH_RB_ENABLE;
		5914	WREG32(IH_CNTL, ih_cntl);
		5915	WREG32(IH_RB_CNTL, ih_rb_cntl);
		5916	rdev->ih.enabled = true;
		5917	}
		5918
		5919	static void si_disable_interrupts(struct radeon_device *rdev)
		5920	{
		5921	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		5922	u32 ih_cntl = RREG32(IH_CNTL);
		5923
		5924	ih_rb_cntl &= ~IH_RB_ENABLE;
		5925	ih_cntl &= ~ENABLE_INTR;
		5926	WREG32(IH_RB_CNTL, ih_rb_cntl);
		5927	WREG32(IH_CNTL, ih_cntl);
		5928	/* set rptr, wptr to 0 */
		5929	WREG32(IH_RB_RPTR, 0);
		5930	WREG32(IH_RB_WPTR, 0);
		5931	rdev->ih.enabled = false;
		5932	rdev->ih.rptr = 0;
		5933	}
		5934
		5935	static void si_disable_interrupt_state(struct radeon_device *rdev)
		5936	{
		5937	u32 tmp;
		5938
5078	serge	5939	tmp = RREG32(CP_INT_CNTL_RING0) &
		5940	(CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		5941	WREG32(CP_INT_CNTL_RING0, tmp);
2997	Serge	5942	WREG32(CP_INT_CNTL_RING1, 0);
		5943	WREG32(CP_INT_CNTL_RING2, 0);
3192	Serge	5944	tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
		5945	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
		5946	tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
		5947	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
2997	Serge	5948	WREG32(GRBM_INT_CNTL, 0);
6104	serge	5949	WREG32(SRBM_INT_CNTL, 0);
3764	Serge	5950	if (rdev->num_crtc >= 2) {
6104	serge	5951	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		5952	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
3764	Serge	5953	}
2997	Serge	5954	if (rdev->num_crtc >= 4) {
		5955	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		5956	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		5957	}
		5958	if (rdev->num_crtc >= 6) {
		5959	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		5960	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		5961	}
		5962
3764	Serge	5963	if (rdev->num_crtc >= 2) {
6104	serge	5964	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		5965	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
3764	Serge	5966	}
2997	Serge	5967	if (rdev->num_crtc >= 4) {
		5968	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		5969	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		5970	}
		5971	if (rdev->num_crtc >= 6) {
		5972	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		5973	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		5974	}
		5975
3764	Serge	5976	if (!ASIC_IS_NODCE(rdev)) {
5078	serge	5977	WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
2997	Serge	5978
6104	serge	5979	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5980	WREG32(DC_HPD1_INT_CONTROL, tmp);
		5981	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5982	WREG32(DC_HPD2_INT_CONTROL, tmp);
		5983	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5984	WREG32(DC_HPD3_INT_CONTROL, tmp);
		5985	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5986	WREG32(DC_HPD4_INT_CONTROL, tmp);
		5987	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5988	WREG32(DC_HPD5_INT_CONTROL, tmp);
		5989	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5990	WREG32(DC_HPD6_INT_CONTROL, tmp);
3764	Serge	5991	}
2997	Serge	5992	}
		5993
		5994	static int si_irq_init(struct radeon_device *rdev)
		5995	{
		5996	int ret = 0;
		5997	int rb_bufsz;
		5998	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
		5999
		6000	/* allocate ring */
		6001	ret = r600_ih_ring_alloc(rdev);
		6002	if (ret)
		6003	return ret;
		6004
		6005	/* disable irqs */
		6006	si_disable_interrupts(rdev);
		6007
		6008	/* init rlc */
		6009	ret = si_rlc_resume(rdev);
		6010	if (ret) {
		6011	r600_ih_ring_fini(rdev);
		6012	return ret;
		6013	}
		6014
		6015	/* setup interrupt control */
		6016	/* set dummy read address to ring address */
		6017	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
		6018	interrupt_cntl = RREG32(INTERRUPT_CNTL);
		6019	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
		6020	* IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
		6021	*/
		6022	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
		6023	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
		6024	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
		6025	WREG32(INTERRUPT_CNTL, interrupt_cntl);
		6026
		6027	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5078	serge	6028	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
2997	Serge	6029
		6030	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE \|
		6031	IH_WPTR_OVERFLOW_CLEAR \|
		6032	(rb_bufsz << 1));
		6033
		6034	if (rdev->wb.enabled)
		6035	ih_rb_cntl \|= IH_WPTR_WRITEBACK_ENABLE;
		6036
		6037	/* set the writeback address whether it's enabled or not */
		6038	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
		6039	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
		6040
		6041	WREG32(IH_RB_CNTL, ih_rb_cntl);
		6042
		6043	/* set rptr, wptr to 0 */
		6044	WREG32(IH_RB_RPTR, 0);
		6045	WREG32(IH_RB_WPTR, 0);
		6046
		6047	/* Default settings for IH_CNTL (disabled at first) */
		6048	ih_cntl = MC_WRREQ_CREDIT(0x10) \| MC_WR_CLEAN_CNT(0x10) \| MC_VMID(0);
		6049	/* RPTR_REARM only works if msi's are enabled */
		6050	if (rdev->msi_enabled)
		6051	ih_cntl \|= RPTR_REARM;
		6052	WREG32(IH_CNTL, ih_cntl);
		6053
		6054	/* force the active interrupt state to all disabled */
		6055	si_disable_interrupt_state(rdev);
		6056
		6057	pci_set_master(rdev->pdev);
		6058
		6059	/* enable irqs */
		6060	si_enable_interrupts(rdev);
		6061
		6062	return ret;
		6063	}
		6064
		6065	int si_irq_set(struct radeon_device *rdev)
		6066	{
5078	serge	6067	u32 cp_int_cntl;
2997	Serge	6068	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
		6069	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
3764	Serge	6070	u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2997	Serge	6071	u32 grbm_int_cntl = 0;
3192	Serge	6072	u32 dma_cntl, dma_cntl1;
5078	serge	6073	u32 thermal_int = 0;
2997	Serge	6074
		6075	if (!rdev->irq.installed) {
		6076	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		6077	return -EINVAL;
		6078	}
		6079	/* don't enable anything if the ih is disabled */
		6080	if (!rdev->ih.enabled) {
		6081	si_disable_interrupts(rdev);
		6082	/* force the active interrupt state to all disabled */
		6083	si_disable_interrupt_state(rdev);
		6084	return 0;
		6085	}
		6086
5078	serge	6087	cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
		6088	(CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		6089
3764	Serge	6090	if (!ASIC_IS_NODCE(rdev)) {
6104	serge	6091	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
		6092	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
		6093	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
		6094	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
		6095	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
		6096	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
3764	Serge	6097	}
2997	Serge	6098
3192	Serge	6099	dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
		6100	dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
		6101
5078	serge	6102	thermal_int = RREG32(CG_THERMAL_INT) &
		6103	~(THERM_INT_MASK_HIGH \| THERM_INT_MASK_LOW);
		6104
2997	Serge	6105	/* enable CP interrupts on all rings */
		6106	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		6107	DRM_DEBUG("si_irq_set: sw int gfx\n");
		6108	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		6109	}
		6110	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
		6111	DRM_DEBUG("si_irq_set: sw int cp1\n");
		6112	cp_int_cntl1 \|= TIME_STAMP_INT_ENABLE;
		6113	}
		6114	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
		6115	DRM_DEBUG("si_irq_set: sw int cp2\n");
		6116	cp_int_cntl2 \|= TIME_STAMP_INT_ENABLE;
		6117	}
3192	Serge	6118	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
		6119	DRM_DEBUG("si_irq_set: sw int dma\n");
		6120	dma_cntl \|= TRAP_ENABLE;
		6121	}
		6122
		6123	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
		6124	DRM_DEBUG("si_irq_set: sw int dma1\n");
		6125	dma_cntl1 \|= TRAP_ENABLE;
		6126	}
2997	Serge	6127	if (rdev->irq.crtc_vblank_int[0] \|\|
		6128	atomic_read(&rdev->irq.pflip[0])) {
		6129	DRM_DEBUG("si_irq_set: vblank 0\n");
		6130	crtc1 \|= VBLANK_INT_MASK;
		6131	}
		6132	if (rdev->irq.crtc_vblank_int[1] \|\|
		6133	atomic_read(&rdev->irq.pflip[1])) {
		6134	DRM_DEBUG("si_irq_set: vblank 1\n");
		6135	crtc2 \|= VBLANK_INT_MASK;
		6136	}
		6137	if (rdev->irq.crtc_vblank_int[2] \|\|
		6138	atomic_read(&rdev->irq.pflip[2])) {
		6139	DRM_DEBUG("si_irq_set: vblank 2\n");
		6140	crtc3 \|= VBLANK_INT_MASK;
		6141	}
		6142	if (rdev->irq.crtc_vblank_int[3] \|\|
		6143	atomic_read(&rdev->irq.pflip[3])) {
		6144	DRM_DEBUG("si_irq_set: vblank 3\n");
		6145	crtc4 \|= VBLANK_INT_MASK;
		6146	}
		6147	if (rdev->irq.crtc_vblank_int[4] \|\|
		6148	atomic_read(&rdev->irq.pflip[4])) {
		6149	DRM_DEBUG("si_irq_set: vblank 4\n");
		6150	crtc5 \|= VBLANK_INT_MASK;
		6151	}
		6152	if (rdev->irq.crtc_vblank_int[5] \|\|
		6153	atomic_read(&rdev->irq.pflip[5])) {
		6154	DRM_DEBUG("si_irq_set: vblank 5\n");
		6155	crtc6 \|= VBLANK_INT_MASK;
		6156	}
		6157	if (rdev->irq.hpd[0]) {
		6158	DRM_DEBUG("si_irq_set: hpd 1\n");
6104	serge	6159	hpd1 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6160	}
		6161	if (rdev->irq.hpd[1]) {
		6162	DRM_DEBUG("si_irq_set: hpd 2\n");
6104	serge	6163	hpd2 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6164	}
		6165	if (rdev->irq.hpd[2]) {
		6166	DRM_DEBUG("si_irq_set: hpd 3\n");
6104	serge	6167	hpd3 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6168	}
		6169	if (rdev->irq.hpd[3]) {
		6170	DRM_DEBUG("si_irq_set: hpd 4\n");
6104	serge	6171	hpd4 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6172	}
		6173	if (rdev->irq.hpd[4]) {
		6174	DRM_DEBUG("si_irq_set: hpd 5\n");
6104	serge	6175	hpd5 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6176	}
		6177	if (rdev->irq.hpd[5]) {
		6178	DRM_DEBUG("si_irq_set: hpd 6\n");
6104	serge	6179	hpd6 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6180	}
		6181
		6182	WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
		6183	WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
		6184	WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
		6185
3192	Serge	6186	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
		6187	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
		6188
2997	Serge	6189	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		6190
5078	serge	6191	if (rdev->irq.dpm_thermal) {
		6192	DRM_DEBUG("dpm thermal\n");
		6193	thermal_int \|= THERM_INT_MASK_HIGH \| THERM_INT_MASK_LOW;
		6194	}
		6195
3764	Serge	6196	if (rdev->num_crtc >= 2) {
6104	serge	6197	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
		6198	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
3764	Serge	6199	}
2997	Serge	6200	if (rdev->num_crtc >= 4) {
		6201	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
		6202	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
		6203	}
		6204	if (rdev->num_crtc >= 6) {
		6205	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
		6206	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
		6207	}
		6208
3764	Serge	6209	if (rdev->num_crtc >= 2) {
5078	serge	6210	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
		6211	GRPH_PFLIP_INT_MASK);
		6212	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
		6213	GRPH_PFLIP_INT_MASK);
3764	Serge	6214	}
2997	Serge	6215	if (rdev->num_crtc >= 4) {
5078	serge	6216	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
		6217	GRPH_PFLIP_INT_MASK);
		6218	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
		6219	GRPH_PFLIP_INT_MASK);
2997	Serge	6220	}
		6221	if (rdev->num_crtc >= 6) {
5078	serge	6222	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
		6223	GRPH_PFLIP_INT_MASK);
		6224	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
		6225	GRPH_PFLIP_INT_MASK);
2997	Serge	6226	}
		6227
3764	Serge	6228	if (!ASIC_IS_NODCE(rdev)) {
6104	serge	6229	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		6230	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		6231	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		6232	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		6233	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		6234	WREG32(DC_HPD6_INT_CONTROL, hpd6);
3764	Serge	6235	}
2997	Serge	6236
5078	serge	6237	WREG32(CG_THERMAL_INT, thermal_int);
		6238
6104	serge	6239	/* posting read */
		6240	RREG32(SRBM_STATUS);
		6241
2997	Serge	6242	return 0;
		6243	}
		6244
		6245	static inline void si_irq_ack(struct radeon_device *rdev)
		6246	{
		6247	u32 tmp;
		6248
3764	Serge	6249	if (ASIC_IS_NODCE(rdev))
		6250	return;
		6251
2997	Serge	6252	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		6253	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		6254	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
		6255	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
		6256	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
		6257	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
		6258	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		6259	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		6260	if (rdev->num_crtc >= 4) {
		6261	rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		6262	rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		6263	}
		6264	if (rdev->num_crtc >= 6) {
		6265	rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		6266	rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		6267	}
		6268
		6269	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
		6270	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6271	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
		6272	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6273	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
		6274	WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
		6275	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
		6276	WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
		6277	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
		6278	WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
		6279	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
		6280	WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
		6281
		6282	if (rdev->num_crtc >= 4) {
		6283	if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
		6284	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6285	if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
		6286	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6287	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
		6288	WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
		6289	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
		6290	WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
		6291	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
		6292	WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
		6293	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
		6294	WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
		6295	}
		6296
		6297	if (rdev->num_crtc >= 6) {
		6298	if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
		6299	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6300	if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
		6301	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6302	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
		6303	WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
		6304	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
		6305	WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
		6306	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
		6307	WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
		6308	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
		6309	WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
		6310	}
		6311
		6312	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		6313	tmp = RREG32(DC_HPD1_INT_CONTROL);
		6314	tmp \|= DC_HPDx_INT_ACK;
		6315	WREG32(DC_HPD1_INT_CONTROL, tmp);
		6316	}
		6317	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		6318	tmp = RREG32(DC_HPD2_INT_CONTROL);
		6319	tmp \|= DC_HPDx_INT_ACK;
		6320	WREG32(DC_HPD2_INT_CONTROL, tmp);
		6321	}
		6322	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		6323	tmp = RREG32(DC_HPD3_INT_CONTROL);
		6324	tmp \|= DC_HPDx_INT_ACK;
		6325	WREG32(DC_HPD3_INT_CONTROL, tmp);
		6326	}
		6327	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		6328	tmp = RREG32(DC_HPD4_INT_CONTROL);
		6329	tmp \|= DC_HPDx_INT_ACK;
		6330	WREG32(DC_HPD4_INT_CONTROL, tmp);
		6331	}
		6332	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		6333	tmp = RREG32(DC_HPD5_INT_CONTROL);
		6334	tmp \|= DC_HPDx_INT_ACK;
		6335	WREG32(DC_HPD5_INT_CONTROL, tmp);
		6336	}
		6337	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		6338	tmp = RREG32(DC_HPD5_INT_CONTROL);
		6339	tmp \|= DC_HPDx_INT_ACK;
		6340	WREG32(DC_HPD6_INT_CONTROL, tmp);
		6341	}
6104	serge	6342
		6343	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
		6344	tmp = RREG32(DC_HPD1_INT_CONTROL);
		6345	tmp \|= DC_HPDx_RX_INT_ACK;
		6346	WREG32(DC_HPD1_INT_CONTROL, tmp);
		6347	}
		6348	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
		6349	tmp = RREG32(DC_HPD2_INT_CONTROL);
		6350	tmp \|= DC_HPDx_RX_INT_ACK;
		6351	WREG32(DC_HPD2_INT_CONTROL, tmp);
		6352	}
		6353	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
		6354	tmp = RREG32(DC_HPD3_INT_CONTROL);
		6355	tmp \|= DC_HPDx_RX_INT_ACK;
		6356	WREG32(DC_HPD3_INT_CONTROL, tmp);
		6357	}
		6358	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
		6359	tmp = RREG32(DC_HPD4_INT_CONTROL);
		6360	tmp \|= DC_HPDx_RX_INT_ACK;
		6361	WREG32(DC_HPD4_INT_CONTROL, tmp);
		6362	}
		6363	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
		6364	tmp = RREG32(DC_HPD5_INT_CONTROL);
		6365	tmp \|= DC_HPDx_RX_INT_ACK;
		6366	WREG32(DC_HPD5_INT_CONTROL, tmp);
		6367	}
		6368	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
		6369	tmp = RREG32(DC_HPD5_INT_CONTROL);
		6370	tmp \|= DC_HPDx_RX_INT_ACK;
		6371	WREG32(DC_HPD6_INT_CONTROL, tmp);
		6372	}
2997	Serge	6373	}
		6374
		6375	static void si_irq_disable(struct radeon_device *rdev)
		6376	{
		6377	si_disable_interrupts(rdev);
		6378	/* Wait and acknowledge irq */
		6379	mdelay(1);
		6380	si_irq_ack(rdev);
		6381	si_disable_interrupt_state(rdev);
		6382	}
		6383
		6384	static void si_irq_suspend(struct radeon_device *rdev)
		6385	{
		6386	si_irq_disable(rdev);
		6387	si_rlc_stop(rdev);
		6388	}
		6389
		6390	static void si_irq_fini(struct radeon_device *rdev)
		6391	{
		6392	si_irq_suspend(rdev);
		6393	r600_ih_ring_fini(rdev);
		6394	}
		6395
		6396	static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
		6397	{
		6398	u32 wptr, tmp;
		6399
		6400	if (rdev->wb.enabled)
		6401	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		6402	else
		6403	wptr = RREG32(IH_RB_WPTR);
		6404
		6405	if (wptr & RB_OVERFLOW) {
5179	serge	6406	wptr &= ~RB_OVERFLOW;
2997	Serge	6407	/* When a ring buffer overflow happen start parsing interrupt
		6408	* from the last not overwritten vector (wptr + 16). Hopefully
		6409	* this should allow us to catchup.
		6410	*/
5179	serge	6411	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
		6412	wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
2997	Serge	6413	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		6414	tmp = RREG32(IH_RB_CNTL);
		6415	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		6416	WREG32(IH_RB_CNTL, tmp);
		6417	}
		6418	return (wptr & rdev->ih.ptr_mask);
		6419	}
		6420
		6421	/* SI IV Ring
		6422	* Each IV ring entry is 128 bits:
		6423	* [7:0] - interrupt source id
		6424	* [31:8] - reserved
		6425	* [59:32] - interrupt source data
		6426	* [63:60] - reserved
		6427	* [71:64] - RINGID
		6428	* [79:72] - VMID
		6429	* [127:80] - reserved
		6430	*/
		6431	int si_irq_process(struct radeon_device *rdev)
		6432	{
		6433	u32 wptr;
		6434	u32 rptr;
		6435	u32 src_id, src_data, ring_id;
		6436	u32 ring_index;
		6437	bool queue_hotplug = false;
6104	serge	6438	bool queue_dp = false;
5078	serge	6439	bool queue_thermal = false;
		6440	u32 status, addr;
2997	Serge	6441
		6442	if (!rdev->ih.enabled \|\| rdev->shutdown)
		6443	return IRQ_NONE;
		6444
		6445	wptr = si_get_ih_wptr(rdev);
		6446
		6447	restart_ih:
		6448	/* is somebody else already processing irqs? */
		6449	if (atomic_xchg(&rdev->ih.lock, 1))
		6450	return IRQ_NONE;
		6451
		6452	rptr = rdev->ih.rptr;
		6453	DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		6454
		6455	/* Order reading of wptr vs. reading of IH ring data */
		6456	rmb();
		6457
		6458	/* display interrupts */
		6459	si_irq_ack(rdev);
		6460
		6461	while (rptr != wptr) {
		6462	/* wptr/rptr are in bytes! */
		6463	ring_index = rptr / 4;
		6464	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		6465	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		6466	ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
		6467
		6468	switch (src_id) {
		6469	case 1: /* D1 vblank/vline */
		6470	switch (src_data) {
		6471	case 0: /* D1 vblank */
6104	serge	6472	if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
		6473	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6474
		6475	if (rdev->irq.crtc_vblank_int[0]) {
		6476	drm_handle_vblank(rdev->ddev, 0);
		6477	rdev->pm.vblank_sync = true;
		6478	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6479	}
6104	serge	6480	if (atomic_read(&rdev->irq.pflip[0]))
		6481	radeon_crtc_handle_vblank(rdev, 0);
		6482	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		6483	DRM_DEBUG("IH: D1 vblank\n");
		6484
2997	Serge	6485	break;
		6486	case 1: /* D1 vline */
6104	serge	6487	if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
		6488	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6489
		6490	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		6491	DRM_DEBUG("IH: D1 vline\n");
		6492
2997	Serge	6493	break;
		6494	default:
		6495	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6496	break;
		6497	}
		6498	break;
		6499	case 2: /* D2 vblank/vline */
		6500	switch (src_data) {
		6501	case 0: /* D2 vblank */
6104	serge	6502	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
		6503	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6504
		6505	if (rdev->irq.crtc_vblank_int[1]) {
		6506	drm_handle_vblank(rdev->ddev, 1);
		6507	rdev->pm.vblank_sync = true;
		6508	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6509	}
6104	serge	6510	if (atomic_read(&rdev->irq.pflip[1]))
		6511	radeon_crtc_handle_vblank(rdev, 1);
		6512	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
		6513	DRM_DEBUG("IH: D2 vblank\n");
		6514
2997	Serge	6515	break;
		6516	case 1: /* D2 vline */
6104	serge	6517	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
		6518	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6519
		6520	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
		6521	DRM_DEBUG("IH: D2 vline\n");
		6522
2997	Serge	6523	break;
		6524	default:
		6525	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6526	break;
		6527	}
		6528	break;
		6529	case 3: /* D3 vblank/vline */
		6530	switch (src_data) {
		6531	case 0: /* D3 vblank */
6104	serge	6532	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
		6533	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6534
		6535	if (rdev->irq.crtc_vblank_int[2]) {
		6536	drm_handle_vblank(rdev->ddev, 2);
		6537	rdev->pm.vblank_sync = true;
		6538	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6539	}
6104	serge	6540	if (atomic_read(&rdev->irq.pflip[2]))
		6541	radeon_crtc_handle_vblank(rdev, 2);
		6542	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
		6543	DRM_DEBUG("IH: D3 vblank\n");
		6544
2997	Serge	6545	break;
		6546	case 1: /* D3 vline */
6104	serge	6547	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
		6548	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6549
		6550	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
		6551	DRM_DEBUG("IH: D3 vline\n");
		6552
2997	Serge	6553	break;
		6554	default:
		6555	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6556	break;
		6557	}
		6558	break;
		6559	case 4: /* D4 vblank/vline */
		6560	switch (src_data) {
		6561	case 0: /* D4 vblank */
6104	serge	6562	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
		6563	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6564
		6565	if (rdev->irq.crtc_vblank_int[3]) {
		6566	drm_handle_vblank(rdev->ddev, 3);
		6567	rdev->pm.vblank_sync = true;
		6568	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6569	}
6104	serge	6570	if (atomic_read(&rdev->irq.pflip[3]))
		6571	radeon_crtc_handle_vblank(rdev, 3);
		6572	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
		6573	DRM_DEBUG("IH: D4 vblank\n");
		6574
2997	Serge	6575	break;
		6576	case 1: /* D4 vline */
6104	serge	6577	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
		6578	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6579
		6580	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
		6581	DRM_DEBUG("IH: D4 vline\n");
		6582
2997	Serge	6583	break;
		6584	default:
		6585	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6586	break;
		6587	}
		6588	break;
		6589	case 5: /* D5 vblank/vline */
		6590	switch (src_data) {
		6591	case 0: /* D5 vblank */
6104	serge	6592	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
		6593	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6594
		6595	if (rdev->irq.crtc_vblank_int[4]) {
		6596	drm_handle_vblank(rdev->ddev, 4);
		6597	rdev->pm.vblank_sync = true;
		6598	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6599	}
6104	serge	6600	if (atomic_read(&rdev->irq.pflip[4]))
		6601	radeon_crtc_handle_vblank(rdev, 4);
		6602	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
		6603	DRM_DEBUG("IH: D5 vblank\n");
		6604
2997	Serge	6605	break;
		6606	case 1: /* D5 vline */
6104	serge	6607	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
		6608	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6609
		6610	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
		6611	DRM_DEBUG("IH: D5 vline\n");
		6612
2997	Serge	6613	break;
		6614	default:
		6615	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6616	break;
		6617	}
		6618	break;
		6619	case 6: /* D6 vblank/vline */
		6620	switch (src_data) {
		6621	case 0: /* D6 vblank */
6104	serge	6622	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
		6623	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6624
		6625	if (rdev->irq.crtc_vblank_int[5]) {
		6626	drm_handle_vblank(rdev->ddev, 5);
		6627	rdev->pm.vblank_sync = true;
		6628	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6629	}
6104	serge	6630	if (atomic_read(&rdev->irq.pflip[5]))
		6631	radeon_crtc_handle_vblank(rdev, 5);
		6632	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
		6633	DRM_DEBUG("IH: D6 vblank\n");
		6634
2997	Serge	6635	break;
		6636	case 1: /* D6 vline */
6104	serge	6637	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
		6638	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6639
		6640	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
		6641	DRM_DEBUG("IH: D6 vline\n");
		6642
2997	Serge	6643	break;
		6644	default:
		6645	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6646	break;
		6647	}
		6648	break;
5078	serge	6649	case 8: /* D1 page flip */
		6650	case 10: /* D2 page flip */
		6651	case 12: /* D3 page flip */
		6652	case 14: /* D4 page flip */
		6653	case 16: /* D5 page flip */
		6654	case 18: /* D6 page flip */
		6655	DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
		6656	break;
2997	Serge	6657	case 42: /* HPD hotplug */
		6658	switch (src_data) {
		6659	case 0:
6104	serge	6660	if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
		6661	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6662
		6663	rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
		6664	queue_hotplug = true;
		6665	DRM_DEBUG("IH: HPD1\n");
		6666
2997	Serge	6667	break;
		6668	case 1:
6104	serge	6669	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
		6670	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6671
		6672	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
		6673	queue_hotplug = true;
		6674	DRM_DEBUG("IH: HPD2\n");
		6675
2997	Serge	6676	break;
		6677	case 2:
6104	serge	6678	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
		6679	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6680
		6681	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
		6682	queue_hotplug = true;
		6683	DRM_DEBUG("IH: HPD3\n");
		6684
2997	Serge	6685	break;
		6686	case 3:
6104	serge	6687	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
		6688	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6689
		6690	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
		6691	queue_hotplug = true;
		6692	DRM_DEBUG("IH: HPD4\n");
		6693
2997	Serge	6694	break;
		6695	case 4:
6104	serge	6696	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
		6697	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6698
		6699	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
		6700	queue_hotplug = true;
		6701	DRM_DEBUG("IH: HPD5\n");
		6702
2997	Serge	6703	break;
		6704	case 5:
6104	serge	6705	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
		6706	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6707
		6708	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
		6709	queue_hotplug = true;
		6710	DRM_DEBUG("IH: HPD6\n");
		6711
2997	Serge	6712	break;
6104	serge	6713	case 6:
		6714	if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
		6715	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6716
		6717	rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
		6718	queue_dp = true;
		6719	DRM_DEBUG("IH: HPD_RX 1\n");
		6720
		6721	break;
		6722	case 7:
		6723	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
		6724	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6725
		6726	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
		6727	queue_dp = true;
		6728	DRM_DEBUG("IH: HPD_RX 2\n");
		6729
		6730	break;
		6731	case 8:
		6732	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
		6733	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6734
		6735	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
		6736	queue_dp = true;
		6737	DRM_DEBUG("IH: HPD_RX 3\n");
		6738
		6739	break;
		6740	case 9:
		6741	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
		6742	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6743
		6744	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
		6745	queue_dp = true;
		6746	DRM_DEBUG("IH: HPD_RX 4\n");
		6747
		6748	break;
		6749	case 10:
		6750	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
		6751	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6752
		6753	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
		6754	queue_dp = true;
		6755	DRM_DEBUG("IH: HPD_RX 5\n");
		6756
		6757	break;
		6758	case 11:
		6759	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
		6760	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6761
		6762	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
		6763	queue_dp = true;
		6764	DRM_DEBUG("IH: HPD_RX 6\n");
		6765
		6766	break;
2997	Serge	6767	default:
		6768	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6769	break;
		6770	}
		6771	break;
6104	serge	6772	case 96:
		6773	DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
		6774	WREG32(SRBM_INT_ACK, 0x1);
		6775	break;
5078	serge	6776	case 124: /* UVD */
		6777	DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
		6778	radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
		6779	break;
3192	Serge	6780	case 146:
		6781	case 147:
5078	serge	6782	addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
		6783	status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
		6784	/* reset addr and status */
		6785	WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
		6786	if (addr == 0x0 && status == 0x0)
		6787	break;
3192	Serge	6788	dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
		6789	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5078	serge	6790	addr);
3192	Serge	6791	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5078	serge	6792	status);
		6793	si_vm_decode_fault(rdev, status, addr);
3192	Serge	6794	break;
2997	Serge	6795	case 176: /* RINGID0 CP_INT */
		6796	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
		6797	break;
		6798	case 177: /* RINGID1 CP_INT */
		6799	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		6800	break;
		6801	case 178: /* RINGID2 CP_INT */
		6802	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		6803	break;
		6804	case 181: /* CP EOP event */
		6805	DRM_DEBUG("IH: CP EOP\n");
		6806	switch (ring_id) {
		6807	case 0:
		6808	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
		6809	break;
		6810	case 1:
		6811	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		6812	break;
		6813	case 2:
		6814	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		6815	break;
		6816	}
		6817	break;
3192	Serge	6818	case 224: /* DMA trap event */
		6819	DRM_DEBUG("IH: DMA trap\n");
		6820	radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
		6821	break;
5078	serge	6822	case 230: /* thermal low to high */
		6823	DRM_DEBUG("IH: thermal low to high\n");
		6824	rdev->pm.dpm.thermal.high_to_low = false;
		6825	queue_thermal = true;
		6826	break;
		6827	case 231: /* thermal high to low */
		6828	DRM_DEBUG("IH: thermal high to low\n");
		6829	rdev->pm.dpm.thermal.high_to_low = true;
		6830	queue_thermal = true;
		6831	break;
2997	Serge	6832	case 233: /* GUI IDLE */
		6833	DRM_DEBUG("IH: GUI idle\n");
		6834	break;
3192	Serge	6835	case 244: /* DMA trap event */
		6836	DRM_DEBUG("IH: DMA1 trap\n");
		6837	radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
		6838	break;
2997	Serge	6839	default:
		6840	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6841	break;
		6842	}
		6843
		6844	/* wptr/rptr are in bytes! */
		6845	rptr += 16;
		6846	rptr &= rdev->ih.ptr_mask;
5179	serge	6847	WREG32(IH_RB_RPTR, rptr);
2997	Serge	6848	}
		6849	// if (queue_hotplug)
		6850	// schedule_work(&rdev->hotplug_work);
		6851	rdev->ih.rptr = rptr;
		6852	atomic_set(&rdev->ih.lock, 0);
		6853
		6854	/* make sure wptr hasn't changed while processing */
		6855	wptr = si_get_ih_wptr(rdev);
		6856	if (wptr != rptr)
		6857	goto restart_ih;
		6858
		6859	return IRQ_HANDLED;
		6860	}
		6861
		6862	/*
		6863	* startup/shutdown callbacks
		6864	*/
		6865	static int si_startup(struct radeon_device *rdev)
		6866	{
		6867	struct radeon_ring *ring;
		6868	int r;
		6869
5078	serge	6870	/* enable pcie gen2/3 link */
		6871	si_pcie_gen3_enable(rdev);
		6872	/* enable aspm */
		6873	si_program_aspm(rdev);
2997	Serge	6874
5078	serge	6875	/* scratch needs to be initialized before MC */
2997	Serge	6876	r = r600_vram_scratch_init(rdev);
		6877	if (r)
		6878	return r;
		6879
		6880	si_mc_program(rdev);
5078	serge	6881
		6882	if (!rdev->pm.dpm_enabled) {
		6883	r = si_mc_load_microcode(rdev);
		6884	if (r) {
		6885	DRM_ERROR("Failed to load MC firmware!\n");
		6886	return r;
		6887	}
		6888	}
		6889
2997	Serge	6890	r = si_pcie_gart_enable(rdev);
		6891	if (r)
		6892	return r;
		6893	si_gpu_init(rdev);
		6894
		6895	/* allocate rlc buffers */
5078	serge	6896	if (rdev->family == CHIP_VERDE) {
		6897	rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
		6898	rdev->rlc.reg_list_size =
		6899	(u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
		6900	}
		6901	rdev->rlc.cs_data = si_cs_data;
		6902	r = sumo_rlc_init(rdev);
2997	Serge	6903	if (r) {
		6904	DRM_ERROR("Failed to init rlc BOs!\n");
		6905	return r;
		6906	}
		6907
		6908	/* allocate wb buffer */
		6909	r = radeon_wb_init(rdev);
		6910	if (r)
		6911	return r;
		6912
		6913	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
		6914	if (r) {
		6915	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		6916	return r;
		6917	}
		6918
		6919	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		6920	if (r) {
		6921	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		6922	return r;
		6923	}
		6924
		6925	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		6926	if (r) {
		6927	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		6928	return r;
		6929	}
		6930
3192	Serge	6931	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
		6932	if (r) {
		6933	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		6934	return r;
		6935	}
		6936
		6937	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
		6938	if (r) {
		6939	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		6940	return r;
		6941	}
		6942
5078	serge	6943	if (rdev->has_uvd) {
		6944	r = uvd_v2_2_resume(rdev);
		6945	if (!r) {
		6946	r = radeon_fence_driver_start_ring(rdev,
		6947	R600_RING_TYPE_UVD_INDEX);
		6948	if (r)
		6949	dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
		6950	}
		6951	if (r)
		6952	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
		6953	}
3764	Serge	6954
6104	serge	6955	r = radeon_vce_resume(rdev);
		6956	if (!r) {
		6957	r = vce_v1_0_resume(rdev);
		6958	if (!r)
		6959	r = radeon_fence_driver_start_ring(rdev,
		6960	TN_RING_TYPE_VCE1_INDEX);
		6961	if (!r)
		6962	r = radeon_fence_driver_start_ring(rdev,
		6963	TN_RING_TYPE_VCE2_INDEX);
		6964	}
		6965	if (r) {
		6966	dev_err(rdev->dev, "VCE init error (%d).\n", r);
		6967	rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
		6968	rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
		6969	}
		6970
2997	Serge	6971	/* Enable IRQ */
3764	Serge	6972	if (!rdev->irq.installed) {
		6973	r = radeon_irq_kms_init(rdev);
		6974	if (r)
		6975	return r;
		6976	}
		6977
2997	Serge	6978	r = si_irq_init(rdev);
		6979	if (r) {
		6980	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		6981	radeon_irq_kms_fini(rdev);
		6982	return r;
		6983	}
		6984	si_irq_set(rdev);
		6985
		6986	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		6987	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5078	serge	6988	RADEON_CP_PACKET2);
2997	Serge	6989	if (r)
		6990	return r;
		6991
		6992	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
		6993	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
5078	serge	6994	RADEON_CP_PACKET2);
2997	Serge	6995	if (r)
		6996	return r;
		6997
		6998	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
		6999	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
5078	serge	7000	RADEON_CP_PACKET2);
2997	Serge	7001	if (r)
		7002	return r;
		7003
3192	Serge	7004	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		7005	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5078	serge	7006	DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
3192	Serge	7007	if (r)
		7008	return r;
		7009
		7010	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		7011	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5078	serge	7012	DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
3192	Serge	7013	if (r)
		7014	return r;
		7015
2997	Serge	7016	r = si_cp_load_microcode(rdev);
		7017	if (r)
		7018	return r;
		7019	r = si_cp_resume(rdev);
		7020	if (r)
		7021	return r;
		7022
3192	Serge	7023	r = cayman_dma_resume(rdev);
		7024	if (r)
		7025	return r;
		7026
5078	serge	7027	if (rdev->has_uvd) {
		7028	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
		7029	if (ring->ring_size) {
		7030	r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
		7031	RADEON_CP_PACKET2);
		7032	if (!r)
		7033	r = uvd_v1_0_init(rdev);
		7034	if (r)
		7035	DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
		7036	}
		7037	}
3764	Serge	7038
6104	serge	7039	r = -ENOENT;
		7040
		7041	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
		7042	if (ring->ring_size)
		7043	r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
		7044	VCE_CMD_NO_OP);
		7045
		7046	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
		7047	if (ring->ring_size)
		7048	r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
		7049	VCE_CMD_NO_OP);
		7050
		7051	if (!r)
		7052	r = vce_v1_0_init(rdev);
		7053	else if (r != -ENOENT)
		7054	DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
		7055
3120	serge	7056	r = radeon_ib_pool_init(rdev);
		7057	if (r) {
		7058	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		7059	return r;
		7060	}
2997	Serge	7061
3764	Serge	7062	r = radeon_vm_manager_init(rdev);
		7063	if (r) {
		7064	dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
		7065	return r;
		7066	}
2997	Serge	7067
6104	serge	7068	r = radeon_audio_init(rdev);
5078	serge	7069	if (r)
		7070	return r;
		7071
2997	Serge	7072	return 0;
		7073	}
		7074
6104	serge	7075	int si_resume(struct radeon_device *rdev)
		7076	{
		7077	int r;
2997	Serge	7078
6104	serge	7079	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
		7080	* posting will perform necessary task to bring back GPU into good
		7081	* shape.
		7082	*/
		7083	/* post card */
		7084	atom_asic_init(rdev->mode_info.atom_context);
3120	serge	7085
6104	serge	7086	/* init golden registers */
		7087	si_init_golden_registers(rdev);
3120	serge	7088
6104	serge	7089	if (rdev->pm.pm_method == PM_METHOD_DPM)
		7090	radeon_pm_resume(rdev);
		7091
		7092	rdev->accel_working = true;
		7093	r = si_startup(rdev);
		7094	if (r) {
		7095	DRM_ERROR("si startup failed on resume\n");
		7096	rdev->accel_working = false;
		7097	return r;
		7098	}
		7099
		7100	return r;
		7101
		7102	}
		7103
2997	Serge	7104	/* Plan is to move initialization in that function and use
		7105	* helper function so that radeon_device_init pretty much
		7106	* do nothing more than calling asic specific function. This
		7107	* should also allow to remove a bunch of callback function
		7108	* like vram_info.
		7109	*/
		7110	int si_init(struct radeon_device *rdev)
		7111	{
		7112	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		7113	int r;
		7114
		7115	/* Read BIOS */
		7116	if (!radeon_get_bios(rdev)) {
		7117	if (ASIC_IS_AVIVO(rdev))
		7118	return -EINVAL;
		7119	}
		7120	/* Must be an ATOMBIOS */
		7121	if (!rdev->is_atom_bios) {
		7122	dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
		7123	return -EINVAL;
		7124	}
		7125	r = radeon_atombios_init(rdev);
		7126	if (r)
		7127	return r;
		7128
		7129	/* Post card if necessary */
		7130	if (!radeon_card_posted(rdev)) {
		7131	if (!rdev->bios) {
		7132	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		7133	return -EINVAL;
		7134	}
		7135	DRM_INFO("GPU not posted. posting now...\n");
		7136	atom_asic_init(rdev->mode_info.atom_context);
		7137	}
3764	Serge	7138	/* init golden registers */
		7139	si_init_golden_registers(rdev);
2997	Serge	7140	/* Initialize scratch registers */
		7141	si_scratch_init(rdev);
		7142	/* Initialize surface registers */
		7143	radeon_surface_init(rdev);
		7144	/* Initialize clocks */
		7145	radeon_get_clock_info(rdev->ddev);
		7146
		7147	/* Fence driver */
		7148	r = radeon_fence_driver_init(rdev);
		7149	if (r)
		7150	return r;
		7151
		7152	/* initialize memory controller */
		7153	r = si_mc_init(rdev);
		7154	if (r)
		7155	return r;
		7156	/* Memory manager */
		7157	r = radeon_bo_init(rdev);
		7158	if (r)
		7159	return r;
		7160
5078	serge	7161	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->ce_fw \|\|
		7162	!rdev->rlc_fw \|\| !rdev->mc_fw) {
		7163	r = si_init_microcode(rdev);
		7164	if (r) {
		7165	DRM_ERROR("Failed to load firmware!\n");
		7166	return r;
		7167	}
		7168	}
		7169
		7170	/* Initialize power management */
		7171	radeon_pm_init(rdev);
		7172
2997	Serge	7173	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		7174	ring->ring_obj = NULL;
		7175	r600_ring_init(rdev, ring, 1024 * 1024);
		7176
		7177	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
		7178	ring->ring_obj = NULL;
		7179	r600_ring_init(rdev, ring, 1024 * 1024);
		7180
		7181	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
		7182	ring->ring_obj = NULL;
		7183	r600_ring_init(rdev, ring, 1024 * 1024);
		7184
3192	Serge	7185	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		7186	ring->ring_obj = NULL;
		7187	r600_ring_init(rdev, ring, 64 * 1024);
		7188
		7189	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		7190	ring->ring_obj = NULL;
		7191	r600_ring_init(rdev, ring, 64 * 1024);
		7192
5078	serge	7193	if (rdev->has_uvd) {
		7194	r = radeon_uvd_init(rdev);
		7195	if (!r) {
		7196	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
		7197	ring->ring_obj = NULL;
		7198	r600_ring_init(rdev, ring, 4096);
		7199	}
		7200	}
		7201
6104	serge	7202	r = radeon_vce_init(rdev);
		7203	if (!r) {
		7204	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
		7205	ring->ring_obj = NULL;
		7206	r600_ring_init(rdev, ring, 4096);
		7207
		7208	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
		7209	ring->ring_obj = NULL;
		7210	r600_ring_init(rdev, ring, 4096);
		7211	}
		7212
		7213	rdev->ih.ring_obj = NULL;
2997	Serge	7214	r600_ih_ring_init(rdev, 64 * 1024);
		7215
6104	serge	7216	r = r600_pcie_gart_init(rdev);
		7217	if (r)
		7218	return r;
2997	Serge	7219
		7220	rdev->accel_working = true;
6104	serge	7221	r = si_startup(rdev);
2997	Serge	7222	if (r) {
		7223	dev_err(rdev->dev, "disabling GPU acceleration\n");
5078	serge	7224	si_cp_fini(rdev);
6104	serge	7225	cayman_dma_fini(rdev);
		7226	si_irq_fini(rdev);
		7227	sumo_rlc_fini(rdev);
		7228	radeon_wb_fini(rdev);
		7229	radeon_ib_pool_fini(rdev);
		7230	radeon_vm_manager_fini(rdev);
		7231	radeon_irq_kms_fini(rdev);
		7232	si_pcie_gart_fini(rdev);
2997	Serge	7233	rdev->accel_working = false;
		7234	}
		7235
		7236	/* Don't start up if the MC ucode is missing.
		7237	* The default clocks and voltages before the MC ucode
		7238	* is loaded are not suffient for advanced operations.
		7239	*/
		7240	if (!rdev->mc_fw) {
		7241	DRM_ERROR("radeon: MC ucode required for NI+.\n");
		7242	return -EINVAL;
		7243	}
		7244
		7245	return 0;
		7246	}
		7247
6104	serge	7248	void si_fini(struct radeon_device *rdev)
		7249	{
		7250	radeon_pm_fini(rdev);
		7251	si_cp_fini(rdev);
		7252	cayman_dma_fini(rdev);
		7253	si_fini_pg(rdev);
		7254	si_fini_cg(rdev);
		7255	si_irq_fini(rdev);
		7256	sumo_rlc_fini(rdev);
		7257	radeon_wb_fini(rdev);
		7258	radeon_vm_manager_fini(rdev);
		7259	radeon_ib_pool_fini(rdev);
		7260	radeon_irq_kms_fini(rdev);
		7261	if (rdev->has_uvd) {
		7262	uvd_v1_0_fini(rdev);
		7263	radeon_uvd_fini(rdev);
		7264	radeon_vce_fini(rdev);
		7265	}
		7266	si_pcie_gart_fini(rdev);
		7267	r600_vram_scratch_fini(rdev);
		7268	radeon_gem_fini(rdev);
		7269	radeon_fence_driver_fini(rdev);
		7270	radeon_bo_fini(rdev);
		7271	radeon_atombios_fini(rdev);
		7272	kfree(rdev->bios);
		7273	rdev->bios = NULL;
		7274	}
		7275
2997	Serge	7276	/**
3764	Serge	7277	* si_get_gpu_clock_counter - return GPU clock counter snapshot
2997	Serge	7278	*
		7279	* @rdev: radeon_device pointer
		7280	*
		7281	* Fetches a GPU clock counter snapshot (SI).
		7282	* Returns the 64 bit clock counter snapshot.
		7283	*/
3764	Serge	7284	uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
2997	Serge	7285	{
		7286	uint64_t clock;
		7287
		7288	mutex_lock(&rdev->gpu_clock_mutex);
		7289	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
		7290	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) \|
		7291	((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
		7292	mutex_unlock(&rdev->gpu_clock_mutex);
		7293	return clock;
		7294	}
3764	Serge	7295
		7296	int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
		7297	{
		7298	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
		7299	int r;
		7300
		7301	/* bypass vclk and dclk with bclk */
		7302	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		7303	VCLK_SRC_SEL(1) \| DCLK_SRC_SEL(1),
		7304	~(VCLK_SRC_SEL_MASK \| DCLK_SRC_SEL_MASK));
		7305
		7306	/* put PLL in bypass mode */
		7307	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
		7308
		7309	if (!vclk \|\| !dclk) {
6104	serge	7310	/* keep the Bypass mode */
3764	Serge	7311	return 0;
		7312	}
		7313
		7314	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
		7315	16384, 0x03FFFFFF, 0, 128, 5,
		7316	&fb_div, &vclk_div, &dclk_div);
		7317	if (r)
		7318	return r;
		7319
		7320	/* set RESET_ANTI_MUX to 0 */
		7321	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
		7322
		7323	/* set VCO_MODE to 1 */
		7324	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
		7325
6104	serge	7326	/* disable sleep mode */
3764	Serge	7327	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
		7328
		7329	/* deassert UPLL_RESET */
		7330	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
		7331
		7332	mdelay(1);
		7333
		7334	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
		7335	if (r)
		7336	return r;
		7337
		7338	/* assert UPLL_RESET again */
		7339	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
		7340
		7341	/* disable spread spectrum. */
		7342	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
		7343
		7344	/* set feedback divider */
		7345	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
		7346
		7347	/* set ref divider to 0 */
		7348	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
		7349
		7350	if (fb_div < 307200)
		7351	WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
		7352	else
		7353	WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
		7354
		7355	/* set PDIV_A and PDIV_B */
		7356	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		7357	UPLL_PDIV_A(vclk_div) \| UPLL_PDIV_B(dclk_div),
		7358	~(UPLL_PDIV_A_MASK \| UPLL_PDIV_B_MASK));
		7359
		7360	/* give the PLL some time to settle */
		7361	mdelay(15);
		7362
		7363	/* deassert PLL_RESET */
		7364	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
		7365
		7366	mdelay(15);
		7367
		7368	/* switch from bypass mode to normal mode */
		7369	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
		7370
		7371	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
		7372	if (r)
		7373	return r;
		7374
		7375	/* switch VCLK and DCLK selection */
		7376	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		7377	VCLK_SRC_SEL(2) \| DCLK_SRC_SEL(2),
		7378	~(VCLK_SRC_SEL_MASK \| DCLK_SRC_SEL_MASK));
		7379
		7380	mdelay(100);
		7381
		7382	return 0;
		7383	}
5078	serge	7384
		7385	static void si_pcie_gen3_enable(struct radeon_device *rdev)
		7386	{
		7387	struct pci_dev *root = rdev->pdev->bus->self;
		7388	int bridge_pos, gpu_pos;
		7389	u32 speed_cntl, mask, current_data_rate;
		7390	int ret, i;
		7391	u16 tmp16;
		7392
		7393	if (radeon_pcie_gen2 == 0)
		7394	return;
		7395
		7396	if (rdev->flags & RADEON_IS_IGP)
		7397	return;
		7398
		7399	if (!(rdev->flags & RADEON_IS_PCIE))
		7400	return;
		7401
		7402	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
		7403	if (ret != 0)
		7404	return;
		7405
		7406	if (!(mask & (DRM_PCIE_SPEED_50 \| DRM_PCIE_SPEED_80)))
		7407	return;
		7408
		7409	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
		7410	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
		7411	LC_CURRENT_DATA_RATE_SHIFT;
		7412	if (mask & DRM_PCIE_SPEED_80) {
		7413	if (current_data_rate == 2) {
		7414	DRM_INFO("PCIE gen 3 link speeds already enabled\n");
		7415	return;
		7416	}
		7417	DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
		7418	} else if (mask & DRM_PCIE_SPEED_50) {
		7419	if (current_data_rate == 1) {
		7420	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
		7421	return;
		7422	}
		7423	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
		7424	}
		7425
		7426	bridge_pos = pci_pcie_cap(root);
		7427	if (!bridge_pos)
		7428	return;
		7429
		7430	gpu_pos = pci_pcie_cap(rdev->pdev);
		7431	if (!gpu_pos)
		7432	return;
		7433
		7434	if (mask & DRM_PCIE_SPEED_80) {
		7435	/* re-try equalization if gen3 is not already enabled */
		7436	if (current_data_rate != 2) {
		7437	u16 bridge_cfg, gpu_cfg;
		7438	u16 bridge_cfg2, gpu_cfg2;
		7439	u32 max_lw, current_lw, tmp;
		7440
		7441	pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
		7442	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
		7443
		7444	tmp16 = bridge_cfg \| PCI_EXP_LNKCTL_HAWD;
		7445	pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
		7446
		7447	tmp16 = gpu_cfg \| PCI_EXP_LNKCTL_HAWD;
		7448	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
		7449
		7450	tmp = RREG32_PCIE(PCIE_LC_STATUS1);
		7451	max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
		7452	current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
		7453
		7454	if (current_lw < max_lw) {
		7455	tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
		7456	if (tmp & LC_RENEGOTIATION_SUPPORT) {
		7457	tmp &= ~(LC_LINK_WIDTH_MASK \| LC_UPCONFIGURE_DIS);
		7458	tmp \|= (max_lw << LC_LINK_WIDTH_SHIFT);
		7459	tmp \|= LC_UPCONFIGURE_SUPPORT \| LC_RENEGOTIATE_EN \| LC_RECONFIG_NOW;
		7460	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
		7461	}
		7462	}
		7463
		7464	for (i = 0; i < 10; i++) {
		7465	/* check status */
		7466	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
		7467	if (tmp16 & PCI_EXP_DEVSTA_TRPND)
		7468	break;
		7469
		7470	pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
		7471	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
		7472
		7473	pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
		7474	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
		7475
		7476	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
		7477	tmp \|= LC_SET_QUIESCE;
		7478	WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
		7479
		7480	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
		7481	tmp \|= LC_REDO_EQ;
		7482	WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
		7483
		7484	mdelay(100);
		7485
		7486	/* linkctl */
		7487	pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
		7488	tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
		7489	tmp16 \|= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
		7490	pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
		7491
		7492	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
		7493	tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
		7494	tmp16 \|= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
		7495	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
		7496
		7497	/* linkctl2 */
		7498	pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
		7499	tmp16 &= ~((1 << 4) \| (7 << 9));
		7500	tmp16 \|= (bridge_cfg2 & ((1 << 4) \| (7 << 9)));
		7501	pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
		7502
		7503	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
		7504	tmp16 &= ~((1 << 4) \| (7 << 9));
		7505	tmp16 \|= (gpu_cfg2 & ((1 << 4) \| (7 << 9)));
		7506	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
		7507
		7508	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
		7509	tmp &= ~LC_SET_QUIESCE;
		7510	WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
		7511	}
		7512	}
		7513	}
		7514
		7515	/* set the link speed */
		7516	speed_cntl \|= LC_FORCE_EN_SW_SPEED_CHANGE \| LC_FORCE_DIS_HW_SPEED_CHANGE;
		7517	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
		7518	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
		7519
		7520	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
		7521	tmp16 &= ~0xf;
		7522	if (mask & DRM_PCIE_SPEED_80)
		7523	tmp16 \|= 3; /* gen3 */
		7524	else if (mask & DRM_PCIE_SPEED_50)
		7525	tmp16 \|= 2; /* gen2 */
		7526	else
		7527	tmp16 \|= 1; /* gen1 */
		7528	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
		7529
		7530	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
		7531	speed_cntl \|= LC_INITIATE_LINK_SPEED_CHANGE;
		7532	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
		7533
		7534	for (i = 0; i < rdev->usec_timeout; i++) {
		7535	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
		7536	if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
		7537	break;
		7538	udelay(1);
		7539	}
		7540	}
		7541
		7542	static void si_program_aspm(struct radeon_device *rdev)
		7543	{
		7544	u32 data, orig;
		7545	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
		7546	bool disable_clkreq = false;
		7547
		7548	if (radeon_aspm == 0)
		7549	return;
		7550
		7551	if (!(rdev->flags & RADEON_IS_PCIE))
		7552	return;
		7553
		7554	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
		7555	data &= ~LC_XMIT_N_FTS_MASK;
		7556	data \|= LC_XMIT_N_FTS(0x24) \| LC_XMIT_N_FTS_OVERRIDE_EN;
		7557	if (orig != data)
		7558	WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
		7559
		7560	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
		7561	data \|= LC_GO_TO_RECOVERY;
		7562	if (orig != data)
		7563	WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
		7564
		7565	orig = data = RREG32_PCIE(PCIE_P_CNTL);
		7566	data \|= P_IGNORE_EDB_ERR;
		7567	if (orig != data)
		7568	WREG32_PCIE(PCIE_P_CNTL, data);
		7569
		7570	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
		7571	data &= ~(LC_L0S_INACTIVITY_MASK \| LC_L1_INACTIVITY_MASK);
		7572	data \|= LC_PMI_TO_L1_DIS;
		7573	if (!disable_l0s)
		7574	data \|= LC_L0S_INACTIVITY(7);
		7575
		7576	if (!disable_l1) {
		7577	data \|= LC_L1_INACTIVITY(7);
		7578	data &= ~LC_PMI_TO_L1_DIS;
		7579	if (orig != data)
		7580	WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
		7581
		7582	if (!disable_plloff_in_l1) {
		7583	bool clk_req_support;
		7584
		7585	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
		7586	data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK \| PLL_POWER_STATE_IN_TXS2_0_MASK);
		7587	data \|= PLL_POWER_STATE_IN_OFF_0(7) \| PLL_POWER_STATE_IN_TXS2_0(7);
		7588	if (orig != data)
		7589	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
		7590
		7591	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
		7592	data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK \| PLL_POWER_STATE_IN_TXS2_1_MASK);
		7593	data \|= PLL_POWER_STATE_IN_OFF_1(7) \| PLL_POWER_STATE_IN_TXS2_1(7);
		7594	if (orig != data)
		7595	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
		7596
		7597	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
		7598	data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK \| PLL_POWER_STATE_IN_TXS2_0_MASK);
		7599	data \|= PLL_POWER_STATE_IN_OFF_0(7) \| PLL_POWER_STATE_IN_TXS2_0(7);
		7600	if (orig != data)
		7601	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
		7602
		7603	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
		7604	data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK \| PLL_POWER_STATE_IN_TXS2_1_MASK);
		7605	data \|= PLL_POWER_STATE_IN_OFF_1(7) \| PLL_POWER_STATE_IN_TXS2_1(7);
		7606	if (orig != data)
		7607	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
		7608
		7609	if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
		7610	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
		7611	data &= ~PLL_RAMP_UP_TIME_0_MASK;
		7612	if (orig != data)
		7613	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
		7614
		7615	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
		7616	data &= ~PLL_RAMP_UP_TIME_1_MASK;
		7617	if (orig != data)
		7618	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
		7619
		7620	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
		7621	data &= ~PLL_RAMP_UP_TIME_2_MASK;
		7622	if (orig != data)
		7623	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
		7624
		7625	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
		7626	data &= ~PLL_RAMP_UP_TIME_3_MASK;
		7627	if (orig != data)
		7628	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
		7629
		7630	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
		7631	data &= ~PLL_RAMP_UP_TIME_0_MASK;
		7632	if (orig != data)
		7633	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
		7634
		7635	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
		7636	data &= ~PLL_RAMP_UP_TIME_1_MASK;
		7637	if (orig != data)
		7638	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
		7639
		7640	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
		7641	data &= ~PLL_RAMP_UP_TIME_2_MASK;
		7642	if (orig != data)
		7643	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
		7644
		7645	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
		7646	data &= ~PLL_RAMP_UP_TIME_3_MASK;
		7647	if (orig != data)
		7648	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
		7649	}
		7650	orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
		7651	data &= ~LC_DYN_LANES_PWR_STATE_MASK;
		7652	data \|= LC_DYN_LANES_PWR_STATE(3);
		7653	if (orig != data)
		7654	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
		7655
		7656	orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
		7657	data &= ~LS2_EXIT_TIME_MASK;
		7658	if ((rdev->family == CHIP_OLAND) \|\| (rdev->family == CHIP_HAINAN))
		7659	data \|= LS2_EXIT_TIME(5);
		7660	if (orig != data)
		7661	WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
		7662
		7663	orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
		7664	data &= ~LS2_EXIT_TIME_MASK;
		7665	if ((rdev->family == CHIP_OLAND) \|\| (rdev->family == CHIP_HAINAN))
		7666	data \|= LS2_EXIT_TIME(5);
		7667	if (orig != data)
		7668	WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
		7669
		7670	if (!disable_clkreq) {
		7671	struct pci_dev *root = rdev->pdev->bus->self;
		7672	u32 lnkcap;
		7673
		7674	clk_req_support = false;
		7675	pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
		7676	if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
		7677	clk_req_support = true;
		7678	} else {
		7679	clk_req_support = false;
		7680	}
		7681
		7682	if (clk_req_support) {
		7683	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
		7684	data \|= LC_ALLOW_PDWN_IN_L1 \| LC_ALLOW_PDWN_IN_L23;
		7685	if (orig != data)
		7686	WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
		7687
		7688	orig = data = RREG32(THM_CLK_CNTL);
		7689	data &= ~(CMON_CLK_SEL_MASK \| TMON_CLK_SEL_MASK);
		7690	data \|= CMON_CLK_SEL(1) \| TMON_CLK_SEL(1);
		7691	if (orig != data)
		7692	WREG32(THM_CLK_CNTL, data);
		7693
		7694	orig = data = RREG32(MISC_CLK_CNTL);
		7695	data &= ~(DEEP_SLEEP_CLK_SEL_MASK \| ZCLK_SEL_MASK);
		7696	data \|= DEEP_SLEEP_CLK_SEL(1) \| ZCLK_SEL(1);
		7697	if (orig != data)
		7698	WREG32(MISC_CLK_CNTL, data);
		7699
		7700	orig = data = RREG32(CG_CLKPIN_CNTL);
		7701	data &= ~BCLK_AS_XCLK;
		7702	if (orig != data)
		7703	WREG32(CG_CLKPIN_CNTL, data);
		7704
		7705	orig = data = RREG32(CG_CLKPIN_CNTL_2);
		7706	data &= ~FORCE_BIF_REFCLK_EN;
		7707	if (orig != data)
		7708	WREG32(CG_CLKPIN_CNTL_2, data);
		7709
		7710	orig = data = RREG32(MPLL_BYPASSCLK_SEL);
		7711	data &= ~MPLL_CLKOUT_SEL_MASK;
		7712	data \|= MPLL_CLKOUT_SEL(4);
		7713	if (orig != data)
		7714	WREG32(MPLL_BYPASSCLK_SEL, data);
		7715
		7716	orig = data = RREG32(SPLL_CNTL_MODE);
		7717	data &= ~SPLL_REFCLK_SEL_MASK;
		7718	if (orig != data)
		7719	WREG32(SPLL_CNTL_MODE, data);
		7720	}
		7721	}
		7722	} else {
		7723	if (orig != data)
		7724	WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
		7725	}
		7726
		7727	orig = data = RREG32_PCIE(PCIE_CNTL2);
		7728	data \|= SLV_MEM_LS_EN \| MST_MEM_LS_EN \| REPLAY_MEM_LS_EN;
		7729	if (orig != data)
		7730	WREG32_PCIE(PCIE_CNTL2, data);
		7731
		7732	if (!disable_l0s) {
		7733	data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
		7734	if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
		7735	data = RREG32_PCIE(PCIE_LC_STATUS1);
		7736	if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
		7737	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
		7738	data &= ~LC_L0S_INACTIVITY_MASK;
		7739	if (orig != data)
		7740	WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
		7741	}
		7742	}
		7743	}
		7744	}
6104	serge	7745
		7746	int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
		7747	{
		7748	unsigned i;
		7749
		7750	/* make sure VCEPLL_CTLREQ is deasserted */
		7751	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
		7752
		7753	mdelay(10);
		7754
		7755	/* assert UPLL_CTLREQ */
		7756	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
		7757
		7758	/* wait for CTLACK and CTLACK2 to get asserted */
		7759	for (i = 0; i < 100; ++i) {
		7760	uint32_t mask = UPLL_CTLACK_MASK \| UPLL_CTLACK2_MASK;
		7761	if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
		7762	break;
		7763	mdelay(10);
		7764	}
		7765
		7766	/* deassert UPLL_CTLREQ */
		7767	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
		7768
		7769	if (i == 100) {
		7770	DRM_ERROR("Timeout setting UVD clocks!\n");
		7771	return -ETIMEDOUT;
		7772	}
		7773
		7774	return 0;
		7775	}
		7776
		7777	int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
		7778	{
		7779	unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
		7780	int r;
		7781
		7782	/* bypass evclk and ecclk with bclk */
		7783	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
		7784	EVCLK_SRC_SEL(1) \| ECCLK_SRC_SEL(1),
		7785	~(EVCLK_SRC_SEL_MASK \| ECCLK_SRC_SEL_MASK));
		7786
		7787	/* put PLL in bypass mode */
		7788	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
		7789	~VCEPLL_BYPASS_EN_MASK);
		7790
		7791	if (!evclk \|\| !ecclk) {
		7792	/* keep the Bypass mode, put PLL to sleep */
		7793	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
		7794	~VCEPLL_SLEEP_MASK);
		7795	return 0;
		7796	}
		7797
		7798	r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
		7799	16384, 0x03FFFFFF, 0, 128, 5,
		7800	&fb_div, &evclk_div, &ecclk_div);
		7801	if (r)
		7802	return r;
		7803
		7804	/* set RESET_ANTI_MUX to 0 */
		7805	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
		7806
		7807	/* set VCO_MODE to 1 */
		7808	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
		7809	~VCEPLL_VCO_MODE_MASK);
		7810
		7811	/* toggle VCEPLL_SLEEP to 1 then back to 0 */
		7812	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
		7813	~VCEPLL_SLEEP_MASK);
		7814	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
		7815
		7816	/* deassert VCEPLL_RESET */
		7817	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
		7818
		7819	mdelay(1);
		7820
		7821	r = si_vce_send_vcepll_ctlreq(rdev);
		7822	if (r)
		7823	return r;
		7824
		7825	/* assert VCEPLL_RESET again */
		7826	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
		7827
		7828	/* disable spread spectrum. */
		7829	WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
		7830
		7831	/* set feedback divider */
		7832	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
		7833
		7834	/* set ref divider to 0 */
		7835	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
		7836
		7837	/* set PDIV_A and PDIV_B */
		7838	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
		7839	VCEPLL_PDIV_A(evclk_div) \| VCEPLL_PDIV_B(ecclk_div),
		7840	~(VCEPLL_PDIV_A_MASK \| VCEPLL_PDIV_B_MASK));
		7841
		7842	/* give the PLL some time to settle */
		7843	mdelay(15);
		7844
		7845	/* deassert PLL_RESET */
		7846	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
		7847
		7848	mdelay(15);
		7849
		7850	/* switch from bypass mode to normal mode */
		7851	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
		7852
		7853	r = si_vce_send_vcepll_ctlreq(rdev);
		7854	if (r)
		7855	return r;
		7856
		7857	/* switch VCLK and DCLK selection */
		7858	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
		7859	EVCLK_SRC_SEL(16) \| ECCLK_SRC_SEL(16),
		7860	~(EVCLK_SRC_SEL_MASK \| ECCLK_SRC_SEL_MASK));
		7861
		7862	mdelay(100);
		7863
		7864	return 0;
		7865	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/si.c – Rev 6104