WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/si.c

Rev	Author	Line No.	Line
2997	Serge	1	/*
		2	* Copyright 2011 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	#include
		26	#include
		27	#include
		28	#include "radeon.h"
		29	#include "radeon_asic.h"
6104	serge	30	#include "radeon_audio.h"
2997	Serge	31	#include
		32	#include "sid.h"
		33	#include "atom.h"
		34	#include "si_blit_shaders.h"
5078	serge	35	#include "clearstate_si.h"
		36	#include "radeon_ucode.h"
2997	Serge	37
		38
		39	MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
		40	MODULE_FIRMWARE("radeon/TAHITI_me.bin");
		41	MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
		42	MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
5078	serge	43	MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
2997	Serge	44	MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
5078	serge	45	MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
		46
		47	MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
		48	MODULE_FIRMWARE("radeon/tahiti_me.bin");
		49	MODULE_FIRMWARE("radeon/tahiti_ce.bin");
		50	MODULE_FIRMWARE("radeon/tahiti_mc.bin");
		51	MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
		52	MODULE_FIRMWARE("radeon/tahiti_smc.bin");
		53
2997	Serge	54	MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
		55	MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
		56	MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
		57	MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
5078	serge	58	MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
2997	Serge	59	MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
5078	serge	60	MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
		61
		62	MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
		63	MODULE_FIRMWARE("radeon/pitcairn_me.bin");
		64	MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
		65	MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
		66	MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
		67	MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
		68
2997	Serge	69	MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
		70	MODULE_FIRMWARE("radeon/VERDE_me.bin");
		71	MODULE_FIRMWARE("radeon/VERDE_ce.bin");
		72	MODULE_FIRMWARE("radeon/VERDE_mc.bin");
5078	serge	73	MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
2997	Serge	74	MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
5078	serge	75	MODULE_FIRMWARE("radeon/VERDE_smc.bin");
		76
		77	MODULE_FIRMWARE("radeon/verde_pfp.bin");
		78	MODULE_FIRMWARE("radeon/verde_me.bin");
		79	MODULE_FIRMWARE("radeon/verde_ce.bin");
		80	MODULE_FIRMWARE("radeon/verde_mc.bin");
		81	MODULE_FIRMWARE("radeon/verde_rlc.bin");
		82	MODULE_FIRMWARE("radeon/verde_smc.bin");
		83
3764	Serge	84	MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
		85	MODULE_FIRMWARE("radeon/OLAND_me.bin");
		86	MODULE_FIRMWARE("radeon/OLAND_ce.bin");
		87	MODULE_FIRMWARE("radeon/OLAND_mc.bin");
5078	serge	88	MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
3764	Serge	89	MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
5078	serge	90	MODULE_FIRMWARE("radeon/OLAND_smc.bin");
		91
		92	MODULE_FIRMWARE("radeon/oland_pfp.bin");
		93	MODULE_FIRMWARE("radeon/oland_me.bin");
		94	MODULE_FIRMWARE("radeon/oland_ce.bin");
		95	MODULE_FIRMWARE("radeon/oland_mc.bin");
		96	MODULE_FIRMWARE("radeon/oland_rlc.bin");
		97	MODULE_FIRMWARE("radeon/oland_smc.bin");
		98
3764	Serge	99	MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
		100	MODULE_FIRMWARE("radeon/HAINAN_me.bin");
		101	MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
		102	MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
5078	serge	103	MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
3764	Serge	104	MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
5078	serge	105	MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
2997	Serge	106
5078	serge	107	MODULE_FIRMWARE("radeon/hainan_pfp.bin");
		108	MODULE_FIRMWARE("radeon/hainan_me.bin");
		109	MODULE_FIRMWARE("radeon/hainan_ce.bin");
		110	MODULE_FIRMWARE("radeon/hainan_mc.bin");
		111	MODULE_FIRMWARE("radeon/hainan_rlc.bin");
		112	MODULE_FIRMWARE("radeon/hainan_smc.bin");
		113
		114	static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
		115	static void si_pcie_gen3_enable(struct radeon_device *rdev);
		116	static void si_program_aspm(struct radeon_device *rdev);
		117	extern void sumo_rlc_fini(struct radeon_device *rdev);
		118	extern int sumo_rlc_init(struct radeon_device *rdev);
2997	Serge	119	extern int r600_ih_ring_alloc(struct radeon_device *rdev);
		120	extern void r600_ih_ring_fini(struct radeon_device *rdev);
		121	extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
		122	extern void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save);
		123	extern void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save);
		124	extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
3764	Serge	125	extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
		126	extern bool evergreen_is_display_hung(struct radeon_device *rdev);
5078	serge	127	static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
		128	bool enable);
		129	static void si_init_pg(struct radeon_device *rdev);
		130	static void si_init_cg(struct radeon_device *rdev);
		131	static void si_fini_pg(struct radeon_device *rdev);
		132	static void si_fini_cg(struct radeon_device *rdev);
		133	static void si_rlc_stop(struct radeon_device *rdev);
2997	Serge	134
5078	serge	135	static const u32 verde_rlc_save_restore_register_list[] =
		136	{
		137	(0x8000 << 16) \| (0x98f4 >> 2),
		138	0x00000000,
		139	(0x8040 << 16) \| (0x98f4 >> 2),
		140	0x00000000,
		141	(0x8000 << 16) \| (0xe80 >> 2),
		142	0x00000000,
		143	(0x8040 << 16) \| (0xe80 >> 2),
		144	0x00000000,
		145	(0x8000 << 16) \| (0x89bc >> 2),
		146	0x00000000,
		147	(0x8040 << 16) \| (0x89bc >> 2),
		148	0x00000000,
		149	(0x8000 << 16) \| (0x8c1c >> 2),
		150	0x00000000,
		151	(0x8040 << 16) \| (0x8c1c >> 2),
		152	0x00000000,
		153	(0x9c00 << 16) \| (0x98f0 >> 2),
		154	0x00000000,
		155	(0x9c00 << 16) \| (0xe7c >> 2),
		156	0x00000000,
		157	(0x8000 << 16) \| (0x9148 >> 2),
		158	0x00000000,
		159	(0x8040 << 16) \| (0x9148 >> 2),
		160	0x00000000,
		161	(0x9c00 << 16) \| (0x9150 >> 2),
		162	0x00000000,
		163	(0x9c00 << 16) \| (0x897c >> 2),
		164	0x00000000,
		165	(0x9c00 << 16) \| (0x8d8c >> 2),
		166	0x00000000,
		167	(0x9c00 << 16) \| (0xac54 >> 2),
		168	0X00000000,
		169	0x3,
		170	(0x9c00 << 16) \| (0x98f8 >> 2),
		171	0x00000000,
		172	(0x9c00 << 16) \| (0x9910 >> 2),
		173	0x00000000,
		174	(0x9c00 << 16) \| (0x9914 >> 2),
		175	0x00000000,
		176	(0x9c00 << 16) \| (0x9918 >> 2),
		177	0x00000000,
		178	(0x9c00 << 16) \| (0x991c >> 2),
		179	0x00000000,
		180	(0x9c00 << 16) \| (0x9920 >> 2),
		181	0x00000000,
		182	(0x9c00 << 16) \| (0x9924 >> 2),
		183	0x00000000,
		184	(0x9c00 << 16) \| (0x9928 >> 2),
		185	0x00000000,
		186	(0x9c00 << 16) \| (0x992c >> 2),
		187	0x00000000,
		188	(0x9c00 << 16) \| (0x9930 >> 2),
		189	0x00000000,
		190	(0x9c00 << 16) \| (0x9934 >> 2),
		191	0x00000000,
		192	(0x9c00 << 16) \| (0x9938 >> 2),
		193	0x00000000,
		194	(0x9c00 << 16) \| (0x993c >> 2),
		195	0x00000000,
		196	(0x9c00 << 16) \| (0x9940 >> 2),
		197	0x00000000,
		198	(0x9c00 << 16) \| (0x9944 >> 2),
		199	0x00000000,
		200	(0x9c00 << 16) \| (0x9948 >> 2),
		201	0x00000000,
		202	(0x9c00 << 16) \| (0x994c >> 2),
		203	0x00000000,
		204	(0x9c00 << 16) \| (0x9950 >> 2),
		205	0x00000000,
		206	(0x9c00 << 16) \| (0x9954 >> 2),
		207	0x00000000,
		208	(0x9c00 << 16) \| (0x9958 >> 2),
		209	0x00000000,
		210	(0x9c00 << 16) \| (0x995c >> 2),
		211	0x00000000,
		212	(0x9c00 << 16) \| (0x9960 >> 2),
		213	0x00000000,
		214	(0x9c00 << 16) \| (0x9964 >> 2),
		215	0x00000000,
		216	(0x9c00 << 16) \| (0x9968 >> 2),
		217	0x00000000,
		218	(0x9c00 << 16) \| (0x996c >> 2),
		219	0x00000000,
		220	(0x9c00 << 16) \| (0x9970 >> 2),
		221	0x00000000,
		222	(0x9c00 << 16) \| (0x9974 >> 2),
		223	0x00000000,
		224	(0x9c00 << 16) \| (0x9978 >> 2),
		225	0x00000000,
		226	(0x9c00 << 16) \| (0x997c >> 2),
		227	0x00000000,
		228	(0x9c00 << 16) \| (0x9980 >> 2),
		229	0x00000000,
		230	(0x9c00 << 16) \| (0x9984 >> 2),
		231	0x00000000,
		232	(0x9c00 << 16) \| (0x9988 >> 2),
		233	0x00000000,
		234	(0x9c00 << 16) \| (0x998c >> 2),
		235	0x00000000,
		236	(0x9c00 << 16) \| (0x8c00 >> 2),
		237	0x00000000,
		238	(0x9c00 << 16) \| (0x8c14 >> 2),
		239	0x00000000,
		240	(0x9c00 << 16) \| (0x8c04 >> 2),
		241	0x00000000,
		242	(0x9c00 << 16) \| (0x8c08 >> 2),
		243	0x00000000,
		244	(0x8000 << 16) \| (0x9b7c >> 2),
		245	0x00000000,
		246	(0x8040 << 16) \| (0x9b7c >> 2),
		247	0x00000000,
		248	(0x8000 << 16) \| (0xe84 >> 2),
		249	0x00000000,
		250	(0x8040 << 16) \| (0xe84 >> 2),
		251	0x00000000,
		252	(0x8000 << 16) \| (0x89c0 >> 2),
		253	0x00000000,
		254	(0x8040 << 16) \| (0x89c0 >> 2),
		255	0x00000000,
		256	(0x8000 << 16) \| (0x914c >> 2),
		257	0x00000000,
		258	(0x8040 << 16) \| (0x914c >> 2),
		259	0x00000000,
		260	(0x8000 << 16) \| (0x8c20 >> 2),
		261	0x00000000,
		262	(0x8040 << 16) \| (0x8c20 >> 2),
		263	0x00000000,
		264	(0x8000 << 16) \| (0x9354 >> 2),
		265	0x00000000,
		266	(0x8040 << 16) \| (0x9354 >> 2),
		267	0x00000000,
		268	(0x9c00 << 16) \| (0x9060 >> 2),
		269	0x00000000,
		270	(0x9c00 << 16) \| (0x9364 >> 2),
		271	0x00000000,
		272	(0x9c00 << 16) \| (0x9100 >> 2),
		273	0x00000000,
		274	(0x9c00 << 16) \| (0x913c >> 2),
		275	0x00000000,
		276	(0x8000 << 16) \| (0x90e0 >> 2),
		277	0x00000000,
		278	(0x8000 << 16) \| (0x90e4 >> 2),
		279	0x00000000,
		280	(0x8000 << 16) \| (0x90e8 >> 2),
		281	0x00000000,
		282	(0x8040 << 16) \| (0x90e0 >> 2),
		283	0x00000000,
		284	(0x8040 << 16) \| (0x90e4 >> 2),
		285	0x00000000,
		286	(0x8040 << 16) \| (0x90e8 >> 2),
		287	0x00000000,
		288	(0x9c00 << 16) \| (0x8bcc >> 2),
		289	0x00000000,
		290	(0x9c00 << 16) \| (0x8b24 >> 2),
		291	0x00000000,
		292	(0x9c00 << 16) \| (0x88c4 >> 2),
		293	0x00000000,
		294	(0x9c00 << 16) \| (0x8e50 >> 2),
		295	0x00000000,
		296	(0x9c00 << 16) \| (0x8c0c >> 2),
		297	0x00000000,
		298	(0x9c00 << 16) \| (0x8e58 >> 2),
		299	0x00000000,
		300	(0x9c00 << 16) \| (0x8e5c >> 2),
		301	0x00000000,
		302	(0x9c00 << 16) \| (0x9508 >> 2),
		303	0x00000000,
		304	(0x9c00 << 16) \| (0x950c >> 2),
		305	0x00000000,
		306	(0x9c00 << 16) \| (0x9494 >> 2),
		307	0x00000000,
		308	(0x9c00 << 16) \| (0xac0c >> 2),
		309	0x00000000,
		310	(0x9c00 << 16) \| (0xac10 >> 2),
		311	0x00000000,
		312	(0x9c00 << 16) \| (0xac14 >> 2),
		313	0x00000000,
		314	(0x9c00 << 16) \| (0xae00 >> 2),
		315	0x00000000,
		316	(0x9c00 << 16) \| (0xac08 >> 2),
		317	0x00000000,
		318	(0x9c00 << 16) \| (0x88d4 >> 2),
		319	0x00000000,
		320	(0x9c00 << 16) \| (0x88c8 >> 2),
		321	0x00000000,
		322	(0x9c00 << 16) \| (0x88cc >> 2),
		323	0x00000000,
		324	(0x9c00 << 16) \| (0x89b0 >> 2),
		325	0x00000000,
		326	(0x9c00 << 16) \| (0x8b10 >> 2),
		327	0x00000000,
		328	(0x9c00 << 16) \| (0x8a14 >> 2),
		329	0x00000000,
		330	(0x9c00 << 16) \| (0x9830 >> 2),
		331	0x00000000,
		332	(0x9c00 << 16) \| (0x9834 >> 2),
		333	0x00000000,
		334	(0x9c00 << 16) \| (0x9838 >> 2),
		335	0x00000000,
		336	(0x9c00 << 16) \| (0x9a10 >> 2),
		337	0x00000000,
		338	(0x8000 << 16) \| (0x9870 >> 2),
		339	0x00000000,
		340	(0x8000 << 16) \| (0x9874 >> 2),
		341	0x00000000,
		342	(0x8001 << 16) \| (0x9870 >> 2),
		343	0x00000000,
		344	(0x8001 << 16) \| (0x9874 >> 2),
		345	0x00000000,
		346	(0x8040 << 16) \| (0x9870 >> 2),
		347	0x00000000,
		348	(0x8040 << 16) \| (0x9874 >> 2),
		349	0x00000000,
		350	(0x8041 << 16) \| (0x9870 >> 2),
		351	0x00000000,
		352	(0x8041 << 16) \| (0x9874 >> 2),
		353	0x00000000,
		354	0x00000000
		355	};
		356
3764	Serge	357	static const u32 tahiti_golden_rlc_registers[] =
		358	{
		359	0xc424, 0xffffffff, 0x00601005,
		360	0xc47c, 0xffffffff, 0x10104040,
		361	0xc488, 0xffffffff, 0x0100000a,
		362	0xc314, 0xffffffff, 0x00000800,
		363	0xc30c, 0xffffffff, 0x800000f4,
		364	0xf4a8, 0xffffffff, 0x00000000
		365	};
		366
		367	static const u32 tahiti_golden_registers[] =
		368	{
		369	0x9a10, 0x00010000, 0x00018208,
		370	0x9830, 0xffffffff, 0x00000000,
		371	0x9834, 0xf00fffff, 0x00000400,
		372	0x9838, 0x0002021c, 0x00020200,
		373	0xc78, 0x00000080, 0x00000000,
		374	0xd030, 0x000300c0, 0x00800040,
		375	0xd830, 0x000300c0, 0x00800040,
		376	0x5bb0, 0x000000f0, 0x00000070,
		377	0x5bc0, 0x00200000, 0x50100000,
		378	0x7030, 0x31000311, 0x00000011,
		379	0x277c, 0x00000003, 0x000007ff,
		380	0x240c, 0x000007ff, 0x00000000,
		381	0x8a14, 0xf000001f, 0x00000007,
		382	0x8b24, 0xffffffff, 0x00ffffff,
		383	0x8b10, 0x0000ff0f, 0x00000000,
		384	0x28a4c, 0x07ffffff, 0x4e000000,
		385	0x28350, 0x3f3f3fff, 0x2a00126a,
		386	0x30, 0x000000ff, 0x0040,
		387	0x34, 0x00000040, 0x00004040,
		388	0x9100, 0x07ffffff, 0x03000000,
		389	0x8e88, 0x01ff1f3f, 0x00000000,
		390	0x8e84, 0x01ff1f3f, 0x00000000,
		391	0x9060, 0x0000007f, 0x00000020,
		392	0x9508, 0x00010000, 0x00010000,
		393	0xac14, 0x00000200, 0x000002fb,
		394	0xac10, 0xffffffff, 0x0000543b,
		395	0xac0c, 0xffffffff, 0xa9210876,
		396	0x88d0, 0xffffffff, 0x000fff40,
		397	0x88d4, 0x0000001f, 0x00000010,
		398	0x1410, 0x20000000, 0x20fffed8,
		399	0x15c0, 0x000c0fc0, 0x000c0400
		400	};
		401
		402	static const u32 tahiti_golden_registers2[] =
		403	{
		404	0xc64, 0x00000001, 0x00000001
		405	};
		406
		407	static const u32 pitcairn_golden_rlc_registers[] =
		408	{
		409	0xc424, 0xffffffff, 0x00601004,
		410	0xc47c, 0xffffffff, 0x10102020,
		411	0xc488, 0xffffffff, 0x01000020,
		412	0xc314, 0xffffffff, 0x00000800,
		413	0xc30c, 0xffffffff, 0x800000a4
		414	};
		415
		416	static const u32 pitcairn_golden_registers[] =
		417	{
		418	0x9a10, 0x00010000, 0x00018208,
		419	0x9830, 0xffffffff, 0x00000000,
		420	0x9834, 0xf00fffff, 0x00000400,
		421	0x9838, 0x0002021c, 0x00020200,
		422	0xc78, 0x00000080, 0x00000000,
		423	0xd030, 0x000300c0, 0x00800040,
		424	0xd830, 0x000300c0, 0x00800040,
		425	0x5bb0, 0x000000f0, 0x00000070,
		426	0x5bc0, 0x00200000, 0x50100000,
		427	0x7030, 0x31000311, 0x00000011,
		428	0x2ae4, 0x00073ffe, 0x000022a2,
		429	0x240c, 0x000007ff, 0x00000000,
		430	0x8a14, 0xf000001f, 0x00000007,
		431	0x8b24, 0xffffffff, 0x00ffffff,
		432	0x8b10, 0x0000ff0f, 0x00000000,
		433	0x28a4c, 0x07ffffff, 0x4e000000,
		434	0x28350, 0x3f3f3fff, 0x2a00126a,
		435	0x30, 0x000000ff, 0x0040,
		436	0x34, 0x00000040, 0x00004040,
		437	0x9100, 0x07ffffff, 0x03000000,
		438	0x9060, 0x0000007f, 0x00000020,
		439	0x9508, 0x00010000, 0x00010000,
		440	0xac14, 0x000003ff, 0x000000f7,
		441	0xac10, 0xffffffff, 0x00000000,
		442	0xac0c, 0xffffffff, 0x32761054,
		443	0x88d4, 0x0000001f, 0x00000010,
		444	0x15c0, 0x000c0fc0, 0x000c0400
		445	};
		446
		447	static const u32 verde_golden_rlc_registers[] =
		448	{
		449	0xc424, 0xffffffff, 0x033f1005,
		450	0xc47c, 0xffffffff, 0x10808020,
		451	0xc488, 0xffffffff, 0x00800008,
		452	0xc314, 0xffffffff, 0x00001000,
		453	0xc30c, 0xffffffff, 0x80010014
		454	};
		455
		456	static const u32 verde_golden_registers[] =
		457	{
		458	0x9a10, 0x00010000, 0x00018208,
		459	0x9830, 0xffffffff, 0x00000000,
		460	0x9834, 0xf00fffff, 0x00000400,
		461	0x9838, 0x0002021c, 0x00020200,
		462	0xc78, 0x00000080, 0x00000000,
		463	0xd030, 0x000300c0, 0x00800040,
		464	0xd030, 0x000300c0, 0x00800040,
		465	0xd830, 0x000300c0, 0x00800040,
		466	0xd830, 0x000300c0, 0x00800040,
		467	0x5bb0, 0x000000f0, 0x00000070,
		468	0x5bc0, 0x00200000, 0x50100000,
		469	0x7030, 0x31000311, 0x00000011,
		470	0x2ae4, 0x00073ffe, 0x000022a2,
		471	0x2ae4, 0x00073ffe, 0x000022a2,
		472	0x2ae4, 0x00073ffe, 0x000022a2,
		473	0x240c, 0x000007ff, 0x00000000,
		474	0x240c, 0x000007ff, 0x00000000,
		475	0x240c, 0x000007ff, 0x00000000,
		476	0x8a14, 0xf000001f, 0x00000007,
		477	0x8a14, 0xf000001f, 0x00000007,
		478	0x8a14, 0xf000001f, 0x00000007,
		479	0x8b24, 0xffffffff, 0x00ffffff,
		480	0x8b10, 0x0000ff0f, 0x00000000,
		481	0x28a4c, 0x07ffffff, 0x4e000000,
		482	0x28350, 0x3f3f3fff, 0x0000124a,
		483	0x28350, 0x3f3f3fff, 0x0000124a,
		484	0x28350, 0x3f3f3fff, 0x0000124a,
		485	0x30, 0x000000ff, 0x0040,
		486	0x34, 0x00000040, 0x00004040,
		487	0x9100, 0x07ffffff, 0x03000000,
		488	0x9100, 0x07ffffff, 0x03000000,
		489	0x8e88, 0x01ff1f3f, 0x00000000,
		490	0x8e88, 0x01ff1f3f, 0x00000000,
		491	0x8e88, 0x01ff1f3f, 0x00000000,
		492	0x8e84, 0x01ff1f3f, 0x00000000,
		493	0x8e84, 0x01ff1f3f, 0x00000000,
		494	0x8e84, 0x01ff1f3f, 0x00000000,
		495	0x9060, 0x0000007f, 0x00000020,
		496	0x9508, 0x00010000, 0x00010000,
		497	0xac14, 0x000003ff, 0x00000003,
		498	0xac14, 0x000003ff, 0x00000003,
		499	0xac14, 0x000003ff, 0x00000003,
		500	0xac10, 0xffffffff, 0x00000000,
		501	0xac10, 0xffffffff, 0x00000000,
		502	0xac10, 0xffffffff, 0x00000000,
		503	0xac0c, 0xffffffff, 0x00001032,
		504	0xac0c, 0xffffffff, 0x00001032,
		505	0xac0c, 0xffffffff, 0x00001032,
		506	0x88d4, 0x0000001f, 0x00000010,
		507	0x88d4, 0x0000001f, 0x00000010,
		508	0x88d4, 0x0000001f, 0x00000010,
		509	0x15c0, 0x000c0fc0, 0x000c0400
		510	};
		511
		512	static const u32 oland_golden_rlc_registers[] =
		513	{
		514	0xc424, 0xffffffff, 0x00601005,
		515	0xc47c, 0xffffffff, 0x10104040,
		516	0xc488, 0xffffffff, 0x0100000a,
		517	0xc314, 0xffffffff, 0x00000800,
		518	0xc30c, 0xffffffff, 0x800000f4
		519	};
		520
		521	static const u32 oland_golden_registers[] =
		522	{
		523	0x9a10, 0x00010000, 0x00018208,
		524	0x9830, 0xffffffff, 0x00000000,
		525	0x9834, 0xf00fffff, 0x00000400,
		526	0x9838, 0x0002021c, 0x00020200,
		527	0xc78, 0x00000080, 0x00000000,
		528	0xd030, 0x000300c0, 0x00800040,
		529	0xd830, 0x000300c0, 0x00800040,
		530	0x5bb0, 0x000000f0, 0x00000070,
		531	0x5bc0, 0x00200000, 0x50100000,
		532	0x7030, 0x31000311, 0x00000011,
		533	0x2ae4, 0x00073ffe, 0x000022a2,
		534	0x240c, 0x000007ff, 0x00000000,
		535	0x8a14, 0xf000001f, 0x00000007,
		536	0x8b24, 0xffffffff, 0x00ffffff,
		537	0x8b10, 0x0000ff0f, 0x00000000,
		538	0x28a4c, 0x07ffffff, 0x4e000000,
		539	0x28350, 0x3f3f3fff, 0x00000082,
		540	0x30, 0x000000ff, 0x0040,
		541	0x34, 0x00000040, 0x00004040,
		542	0x9100, 0x07ffffff, 0x03000000,
		543	0x9060, 0x0000007f, 0x00000020,
		544	0x9508, 0x00010000, 0x00010000,
		545	0xac14, 0x000003ff, 0x000000f3,
		546	0xac10, 0xffffffff, 0x00000000,
		547	0xac0c, 0xffffffff, 0x00003210,
		548	0x88d4, 0x0000001f, 0x00000010,
		549	0x15c0, 0x000c0fc0, 0x000c0400
		550	};
		551
		552	static const u32 hainan_golden_registers[] =
		553	{
		554	0x9a10, 0x00010000, 0x00018208,
		555	0x9830, 0xffffffff, 0x00000000,
		556	0x9834, 0xf00fffff, 0x00000400,
		557	0x9838, 0x0002021c, 0x00020200,
		558	0xd0c0, 0xff000fff, 0x00000100,
		559	0xd030, 0x000300c0, 0x00800040,
		560	0xd8c0, 0xff000fff, 0x00000100,
		561	0xd830, 0x000300c0, 0x00800040,
		562	0x2ae4, 0x00073ffe, 0x000022a2,
		563	0x240c, 0x000007ff, 0x00000000,
		564	0x8a14, 0xf000001f, 0x00000007,
		565	0x8b24, 0xffffffff, 0x00ffffff,
		566	0x8b10, 0x0000ff0f, 0x00000000,
		567	0x28a4c, 0x07ffffff, 0x4e000000,
		568	0x28350, 0x3f3f3fff, 0x00000000,
		569	0x30, 0x000000ff, 0x0040,
		570	0x34, 0x00000040, 0x00004040,
		571	0x9100, 0x03e00000, 0x03600000,
		572	0x9060, 0x0000007f, 0x00000020,
		573	0x9508, 0x00010000, 0x00010000,
		574	0xac14, 0x000003ff, 0x000000f1,
		575	0xac10, 0xffffffff, 0x00000000,
		576	0xac0c, 0xffffffff, 0x00003210,
		577	0x88d4, 0x0000001f, 0x00000010,
		578	0x15c0, 0x000c0fc0, 0x000c0400
		579	};
		580
		581	static const u32 hainan_golden_registers2[] =
		582	{
		583	0x98f8, 0xffffffff, 0x02010001
		584	};
		585
		586	static const u32 tahiti_mgcg_cgcg_init[] =
		587	{
		588	0xc400, 0xffffffff, 0xfffffffc,
		589	0x802c, 0xffffffff, 0xe0000000,
		590	0x9a60, 0xffffffff, 0x00000100,
		591	0x92a4, 0xffffffff, 0x00000100,
		592	0xc164, 0xffffffff, 0x00000100,
		593	0x9774, 0xffffffff, 0x00000100,
		594	0x8984, 0xffffffff, 0x06000100,
		595	0x8a18, 0xffffffff, 0x00000100,
		596	0x92a0, 0xffffffff, 0x00000100,
		597	0xc380, 0xffffffff, 0x00000100,
		598	0x8b28, 0xffffffff, 0x00000100,
		599	0x9144, 0xffffffff, 0x00000100,
		600	0x8d88, 0xffffffff, 0x00000100,
		601	0x8d8c, 0xffffffff, 0x00000100,
		602	0x9030, 0xffffffff, 0x00000100,
		603	0x9034, 0xffffffff, 0x00000100,
		604	0x9038, 0xffffffff, 0x00000100,
		605	0x903c, 0xffffffff, 0x00000100,
		606	0xad80, 0xffffffff, 0x00000100,
		607	0xac54, 0xffffffff, 0x00000100,
		608	0x897c, 0xffffffff, 0x06000100,
		609	0x9868, 0xffffffff, 0x00000100,
		610	0x9510, 0xffffffff, 0x00000100,
		611	0xaf04, 0xffffffff, 0x00000100,
		612	0xae04, 0xffffffff, 0x00000100,
		613	0x949c, 0xffffffff, 0x00000100,
		614	0x802c, 0xffffffff, 0xe0000000,
		615	0x9160, 0xffffffff, 0x00010000,
		616	0x9164, 0xffffffff, 0x00030002,
		617	0x9168, 0xffffffff, 0x00040007,
		618	0x916c, 0xffffffff, 0x00060005,
		619	0x9170, 0xffffffff, 0x00090008,
		620	0x9174, 0xffffffff, 0x00020001,
		621	0x9178, 0xffffffff, 0x00040003,
		622	0x917c, 0xffffffff, 0x00000007,
		623	0x9180, 0xffffffff, 0x00060005,
		624	0x9184, 0xffffffff, 0x00090008,
		625	0x9188, 0xffffffff, 0x00030002,
		626	0x918c, 0xffffffff, 0x00050004,
		627	0x9190, 0xffffffff, 0x00000008,
		628	0x9194, 0xffffffff, 0x00070006,
		629	0x9198, 0xffffffff, 0x000a0009,
		630	0x919c, 0xffffffff, 0x00040003,
		631	0x91a0, 0xffffffff, 0x00060005,
		632	0x91a4, 0xffffffff, 0x00000009,
		633	0x91a8, 0xffffffff, 0x00080007,
		634	0x91ac, 0xffffffff, 0x000b000a,
		635	0x91b0, 0xffffffff, 0x00050004,
		636	0x91b4, 0xffffffff, 0x00070006,
		637	0x91b8, 0xffffffff, 0x0008000b,
		638	0x91bc, 0xffffffff, 0x000a0009,
		639	0x91c0, 0xffffffff, 0x000d000c,
		640	0x91c4, 0xffffffff, 0x00060005,
		641	0x91c8, 0xffffffff, 0x00080007,
		642	0x91cc, 0xffffffff, 0x0000000b,
		643	0x91d0, 0xffffffff, 0x000a0009,
		644	0x91d4, 0xffffffff, 0x000d000c,
		645	0x91d8, 0xffffffff, 0x00070006,
		646	0x91dc, 0xffffffff, 0x00090008,
		647	0x91e0, 0xffffffff, 0x0000000c,
		648	0x91e4, 0xffffffff, 0x000b000a,
		649	0x91e8, 0xffffffff, 0x000e000d,
		650	0x91ec, 0xffffffff, 0x00080007,
		651	0x91f0, 0xffffffff, 0x000a0009,
		652	0x91f4, 0xffffffff, 0x0000000d,
		653	0x91f8, 0xffffffff, 0x000c000b,
		654	0x91fc, 0xffffffff, 0x000f000e,
		655	0x9200, 0xffffffff, 0x00090008,
		656	0x9204, 0xffffffff, 0x000b000a,
		657	0x9208, 0xffffffff, 0x000c000f,
		658	0x920c, 0xffffffff, 0x000e000d,
		659	0x9210, 0xffffffff, 0x00110010,
		660	0x9214, 0xffffffff, 0x000a0009,
		661	0x9218, 0xffffffff, 0x000c000b,
		662	0x921c, 0xffffffff, 0x0000000f,
		663	0x9220, 0xffffffff, 0x000e000d,
		664	0x9224, 0xffffffff, 0x00110010,
		665	0x9228, 0xffffffff, 0x000b000a,
		666	0x922c, 0xffffffff, 0x000d000c,
		667	0x9230, 0xffffffff, 0x00000010,
		668	0x9234, 0xffffffff, 0x000f000e,
		669	0x9238, 0xffffffff, 0x00120011,
		670	0x923c, 0xffffffff, 0x000c000b,
		671	0x9240, 0xffffffff, 0x000e000d,
		672	0x9244, 0xffffffff, 0x00000011,
		673	0x9248, 0xffffffff, 0x0010000f,
		674	0x924c, 0xffffffff, 0x00130012,
		675	0x9250, 0xffffffff, 0x000d000c,
		676	0x9254, 0xffffffff, 0x000f000e,
		677	0x9258, 0xffffffff, 0x00100013,
		678	0x925c, 0xffffffff, 0x00120011,
		679	0x9260, 0xffffffff, 0x00150014,
		680	0x9264, 0xffffffff, 0x000e000d,
		681	0x9268, 0xffffffff, 0x0010000f,
		682	0x926c, 0xffffffff, 0x00000013,
		683	0x9270, 0xffffffff, 0x00120011,
		684	0x9274, 0xffffffff, 0x00150014,
		685	0x9278, 0xffffffff, 0x000f000e,
		686	0x927c, 0xffffffff, 0x00110010,
		687	0x9280, 0xffffffff, 0x00000014,
		688	0x9284, 0xffffffff, 0x00130012,
		689	0x9288, 0xffffffff, 0x00160015,
		690	0x928c, 0xffffffff, 0x0010000f,
		691	0x9290, 0xffffffff, 0x00120011,
		692	0x9294, 0xffffffff, 0x00000015,
		693	0x9298, 0xffffffff, 0x00140013,
		694	0x929c, 0xffffffff, 0x00170016,
		695	0x9150, 0xffffffff, 0x96940200,
		696	0x8708, 0xffffffff, 0x00900100,
		697	0xc478, 0xffffffff, 0x00000080,
		698	0xc404, 0xffffffff, 0x0020003f,
		699	0x30, 0xffffffff, 0x0000001c,
		700	0x34, 0x000f0000, 0x000f0000,
		701	0x160c, 0xffffffff, 0x00000100,
		702	0x1024, 0xffffffff, 0x00000100,
		703	0x102c, 0x00000101, 0x00000000,
		704	0x20a8, 0xffffffff, 0x00000104,
		705	0x264c, 0x000c0000, 0x000c0000,
		706	0x2648, 0x000c0000, 0x000c0000,
		707	0x55e4, 0xff000fff, 0x00000100,
		708	0x55e8, 0x00000001, 0x00000001,
		709	0x2f50, 0x00000001, 0x00000001,
		710	0x30cc, 0xc0000fff, 0x00000104,
		711	0xc1e4, 0x00000001, 0x00000001,
		712	0xd0c0, 0xfffffff0, 0x00000100,
		713	0xd8c0, 0xfffffff0, 0x00000100
		714	};
		715
		716	static const u32 pitcairn_mgcg_cgcg_init[] =
		717	{
		718	0xc400, 0xffffffff, 0xfffffffc,
		719	0x802c, 0xffffffff, 0xe0000000,
		720	0x9a60, 0xffffffff, 0x00000100,
		721	0x92a4, 0xffffffff, 0x00000100,
		722	0xc164, 0xffffffff, 0x00000100,
		723	0x9774, 0xffffffff, 0x00000100,
		724	0x8984, 0xffffffff, 0x06000100,
		725	0x8a18, 0xffffffff, 0x00000100,
		726	0x92a0, 0xffffffff, 0x00000100,
		727	0xc380, 0xffffffff, 0x00000100,
		728	0x8b28, 0xffffffff, 0x00000100,
		729	0x9144, 0xffffffff, 0x00000100,
		730	0x8d88, 0xffffffff, 0x00000100,
		731	0x8d8c, 0xffffffff, 0x00000100,
		732	0x9030, 0xffffffff, 0x00000100,
		733	0x9034, 0xffffffff, 0x00000100,
		734	0x9038, 0xffffffff, 0x00000100,
		735	0x903c, 0xffffffff, 0x00000100,
		736	0xad80, 0xffffffff, 0x00000100,
		737	0xac54, 0xffffffff, 0x00000100,
		738	0x897c, 0xffffffff, 0x06000100,
		739	0x9868, 0xffffffff, 0x00000100,
		740	0x9510, 0xffffffff, 0x00000100,
		741	0xaf04, 0xffffffff, 0x00000100,
		742	0xae04, 0xffffffff, 0x00000100,
		743	0x949c, 0xffffffff, 0x00000100,
		744	0x802c, 0xffffffff, 0xe0000000,
		745	0x9160, 0xffffffff, 0x00010000,
		746	0x9164, 0xffffffff, 0x00030002,
		747	0x9168, 0xffffffff, 0x00040007,
		748	0x916c, 0xffffffff, 0x00060005,
		749	0x9170, 0xffffffff, 0x00090008,
		750	0x9174, 0xffffffff, 0x00020001,
		751	0x9178, 0xffffffff, 0x00040003,
		752	0x917c, 0xffffffff, 0x00000007,
		753	0x9180, 0xffffffff, 0x00060005,
		754	0x9184, 0xffffffff, 0x00090008,
		755	0x9188, 0xffffffff, 0x00030002,
		756	0x918c, 0xffffffff, 0x00050004,
		757	0x9190, 0xffffffff, 0x00000008,
		758	0x9194, 0xffffffff, 0x00070006,
		759	0x9198, 0xffffffff, 0x000a0009,
		760	0x919c, 0xffffffff, 0x00040003,
		761	0x91a0, 0xffffffff, 0x00060005,
		762	0x91a4, 0xffffffff, 0x00000009,
		763	0x91a8, 0xffffffff, 0x00080007,
		764	0x91ac, 0xffffffff, 0x000b000a,
		765	0x91b0, 0xffffffff, 0x00050004,
		766	0x91b4, 0xffffffff, 0x00070006,
		767	0x91b8, 0xffffffff, 0x0008000b,
		768	0x91bc, 0xffffffff, 0x000a0009,
		769	0x91c0, 0xffffffff, 0x000d000c,
		770	0x9200, 0xffffffff, 0x00090008,
		771	0x9204, 0xffffffff, 0x000b000a,
		772	0x9208, 0xffffffff, 0x000c000f,
		773	0x920c, 0xffffffff, 0x000e000d,
		774	0x9210, 0xffffffff, 0x00110010,
		775	0x9214, 0xffffffff, 0x000a0009,
		776	0x9218, 0xffffffff, 0x000c000b,
		777	0x921c, 0xffffffff, 0x0000000f,
		778	0x9220, 0xffffffff, 0x000e000d,
		779	0x9224, 0xffffffff, 0x00110010,
		780	0x9228, 0xffffffff, 0x000b000a,
		781	0x922c, 0xffffffff, 0x000d000c,
		782	0x9230, 0xffffffff, 0x00000010,
		783	0x9234, 0xffffffff, 0x000f000e,
		784	0x9238, 0xffffffff, 0x00120011,
		785	0x923c, 0xffffffff, 0x000c000b,
		786	0x9240, 0xffffffff, 0x000e000d,
		787	0x9244, 0xffffffff, 0x00000011,
		788	0x9248, 0xffffffff, 0x0010000f,
		789	0x924c, 0xffffffff, 0x00130012,
		790	0x9250, 0xffffffff, 0x000d000c,
		791	0x9254, 0xffffffff, 0x000f000e,
		792	0x9258, 0xffffffff, 0x00100013,
		793	0x925c, 0xffffffff, 0x00120011,
		794	0x9260, 0xffffffff, 0x00150014,
		795	0x9150, 0xffffffff, 0x96940200,
		796	0x8708, 0xffffffff, 0x00900100,
		797	0xc478, 0xffffffff, 0x00000080,
		798	0xc404, 0xffffffff, 0x0020003f,
		799	0x30, 0xffffffff, 0x0000001c,
		800	0x34, 0x000f0000, 0x000f0000,
		801	0x160c, 0xffffffff, 0x00000100,
		802	0x1024, 0xffffffff, 0x00000100,
		803	0x102c, 0x00000101, 0x00000000,
		804	0x20a8, 0xffffffff, 0x00000104,
		805	0x55e4, 0xff000fff, 0x00000100,
		806	0x55e8, 0x00000001, 0x00000001,
		807	0x2f50, 0x00000001, 0x00000001,
		808	0x30cc, 0xc0000fff, 0x00000104,
		809	0xc1e4, 0x00000001, 0x00000001,
		810	0xd0c0, 0xfffffff0, 0x00000100,
		811	0xd8c0, 0xfffffff0, 0x00000100
		812	};
		813
		814	static const u32 verde_mgcg_cgcg_init[] =
		815	{
		816	0xc400, 0xffffffff, 0xfffffffc,
		817	0x802c, 0xffffffff, 0xe0000000,
		818	0x9a60, 0xffffffff, 0x00000100,
		819	0x92a4, 0xffffffff, 0x00000100,
		820	0xc164, 0xffffffff, 0x00000100,
		821	0x9774, 0xffffffff, 0x00000100,
		822	0x8984, 0xffffffff, 0x06000100,
		823	0x8a18, 0xffffffff, 0x00000100,
		824	0x92a0, 0xffffffff, 0x00000100,
		825	0xc380, 0xffffffff, 0x00000100,
		826	0x8b28, 0xffffffff, 0x00000100,
		827	0x9144, 0xffffffff, 0x00000100,
		828	0x8d88, 0xffffffff, 0x00000100,
		829	0x8d8c, 0xffffffff, 0x00000100,
		830	0x9030, 0xffffffff, 0x00000100,
		831	0x9034, 0xffffffff, 0x00000100,
		832	0x9038, 0xffffffff, 0x00000100,
		833	0x903c, 0xffffffff, 0x00000100,
		834	0xad80, 0xffffffff, 0x00000100,
		835	0xac54, 0xffffffff, 0x00000100,
		836	0x897c, 0xffffffff, 0x06000100,
		837	0x9868, 0xffffffff, 0x00000100,
		838	0x9510, 0xffffffff, 0x00000100,
		839	0xaf04, 0xffffffff, 0x00000100,
		840	0xae04, 0xffffffff, 0x00000100,
		841	0x949c, 0xffffffff, 0x00000100,
		842	0x802c, 0xffffffff, 0xe0000000,
		843	0x9160, 0xffffffff, 0x00010000,
		844	0x9164, 0xffffffff, 0x00030002,
		845	0x9168, 0xffffffff, 0x00040007,
		846	0x916c, 0xffffffff, 0x00060005,
		847	0x9170, 0xffffffff, 0x00090008,
		848	0x9174, 0xffffffff, 0x00020001,
		849	0x9178, 0xffffffff, 0x00040003,
		850	0x917c, 0xffffffff, 0x00000007,
		851	0x9180, 0xffffffff, 0x00060005,
		852	0x9184, 0xffffffff, 0x00090008,
		853	0x9188, 0xffffffff, 0x00030002,
		854	0x918c, 0xffffffff, 0x00050004,
		855	0x9190, 0xffffffff, 0x00000008,
		856	0x9194, 0xffffffff, 0x00070006,
		857	0x9198, 0xffffffff, 0x000a0009,
		858	0x919c, 0xffffffff, 0x00040003,
		859	0x91a0, 0xffffffff, 0x00060005,
		860	0x91a4, 0xffffffff, 0x00000009,
		861	0x91a8, 0xffffffff, 0x00080007,
		862	0x91ac, 0xffffffff, 0x000b000a,
		863	0x91b0, 0xffffffff, 0x00050004,
		864	0x91b4, 0xffffffff, 0x00070006,
		865	0x91b8, 0xffffffff, 0x0008000b,
		866	0x91bc, 0xffffffff, 0x000a0009,
		867	0x91c0, 0xffffffff, 0x000d000c,
		868	0x9200, 0xffffffff, 0x00090008,
		869	0x9204, 0xffffffff, 0x000b000a,
		870	0x9208, 0xffffffff, 0x000c000f,
		871	0x920c, 0xffffffff, 0x000e000d,
		872	0x9210, 0xffffffff, 0x00110010,
		873	0x9214, 0xffffffff, 0x000a0009,
		874	0x9218, 0xffffffff, 0x000c000b,
		875	0x921c, 0xffffffff, 0x0000000f,
		876	0x9220, 0xffffffff, 0x000e000d,
		877	0x9224, 0xffffffff, 0x00110010,
		878	0x9228, 0xffffffff, 0x000b000a,
		879	0x922c, 0xffffffff, 0x000d000c,
		880	0x9230, 0xffffffff, 0x00000010,
		881	0x9234, 0xffffffff, 0x000f000e,
		882	0x9238, 0xffffffff, 0x00120011,
		883	0x923c, 0xffffffff, 0x000c000b,
		884	0x9240, 0xffffffff, 0x000e000d,
		885	0x9244, 0xffffffff, 0x00000011,
		886	0x9248, 0xffffffff, 0x0010000f,
		887	0x924c, 0xffffffff, 0x00130012,
		888	0x9250, 0xffffffff, 0x000d000c,
		889	0x9254, 0xffffffff, 0x000f000e,
		890	0x9258, 0xffffffff, 0x00100013,
		891	0x925c, 0xffffffff, 0x00120011,
		892	0x9260, 0xffffffff, 0x00150014,
		893	0x9150, 0xffffffff, 0x96940200,
		894	0x8708, 0xffffffff, 0x00900100,
		895	0xc478, 0xffffffff, 0x00000080,
		896	0xc404, 0xffffffff, 0x0020003f,
		897	0x30, 0xffffffff, 0x0000001c,
		898	0x34, 0x000f0000, 0x000f0000,
		899	0x160c, 0xffffffff, 0x00000100,
		900	0x1024, 0xffffffff, 0x00000100,
		901	0x102c, 0x00000101, 0x00000000,
		902	0x20a8, 0xffffffff, 0x00000104,
		903	0x264c, 0x000c0000, 0x000c0000,
		904	0x2648, 0x000c0000, 0x000c0000,
		905	0x55e4, 0xff000fff, 0x00000100,
		906	0x55e8, 0x00000001, 0x00000001,
		907	0x2f50, 0x00000001, 0x00000001,
		908	0x30cc, 0xc0000fff, 0x00000104,
		909	0xc1e4, 0x00000001, 0x00000001,
		910	0xd0c0, 0xfffffff0, 0x00000100,
		911	0xd8c0, 0xfffffff0, 0x00000100
		912	};
		913
		914	static const u32 oland_mgcg_cgcg_init[] =
		915	{
		916	0xc400, 0xffffffff, 0xfffffffc,
		917	0x802c, 0xffffffff, 0xe0000000,
		918	0x9a60, 0xffffffff, 0x00000100,
		919	0x92a4, 0xffffffff, 0x00000100,
		920	0xc164, 0xffffffff, 0x00000100,
		921	0x9774, 0xffffffff, 0x00000100,
		922	0x8984, 0xffffffff, 0x06000100,
		923	0x8a18, 0xffffffff, 0x00000100,
		924	0x92a0, 0xffffffff, 0x00000100,
		925	0xc380, 0xffffffff, 0x00000100,
		926	0x8b28, 0xffffffff, 0x00000100,
		927	0x9144, 0xffffffff, 0x00000100,
		928	0x8d88, 0xffffffff, 0x00000100,
		929	0x8d8c, 0xffffffff, 0x00000100,
		930	0x9030, 0xffffffff, 0x00000100,
		931	0x9034, 0xffffffff, 0x00000100,
		932	0x9038, 0xffffffff, 0x00000100,
		933	0x903c, 0xffffffff, 0x00000100,
		934	0xad80, 0xffffffff, 0x00000100,
		935	0xac54, 0xffffffff, 0x00000100,
		936	0x897c, 0xffffffff, 0x06000100,
		937	0x9868, 0xffffffff, 0x00000100,
		938	0x9510, 0xffffffff, 0x00000100,
		939	0xaf04, 0xffffffff, 0x00000100,
		940	0xae04, 0xffffffff, 0x00000100,
		941	0x949c, 0xffffffff, 0x00000100,
		942	0x802c, 0xffffffff, 0xe0000000,
		943	0x9160, 0xffffffff, 0x00010000,
		944	0x9164, 0xffffffff, 0x00030002,
		945	0x9168, 0xffffffff, 0x00040007,
		946	0x916c, 0xffffffff, 0x00060005,
		947	0x9170, 0xffffffff, 0x00090008,
		948	0x9174, 0xffffffff, 0x00020001,
		949	0x9178, 0xffffffff, 0x00040003,
		950	0x917c, 0xffffffff, 0x00000007,
		951	0x9180, 0xffffffff, 0x00060005,
		952	0x9184, 0xffffffff, 0x00090008,
		953	0x9188, 0xffffffff, 0x00030002,
		954	0x918c, 0xffffffff, 0x00050004,
		955	0x9190, 0xffffffff, 0x00000008,
		956	0x9194, 0xffffffff, 0x00070006,
		957	0x9198, 0xffffffff, 0x000a0009,
		958	0x919c, 0xffffffff, 0x00040003,
		959	0x91a0, 0xffffffff, 0x00060005,
		960	0x91a4, 0xffffffff, 0x00000009,
		961	0x91a8, 0xffffffff, 0x00080007,
		962	0x91ac, 0xffffffff, 0x000b000a,
		963	0x91b0, 0xffffffff, 0x00050004,
		964	0x91b4, 0xffffffff, 0x00070006,
		965	0x91b8, 0xffffffff, 0x0008000b,
		966	0x91bc, 0xffffffff, 0x000a0009,
		967	0x91c0, 0xffffffff, 0x000d000c,
		968	0x91c4, 0xffffffff, 0x00060005,
		969	0x91c8, 0xffffffff, 0x00080007,
		970	0x91cc, 0xffffffff, 0x0000000b,
		971	0x91d0, 0xffffffff, 0x000a0009,
		972	0x91d4, 0xffffffff, 0x000d000c,
		973	0x9150, 0xffffffff, 0x96940200,
		974	0x8708, 0xffffffff, 0x00900100,
		975	0xc478, 0xffffffff, 0x00000080,
		976	0xc404, 0xffffffff, 0x0020003f,
		977	0x30, 0xffffffff, 0x0000001c,
		978	0x34, 0x000f0000, 0x000f0000,
		979	0x160c, 0xffffffff, 0x00000100,
		980	0x1024, 0xffffffff, 0x00000100,
		981	0x102c, 0x00000101, 0x00000000,
		982	0x20a8, 0xffffffff, 0x00000104,
		983	0x264c, 0x000c0000, 0x000c0000,
		984	0x2648, 0x000c0000, 0x000c0000,
		985	0x55e4, 0xff000fff, 0x00000100,
		986	0x55e8, 0x00000001, 0x00000001,
		987	0x2f50, 0x00000001, 0x00000001,
		988	0x30cc, 0xc0000fff, 0x00000104,
		989	0xc1e4, 0x00000001, 0x00000001,
		990	0xd0c0, 0xfffffff0, 0x00000100,
		991	0xd8c0, 0xfffffff0, 0x00000100
		992	};
		993
		994	static const u32 hainan_mgcg_cgcg_init[] =
		995	{
		996	0xc400, 0xffffffff, 0xfffffffc,
		997	0x802c, 0xffffffff, 0xe0000000,
		998	0x9a60, 0xffffffff, 0x00000100,
		999	0x92a4, 0xffffffff, 0x00000100,
		1000	0xc164, 0xffffffff, 0x00000100,
		1001	0x9774, 0xffffffff, 0x00000100,
		1002	0x8984, 0xffffffff, 0x06000100,
		1003	0x8a18, 0xffffffff, 0x00000100,
		1004	0x92a0, 0xffffffff, 0x00000100,
		1005	0xc380, 0xffffffff, 0x00000100,
		1006	0x8b28, 0xffffffff, 0x00000100,
		1007	0x9144, 0xffffffff, 0x00000100,
		1008	0x8d88, 0xffffffff, 0x00000100,
		1009	0x8d8c, 0xffffffff, 0x00000100,
		1010	0x9030, 0xffffffff, 0x00000100,
		1011	0x9034, 0xffffffff, 0x00000100,
		1012	0x9038, 0xffffffff, 0x00000100,
		1013	0x903c, 0xffffffff, 0x00000100,
		1014	0xad80, 0xffffffff, 0x00000100,
		1015	0xac54, 0xffffffff, 0x00000100,
		1016	0x897c, 0xffffffff, 0x06000100,
		1017	0x9868, 0xffffffff, 0x00000100,
		1018	0x9510, 0xffffffff, 0x00000100,
		1019	0xaf04, 0xffffffff, 0x00000100,
		1020	0xae04, 0xffffffff, 0x00000100,
		1021	0x949c, 0xffffffff, 0x00000100,
		1022	0x802c, 0xffffffff, 0xe0000000,
		1023	0x9160, 0xffffffff, 0x00010000,
		1024	0x9164, 0xffffffff, 0x00030002,
		1025	0x9168, 0xffffffff, 0x00040007,
		1026	0x916c, 0xffffffff, 0x00060005,
		1027	0x9170, 0xffffffff, 0x00090008,
		1028	0x9174, 0xffffffff, 0x00020001,
		1029	0x9178, 0xffffffff, 0x00040003,
		1030	0x917c, 0xffffffff, 0x00000007,
		1031	0x9180, 0xffffffff, 0x00060005,
		1032	0x9184, 0xffffffff, 0x00090008,
		1033	0x9188, 0xffffffff, 0x00030002,
		1034	0x918c, 0xffffffff, 0x00050004,
		1035	0x9190, 0xffffffff, 0x00000008,
		1036	0x9194, 0xffffffff, 0x00070006,
		1037	0x9198, 0xffffffff, 0x000a0009,
		1038	0x919c, 0xffffffff, 0x00040003,
		1039	0x91a0, 0xffffffff, 0x00060005,
		1040	0x91a4, 0xffffffff, 0x00000009,
		1041	0x91a8, 0xffffffff, 0x00080007,
		1042	0x91ac, 0xffffffff, 0x000b000a,
		1043	0x91b0, 0xffffffff, 0x00050004,
		1044	0x91b4, 0xffffffff, 0x00070006,
		1045	0x91b8, 0xffffffff, 0x0008000b,
		1046	0x91bc, 0xffffffff, 0x000a0009,
		1047	0x91c0, 0xffffffff, 0x000d000c,
		1048	0x91c4, 0xffffffff, 0x00060005,
		1049	0x91c8, 0xffffffff, 0x00080007,
		1050	0x91cc, 0xffffffff, 0x0000000b,
		1051	0x91d0, 0xffffffff, 0x000a0009,
		1052	0x91d4, 0xffffffff, 0x000d000c,
		1053	0x9150, 0xffffffff, 0x96940200,
		1054	0x8708, 0xffffffff, 0x00900100,
		1055	0xc478, 0xffffffff, 0x00000080,
		1056	0xc404, 0xffffffff, 0x0020003f,
		1057	0x30, 0xffffffff, 0x0000001c,
		1058	0x34, 0x000f0000, 0x000f0000,
		1059	0x160c, 0xffffffff, 0x00000100,
		1060	0x1024, 0xffffffff, 0x00000100,
		1061	0x20a8, 0xffffffff, 0x00000104,
		1062	0x264c, 0x000c0000, 0x000c0000,
		1063	0x2648, 0x000c0000, 0x000c0000,
		1064	0x2f50, 0x00000001, 0x00000001,
		1065	0x30cc, 0xc0000fff, 0x00000104,
		1066	0xc1e4, 0x00000001, 0x00000001,
		1067	0xd0c0, 0xfffffff0, 0x00000100,
		1068	0xd8c0, 0xfffffff0, 0x00000100
		1069	};
		1070
		1071	static u32 verde_pg_init[] =
		1072	{
		1073	0x353c, 0xffffffff, 0x40000,
		1074	0x3538, 0xffffffff, 0x200010ff,
		1075	0x353c, 0xffffffff, 0x0,
		1076	0x353c, 0xffffffff, 0x0,
		1077	0x353c, 0xffffffff, 0x0,
		1078	0x353c, 0xffffffff, 0x0,
		1079	0x353c, 0xffffffff, 0x0,
		1080	0x353c, 0xffffffff, 0x7007,
		1081	0x3538, 0xffffffff, 0x300010ff,
		1082	0x353c, 0xffffffff, 0x0,
		1083	0x353c, 0xffffffff, 0x0,
		1084	0x353c, 0xffffffff, 0x0,
		1085	0x353c, 0xffffffff, 0x0,
		1086	0x353c, 0xffffffff, 0x0,
		1087	0x353c, 0xffffffff, 0x400000,
		1088	0x3538, 0xffffffff, 0x100010ff,
		1089	0x353c, 0xffffffff, 0x0,
		1090	0x353c, 0xffffffff, 0x0,
		1091	0x353c, 0xffffffff, 0x0,
		1092	0x353c, 0xffffffff, 0x0,
		1093	0x353c, 0xffffffff, 0x0,
		1094	0x353c, 0xffffffff, 0x120200,
		1095	0x3538, 0xffffffff, 0x500010ff,
		1096	0x353c, 0xffffffff, 0x0,
		1097	0x353c, 0xffffffff, 0x0,
		1098	0x353c, 0xffffffff, 0x0,
		1099	0x353c, 0xffffffff, 0x0,
		1100	0x353c, 0xffffffff, 0x0,
		1101	0x353c, 0xffffffff, 0x1e1e16,
		1102	0x3538, 0xffffffff, 0x600010ff,
		1103	0x353c, 0xffffffff, 0x0,
		1104	0x353c, 0xffffffff, 0x0,
		1105	0x353c, 0xffffffff, 0x0,
		1106	0x353c, 0xffffffff, 0x0,
		1107	0x353c, 0xffffffff, 0x0,
		1108	0x353c, 0xffffffff, 0x171f1e,
		1109	0x3538, 0xffffffff, 0x700010ff,
		1110	0x353c, 0xffffffff, 0x0,
		1111	0x353c, 0xffffffff, 0x0,
		1112	0x353c, 0xffffffff, 0x0,
		1113	0x353c, 0xffffffff, 0x0,
		1114	0x353c, 0xffffffff, 0x0,
		1115	0x353c, 0xffffffff, 0x0,
		1116	0x3538, 0xffffffff, 0x9ff,
		1117	0x3500, 0xffffffff, 0x0,
		1118	0x3504, 0xffffffff, 0x10000800,
		1119	0x3504, 0xffffffff, 0xf,
		1120	0x3504, 0xffffffff, 0xf,
		1121	0x3500, 0xffffffff, 0x4,
		1122	0x3504, 0xffffffff, 0x1000051e,
		1123	0x3504, 0xffffffff, 0xffff,
		1124	0x3504, 0xffffffff, 0xffff,
		1125	0x3500, 0xffffffff, 0x8,
		1126	0x3504, 0xffffffff, 0x80500,
		1127	0x3500, 0xffffffff, 0x12,
		1128	0x3504, 0xffffffff, 0x9050c,
		1129	0x3500, 0xffffffff, 0x1d,
		1130	0x3504, 0xffffffff, 0xb052c,
		1131	0x3500, 0xffffffff, 0x2a,
		1132	0x3504, 0xffffffff, 0x1053e,
		1133	0x3500, 0xffffffff, 0x2d,
		1134	0x3504, 0xffffffff, 0x10546,
		1135	0x3500, 0xffffffff, 0x30,
		1136	0x3504, 0xffffffff, 0xa054e,
		1137	0x3500, 0xffffffff, 0x3c,
		1138	0x3504, 0xffffffff, 0x1055f,
		1139	0x3500, 0xffffffff, 0x3f,
		1140	0x3504, 0xffffffff, 0x10567,
		1141	0x3500, 0xffffffff, 0x42,
		1142	0x3504, 0xffffffff, 0x1056f,
		1143	0x3500, 0xffffffff, 0x45,
		1144	0x3504, 0xffffffff, 0x10572,
		1145	0x3500, 0xffffffff, 0x48,
		1146	0x3504, 0xffffffff, 0x20575,
		1147	0x3500, 0xffffffff, 0x4c,
		1148	0x3504, 0xffffffff, 0x190801,
		1149	0x3500, 0xffffffff, 0x67,
		1150	0x3504, 0xffffffff, 0x1082a,
		1151	0x3500, 0xffffffff, 0x6a,
		1152	0x3504, 0xffffffff, 0x1b082d,
		1153	0x3500, 0xffffffff, 0x87,
		1154	0x3504, 0xffffffff, 0x310851,
		1155	0x3500, 0xffffffff, 0xba,
		1156	0x3504, 0xffffffff, 0x891,
		1157	0x3500, 0xffffffff, 0xbc,
		1158	0x3504, 0xffffffff, 0x893,
		1159	0x3500, 0xffffffff, 0xbe,
		1160	0x3504, 0xffffffff, 0x20895,
		1161	0x3500, 0xffffffff, 0xc2,
		1162	0x3504, 0xffffffff, 0x20899,
		1163	0x3500, 0xffffffff, 0xc6,
		1164	0x3504, 0xffffffff, 0x2089d,
		1165	0x3500, 0xffffffff, 0xca,
		1166	0x3504, 0xffffffff, 0x8a1,
		1167	0x3500, 0xffffffff, 0xcc,
		1168	0x3504, 0xffffffff, 0x8a3,
		1169	0x3500, 0xffffffff, 0xce,
		1170	0x3504, 0xffffffff, 0x308a5,
		1171	0x3500, 0xffffffff, 0xd3,
		1172	0x3504, 0xffffffff, 0x6d08cd,
		1173	0x3500, 0xffffffff, 0x142,
		1174	0x3504, 0xffffffff, 0x2000095a,
		1175	0x3504, 0xffffffff, 0x1,
		1176	0x3500, 0xffffffff, 0x144,
		1177	0x3504, 0xffffffff, 0x301f095b,
		1178	0x3500, 0xffffffff, 0x165,
		1179	0x3504, 0xffffffff, 0xc094d,
		1180	0x3500, 0xffffffff, 0x173,
		1181	0x3504, 0xffffffff, 0xf096d,
		1182	0x3500, 0xffffffff, 0x184,
		1183	0x3504, 0xffffffff, 0x15097f,
		1184	0x3500, 0xffffffff, 0x19b,
		1185	0x3504, 0xffffffff, 0xc0998,
		1186	0x3500, 0xffffffff, 0x1a9,
		1187	0x3504, 0xffffffff, 0x409a7,
		1188	0x3500, 0xffffffff, 0x1af,
		1189	0x3504, 0xffffffff, 0xcdc,
		1190	0x3500, 0xffffffff, 0x1b1,
		1191	0x3504, 0xffffffff, 0x800,
		1192	0x3508, 0xffffffff, 0x6c9b2000,
		1193	0x3510, 0xfc00, 0x2000,
		1194	0x3544, 0xffffffff, 0xfc0,
		1195	0x28d4, 0x00000100, 0x100
		1196	};
		1197
		1198	static void si_init_golden_registers(struct radeon_device *rdev)
		1199	{
		1200	switch (rdev->family) {
		1201	case CHIP_TAHITI:
		1202	radeon_program_register_sequence(rdev,
		1203	tahiti_golden_registers,
		1204	(const u32)ARRAY_SIZE(tahiti_golden_registers));
		1205	radeon_program_register_sequence(rdev,
		1206	tahiti_golden_rlc_registers,
		1207	(const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
		1208	radeon_program_register_sequence(rdev,
		1209	tahiti_mgcg_cgcg_init,
		1210	(const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
		1211	radeon_program_register_sequence(rdev,
		1212	tahiti_golden_registers2,
		1213	(const u32)ARRAY_SIZE(tahiti_golden_registers2));
		1214	break;
		1215	case CHIP_PITCAIRN:
		1216	radeon_program_register_sequence(rdev,
		1217	pitcairn_golden_registers,
		1218	(const u32)ARRAY_SIZE(pitcairn_golden_registers));
		1219	radeon_program_register_sequence(rdev,
		1220	pitcairn_golden_rlc_registers,
		1221	(const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
		1222	radeon_program_register_sequence(rdev,
		1223	pitcairn_mgcg_cgcg_init,
		1224	(const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
		1225	break;
		1226	case CHIP_VERDE:
		1227	radeon_program_register_sequence(rdev,
		1228	verde_golden_registers,
		1229	(const u32)ARRAY_SIZE(verde_golden_registers));
		1230	radeon_program_register_sequence(rdev,
		1231	verde_golden_rlc_registers,
		1232	(const u32)ARRAY_SIZE(verde_golden_rlc_registers));
		1233	radeon_program_register_sequence(rdev,
		1234	verde_mgcg_cgcg_init,
		1235	(const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
		1236	radeon_program_register_sequence(rdev,
		1237	verde_pg_init,
		1238	(const u32)ARRAY_SIZE(verde_pg_init));
		1239	break;
		1240	case CHIP_OLAND:
		1241	radeon_program_register_sequence(rdev,
		1242	oland_golden_registers,
		1243	(const u32)ARRAY_SIZE(oland_golden_registers));
		1244	radeon_program_register_sequence(rdev,
		1245	oland_golden_rlc_registers,
		1246	(const u32)ARRAY_SIZE(oland_golden_rlc_registers));
		1247	radeon_program_register_sequence(rdev,
		1248	oland_mgcg_cgcg_init,
		1249	(const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
		1250	break;
		1251	case CHIP_HAINAN:
		1252	radeon_program_register_sequence(rdev,
		1253	hainan_golden_registers,
		1254	(const u32)ARRAY_SIZE(hainan_golden_registers));
		1255	radeon_program_register_sequence(rdev,
		1256	hainan_golden_registers2,
		1257	(const u32)ARRAY_SIZE(hainan_golden_registers2));
		1258	radeon_program_register_sequence(rdev,
		1259	hainan_mgcg_cgcg_init,
		1260	(const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
		1261	break;
		1262	default:
		1263	break;
		1264	}
		1265	}
		1266
6104	serge	1267	/**
		1268	* si_get_allowed_info_register - fetch the register for the info ioctl
		1269	*
		1270	* @rdev: radeon_device pointer
		1271	* @reg: register offset in bytes
		1272	* @val: register value
		1273	*
		1274	* Returns 0 for success or -EINVAL for an invalid register
		1275	*
		1276	*/
		1277	int si_get_allowed_info_register(struct radeon_device *rdev,
		1278	u32 reg, u32 *val)
		1279	{
		1280	switch (reg) {
		1281	case GRBM_STATUS:
		1282	case GRBM_STATUS2:
		1283	case GRBM_STATUS_SE0:
		1284	case GRBM_STATUS_SE1:
		1285	case SRBM_STATUS:
		1286	case SRBM_STATUS2:
		1287	case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
		1288	case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
		1289	case UVD_STATUS:
		1290	*val = RREG32(reg);
		1291	return 0;
		1292	default:
		1293	return -EINVAL;
		1294	}
		1295	}
		1296
3764	Serge	1297	#define PCIE_BUS_CLK 10000
		1298	#define TCLK (PCIE_BUS_CLK / 10)
		1299
		1300	/**
		1301	* si_get_xclk - get the xclk
		1302	*
		1303	* @rdev: radeon_device pointer
		1304	*
		1305	* Returns the reference clock used by the gfx engine
		1306	* (SI).
		1307	*/
		1308	u32 si_get_xclk(struct radeon_device *rdev)
		1309	{
7146	serge	1310	u32 reference_clock = rdev->clock.spll.reference_freq;
3764	Serge	1311	u32 tmp;
		1312
		1313	tmp = RREG32(CG_CLKPIN_CNTL_2);
		1314	if (tmp & MUX_TCLK_TO_XCLK)
		1315	return TCLK;
		1316
		1317	tmp = RREG32(CG_CLKPIN_CNTL);
		1318	if (tmp & XTALIN_DIVIDE)
		1319	return reference_clock / 4;
		1320
		1321	return reference_clock;
		1322	}
		1323
2997	Serge	1324	/* get temperature in millidegrees */
		1325	int si_get_temp(struct radeon_device *rdev)
		1326	{
		1327	u32 temp;
		1328	int actual_temp = 0;
		1329
		1330	temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
		1331	CTF_TEMP_SHIFT;
		1332
		1333	if (temp & 0x200)
		1334	actual_temp = 255;
		1335	else
		1336	actual_temp = temp & 0x1ff;
		1337
		1338	actual_temp = (actual_temp * 1000);
		1339
		1340	return actual_temp;
		1341	}
		1342
		1343	#define TAHITI_IO_MC_REGS_SIZE 36
		1344
		1345	static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1346	{0x0000006f, 0x03044000},
		1347	{0x00000070, 0x0480c018},
		1348	{0x00000071, 0x00000040},
		1349	{0x00000072, 0x01000000},
		1350	{0x00000074, 0x000000ff},
		1351	{0x00000075, 0x00143400},
		1352	{0x00000076, 0x08ec0800},
		1353	{0x00000077, 0x040000cc},
		1354	{0x00000079, 0x00000000},
		1355	{0x0000007a, 0x21000409},
		1356	{0x0000007c, 0x00000000},
		1357	{0x0000007d, 0xe8000000},
		1358	{0x0000007e, 0x044408a8},
		1359	{0x0000007f, 0x00000003},
		1360	{0x00000080, 0x00000000},
		1361	{0x00000081, 0x01000000},
		1362	{0x00000082, 0x02000000},
		1363	{0x00000083, 0x00000000},
		1364	{0x00000084, 0xe3f3e4f4},
		1365	{0x00000085, 0x00052024},
		1366	{0x00000087, 0x00000000},
		1367	{0x00000088, 0x66036603},
		1368	{0x00000089, 0x01000000},
		1369	{0x0000008b, 0x1c0a0000},
		1370	{0x0000008c, 0xff010000},
		1371	{0x0000008e, 0xffffefff},
		1372	{0x0000008f, 0xfff3efff},
		1373	{0x00000090, 0xfff3efbf},
		1374	{0x00000094, 0x00101101},
		1375	{0x00000095, 0x00000fff},
		1376	{0x00000096, 0x00116fff},
		1377	{0x00000097, 0x60010000},
		1378	{0x00000098, 0x10010000},
		1379	{0x00000099, 0x00006000},
		1380	{0x0000009a, 0x00001000},
		1381	{0x0000009f, 0x00a77400}
		1382	};
		1383
		1384	static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1385	{0x0000006f, 0x03044000},
		1386	{0x00000070, 0x0480c018},
		1387	{0x00000071, 0x00000040},
		1388	{0x00000072, 0x01000000},
		1389	{0x00000074, 0x000000ff},
		1390	{0x00000075, 0x00143400},
		1391	{0x00000076, 0x08ec0800},
		1392	{0x00000077, 0x040000cc},
		1393	{0x00000079, 0x00000000},
		1394	{0x0000007a, 0x21000409},
		1395	{0x0000007c, 0x00000000},
		1396	{0x0000007d, 0xe8000000},
		1397	{0x0000007e, 0x044408a8},
		1398	{0x0000007f, 0x00000003},
		1399	{0x00000080, 0x00000000},
		1400	{0x00000081, 0x01000000},
		1401	{0x00000082, 0x02000000},
		1402	{0x00000083, 0x00000000},
		1403	{0x00000084, 0xe3f3e4f4},
		1404	{0x00000085, 0x00052024},
		1405	{0x00000087, 0x00000000},
		1406	{0x00000088, 0x66036603},
		1407	{0x00000089, 0x01000000},
		1408	{0x0000008b, 0x1c0a0000},
		1409	{0x0000008c, 0xff010000},
		1410	{0x0000008e, 0xffffefff},
		1411	{0x0000008f, 0xfff3efff},
		1412	{0x00000090, 0xfff3efbf},
		1413	{0x00000094, 0x00101101},
		1414	{0x00000095, 0x00000fff},
		1415	{0x00000096, 0x00116fff},
		1416	{0x00000097, 0x60010000},
		1417	{0x00000098, 0x10010000},
		1418	{0x00000099, 0x00006000},
		1419	{0x0000009a, 0x00001000},
		1420	{0x0000009f, 0x00a47400}
		1421	};
		1422
		1423	static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1424	{0x0000006f, 0x03044000},
		1425	{0x00000070, 0x0480c018},
		1426	{0x00000071, 0x00000040},
		1427	{0x00000072, 0x01000000},
		1428	{0x00000074, 0x000000ff},
		1429	{0x00000075, 0x00143400},
		1430	{0x00000076, 0x08ec0800},
		1431	{0x00000077, 0x040000cc},
		1432	{0x00000079, 0x00000000},
		1433	{0x0000007a, 0x21000409},
		1434	{0x0000007c, 0x00000000},
		1435	{0x0000007d, 0xe8000000},
		1436	{0x0000007e, 0x044408a8},
		1437	{0x0000007f, 0x00000003},
		1438	{0x00000080, 0x00000000},
		1439	{0x00000081, 0x01000000},
		1440	{0x00000082, 0x02000000},
		1441	{0x00000083, 0x00000000},
		1442	{0x00000084, 0xe3f3e4f4},
		1443	{0x00000085, 0x00052024},
		1444	{0x00000087, 0x00000000},
		1445	{0x00000088, 0x66036603},
		1446	{0x00000089, 0x01000000},
		1447	{0x0000008b, 0x1c0a0000},
		1448	{0x0000008c, 0xff010000},
		1449	{0x0000008e, 0xffffefff},
		1450	{0x0000008f, 0xfff3efff},
		1451	{0x00000090, 0xfff3efbf},
		1452	{0x00000094, 0x00101101},
		1453	{0x00000095, 0x00000fff},
		1454	{0x00000096, 0x00116fff},
		1455	{0x00000097, 0x60010000},
		1456	{0x00000098, 0x10010000},
		1457	{0x00000099, 0x00006000},
		1458	{0x0000009a, 0x00001000},
		1459	{0x0000009f, 0x00a37400}
		1460	};
		1461
3764	Serge	1462	static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1463	{0x0000006f, 0x03044000},
		1464	{0x00000070, 0x0480c018},
		1465	{0x00000071, 0x00000040},
		1466	{0x00000072, 0x01000000},
		1467	{0x00000074, 0x000000ff},
		1468	{0x00000075, 0x00143400},
		1469	{0x00000076, 0x08ec0800},
		1470	{0x00000077, 0x040000cc},
		1471	{0x00000079, 0x00000000},
		1472	{0x0000007a, 0x21000409},
		1473	{0x0000007c, 0x00000000},
		1474	{0x0000007d, 0xe8000000},
		1475	{0x0000007e, 0x044408a8},
		1476	{0x0000007f, 0x00000003},
		1477	{0x00000080, 0x00000000},
		1478	{0x00000081, 0x01000000},
		1479	{0x00000082, 0x02000000},
		1480	{0x00000083, 0x00000000},
		1481	{0x00000084, 0xe3f3e4f4},
		1482	{0x00000085, 0x00052024},
		1483	{0x00000087, 0x00000000},
		1484	{0x00000088, 0x66036603},
		1485	{0x00000089, 0x01000000},
		1486	{0x0000008b, 0x1c0a0000},
		1487	{0x0000008c, 0xff010000},
		1488	{0x0000008e, 0xffffefff},
		1489	{0x0000008f, 0xfff3efff},
		1490	{0x00000090, 0xfff3efbf},
		1491	{0x00000094, 0x00101101},
		1492	{0x00000095, 0x00000fff},
		1493	{0x00000096, 0x00116fff},
		1494	{0x00000097, 0x60010000},
		1495	{0x00000098, 0x10010000},
		1496	{0x00000099, 0x00006000},
		1497	{0x0000009a, 0x00001000},
		1498	{0x0000009f, 0x00a17730}
		1499	};
		1500
		1501	static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1502	{0x0000006f, 0x03044000},
		1503	{0x00000070, 0x0480c018},
		1504	{0x00000071, 0x00000040},
		1505	{0x00000072, 0x01000000},
		1506	{0x00000074, 0x000000ff},
		1507	{0x00000075, 0x00143400},
		1508	{0x00000076, 0x08ec0800},
		1509	{0x00000077, 0x040000cc},
		1510	{0x00000079, 0x00000000},
		1511	{0x0000007a, 0x21000409},
		1512	{0x0000007c, 0x00000000},
		1513	{0x0000007d, 0xe8000000},
		1514	{0x0000007e, 0x044408a8},
		1515	{0x0000007f, 0x00000003},
		1516	{0x00000080, 0x00000000},
		1517	{0x00000081, 0x01000000},
		1518	{0x00000082, 0x02000000},
		1519	{0x00000083, 0x00000000},
		1520	{0x00000084, 0xe3f3e4f4},
		1521	{0x00000085, 0x00052024},
		1522	{0x00000087, 0x00000000},
		1523	{0x00000088, 0x66036603},
		1524	{0x00000089, 0x01000000},
		1525	{0x0000008b, 0x1c0a0000},
		1526	{0x0000008c, 0xff010000},
		1527	{0x0000008e, 0xffffefff},
		1528	{0x0000008f, 0xfff3efff},
		1529	{0x00000090, 0xfff3efbf},
		1530	{0x00000094, 0x00101101},
		1531	{0x00000095, 0x00000fff},
		1532	{0x00000096, 0x00116fff},
		1533	{0x00000097, 0x60010000},
		1534	{0x00000098, 0x10010000},
		1535	{0x00000099, 0x00006000},
		1536	{0x0000009a, 0x00001000},
		1537	{0x0000009f, 0x00a07730}
		1538	};
		1539
2997	Serge	1540	/* ucode loading */
5078	serge	1541	int si_mc_load_microcode(struct radeon_device *rdev)
2997	Serge	1542	{
5078	serge	1543	const __be32 *fw_data = NULL;
		1544	const __le32 *new_fw_data = NULL;
2997	Serge	1545	u32 running, blackout = 0;
5078	serge	1546	u32 *io_mc_regs = NULL;
		1547	const __le32 *new_io_mc_regs = NULL;
		1548	int i, regs_size, ucode_size;
2997	Serge	1549
		1550	if (!rdev->mc_fw)
		1551	return -EINVAL;
		1552
5078	serge	1553	if (rdev->new_fw) {
		1554	const struct mc_firmware_header_v1_0 *hdr =
		1555	(const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
		1556
		1557	radeon_ucode_print_mc_hdr(&hdr->header);
		1558	regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
		1559	new_io_mc_regs = (const __le32 *)
		1560	(rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
		1561	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
		1562	new_fw_data = (const __le32 *)
		1563	(rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
		1564	} else {
6104	serge	1565	ucode_size = rdev->mc_fw->size / 4;
5078	serge	1566
6104	serge	1567	switch (rdev->family) {
		1568	case CHIP_TAHITI:
		1569	io_mc_regs = (u32 *)&tahiti_io_mc_regs;
		1570	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1571	break;
		1572	case CHIP_PITCAIRN:
		1573	io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
		1574	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1575	break;
		1576	case CHIP_VERDE:
		1577	default:
		1578	io_mc_regs = (u32 *)&verde_io_mc_regs;
		1579	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1580	break;
		1581	case CHIP_OLAND:
		1582	io_mc_regs = (u32 *)&oland_io_mc_regs;
		1583	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1584	break;
		1585	case CHIP_HAINAN:
		1586	io_mc_regs = (u32 *)&hainan_io_mc_regs;
		1587	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1588	break;
		1589	}
5078	serge	1590	fw_data = (const __be32 *)rdev->mc_fw->data;
		1591	}
2997	Serge	1592
		1593	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
		1594
		1595	if (running == 0) {
		1596	if (running) {
		1597	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
		1598	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout \| 1);
		1599	}
		1600
		1601	/* reset the engine and set to writable */
		1602	WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
		1603	WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
		1604
		1605	/* load mc io regs */
		1606	for (i = 0; i < regs_size; i++) {
5078	serge	1607	if (rdev->new_fw) {
		1608	WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
		1609	WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
		1610	} else {
6104	serge	1611	WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
		1612	WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
		1613	}
2997	Serge	1614	}
		1615	/* load the MC ucode */
5078	serge	1616	for (i = 0; i < ucode_size; i++) {
		1617	if (rdev->new_fw)
		1618	WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
		1619	else
6104	serge	1620	WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
5078	serge	1621	}
2997	Serge	1622
		1623	/* put the engine back into the active state */
		1624	WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
		1625	WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
		1626	WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
		1627
		1628	/* wait for training to complete */
		1629	for (i = 0; i < rdev->usec_timeout; i++) {
		1630	if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
		1631	break;
		1632	udelay(1);
		1633	}
		1634	for (i = 0; i < rdev->usec_timeout; i++) {
		1635	if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
		1636	break;
		1637	udelay(1);
		1638	}
		1639
		1640	if (running)
		1641	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
		1642	}
		1643
		1644	return 0;
		1645	}
		1646
		1647	static int si_init_microcode(struct radeon_device *rdev)
		1648	{
		1649	const char *chip_name;
5078	serge	1650	const char *new_chip_name;
2997	Serge	1651	size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
5078	serge	1652	size_t smc_req_size, mc2_req_size;
2997	Serge	1653	char fw_name[30];
		1654	int err;
5078	serge	1655	int new_fw = 0;
2997	Serge	1656
		1657	DRM_DEBUG("\n");
		1658
		1659	switch (rdev->family) {
		1660	case CHIP_TAHITI:
		1661	chip_name = "TAHITI";
5078	serge	1662	new_chip_name = "tahiti";
2997	Serge	1663	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1664	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1665	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1666	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1667	mc_req_size = SI_MC_UCODE_SIZE * 4;
5078	serge	1668	mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
		1669	smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
2997	Serge	1670	break;
		1671	case CHIP_PITCAIRN:
		1672	chip_name = "PITCAIRN";
5078	serge	1673	new_chip_name = "pitcairn";
2997	Serge	1674	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1675	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1676	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1677	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1678	mc_req_size = SI_MC_UCODE_SIZE * 4;
5078	serge	1679	mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
		1680	smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
2997	Serge	1681	break;
		1682	case CHIP_VERDE:
		1683	chip_name = "VERDE";
5078	serge	1684	new_chip_name = "verde";
2997	Serge	1685	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1686	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1687	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1688	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1689	mc_req_size = SI_MC_UCODE_SIZE * 4;
5078	serge	1690	mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
		1691	smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
2997	Serge	1692	break;
3764	Serge	1693	case CHIP_OLAND:
		1694	chip_name = "OLAND";
5078	serge	1695	new_chip_name = "oland";
3764	Serge	1696	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1697	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1698	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1699	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
5078	serge	1700	mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
		1701	smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
3764	Serge	1702	break;
		1703	case CHIP_HAINAN:
		1704	chip_name = "HAINAN";
5078	serge	1705	new_chip_name = "hainan";
3764	Serge	1706	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1707	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1708	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1709	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
5078	serge	1710	mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
		1711	smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
3764	Serge	1712	break;
2997	Serge	1713	default: BUG();
		1714	}
		1715
5078	serge	1716	DRM_INFO("Loading %s Microcode\n", new_chip_name);
2997	Serge	1717
5078	serge	1718	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
		1719	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
		1720	if (err) {
6104	serge	1721	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
		1722	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
		1723	if (err)
		1724	goto out;
		1725	if (rdev->pfp_fw->size != pfp_req_size) {
		1726	printk(KERN_ERR
		1727	"si_cp: Bogus length %zu in firmware \"%s\"\n",
		1728	rdev->pfp_fw->size, fw_name);
		1729	err = -EINVAL;
		1730	goto out;
		1731	}
5078	serge	1732	} else {
		1733	err = radeon_ucode_validate(rdev->pfp_fw);
		1734	if (err) {
		1735	printk(KERN_ERR
		1736	"si_cp: validation failed for firmware \"%s\"\n",
		1737	fw_name);
		1738	goto out;
		1739	} else {
		1740	new_fw++;
		1741	}
		1742	}
2997	Serge	1743
5078	serge	1744	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
		1745	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
		1746	if (err) {
6104	serge	1747	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
		1748	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
		1749	if (err)
		1750	goto out;
		1751	if (rdev->me_fw->size != me_req_size) {
		1752	printk(KERN_ERR
		1753	"si_cp: Bogus length %zu in firmware \"%s\"\n",
		1754	rdev->me_fw->size, fw_name);
		1755	err = -EINVAL;
		1756	}
5078	serge	1757	} else {
		1758	err = radeon_ucode_validate(rdev->me_fw);
		1759	if (err) {
		1760	printk(KERN_ERR
		1761	"si_cp: validation failed for firmware \"%s\"\n",
		1762	fw_name);
		1763	goto out;
		1764	} else {
		1765	new_fw++;
		1766	}
		1767	}
2997	Serge	1768
5078	serge	1769	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
		1770	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
		1771	if (err) {
6104	serge	1772	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
		1773	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
		1774	if (err)
		1775	goto out;
		1776	if (rdev->ce_fw->size != ce_req_size) {
		1777	printk(KERN_ERR
		1778	"si_cp: Bogus length %zu in firmware \"%s\"\n",
		1779	rdev->ce_fw->size, fw_name);
		1780	err = -EINVAL;
		1781	}
5078	serge	1782	} else {
		1783	err = radeon_ucode_validate(rdev->ce_fw);
		1784	if (err) {
		1785	printk(KERN_ERR
		1786	"si_cp: validation failed for firmware \"%s\"\n",
		1787	fw_name);
		1788	goto out;
		1789	} else {
		1790	new_fw++;
		1791	}
		1792	}
2997	Serge	1793
5078	serge	1794	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
		1795	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
		1796	if (err) {
		1797	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
6104	serge	1798	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
		1799	if (err)
		1800	goto out;
		1801	if (rdev->rlc_fw->size != rlc_req_size) {
		1802	printk(KERN_ERR
		1803	"si_rlc: Bogus length %zu in firmware \"%s\"\n",
		1804	rdev->rlc_fw->size, fw_name);
		1805	err = -EINVAL;
		1806	}
5078	serge	1807	} else {
		1808	err = radeon_ucode_validate(rdev->rlc_fw);
		1809	if (err) {
		1810	printk(KERN_ERR
		1811	"si_cp: validation failed for firmware \"%s\"\n",
		1812	fw_name);
		1813	goto out;
		1814	} else {
		1815	new_fw++;
		1816	}
		1817	}
2997	Serge	1818
5078	serge	1819	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
		1820	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
		1821	if (err) {
6104	serge	1822	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
		1823	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
		1824	if (err) {
		1825	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
		1826	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
		1827	if (err)
		1828	goto out;
		1829	}
		1830	if ((rdev->mc_fw->size != mc_req_size) &&
		1831	(rdev->mc_fw->size != mc2_req_size)) {
		1832	printk(KERN_ERR
		1833	"si_mc: Bogus length %zu in firmware \"%s\"\n",
		1834	rdev->mc_fw->size, fw_name);
		1835	err = -EINVAL;
		1836	}
		1837	DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
5078	serge	1838	} else {
		1839	err = radeon_ucode_validate(rdev->mc_fw);
		1840	if (err) {
		1841	printk(KERN_ERR
		1842	"si_cp: validation failed for firmware \"%s\"\n",
		1843	fw_name);
		1844	goto out;
		1845	} else {
		1846	new_fw++;
		1847	}
		1848	}
2997	Serge	1849
5078	serge	1850	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
		1851	err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
		1852	if (err) {
6104	serge	1853	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
		1854	err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
		1855	if (err) {
		1856	printk(KERN_ERR
		1857	"smc: error loading firmware \"%s\"\n",
		1858	fw_name);
		1859	release_firmware(rdev->smc_fw);
		1860	rdev->smc_fw = NULL;
		1861	err = 0;
		1862	} else if (rdev->smc_fw->size != smc_req_size) {
		1863	printk(KERN_ERR
		1864	"si_smc: Bogus length %zu in firmware \"%s\"\n",
		1865	rdev->smc_fw->size, fw_name);
		1866	err = -EINVAL;
		1867	}
5078	serge	1868	} else {
		1869	err = radeon_ucode_validate(rdev->smc_fw);
		1870	if (err) {
		1871	printk(KERN_ERR
		1872	"si_cp: validation failed for firmware \"%s\"\n",
		1873	fw_name);
		1874	goto out;
		1875	} else {
		1876	new_fw++;
		1877	}
		1878	}
		1879
		1880	if (new_fw == 0) {
		1881	rdev->new_fw = false;
		1882	} else if (new_fw < 6) {
		1883	printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
		1884	err = -EINVAL;
		1885	} else {
		1886	rdev->new_fw = true;
		1887	}
2997	Serge	1888	out:
		1889	if (err) {
		1890	if (err != -EINVAL)
		1891	printk(KERN_ERR
		1892	"si_cp: Failed to load firmware \"%s\"\n",
		1893	fw_name);
		1894	release_firmware(rdev->pfp_fw);
		1895	rdev->pfp_fw = NULL;
		1896	release_firmware(rdev->me_fw);
		1897	rdev->me_fw = NULL;
		1898	release_firmware(rdev->ce_fw);
		1899	rdev->ce_fw = NULL;
		1900	release_firmware(rdev->rlc_fw);
		1901	rdev->rlc_fw = NULL;
		1902	release_firmware(rdev->mc_fw);
		1903	rdev->mc_fw = NULL;
5078	serge	1904	release_firmware(rdev->smc_fw);
		1905	rdev->smc_fw = NULL;
2997	Serge	1906	}
		1907	return err;
		1908	}
		1909
		1910	/* watermark setup */
		1911	static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
		1912	struct radeon_crtc *radeon_crtc,
		1913	struct drm_display_mode *mode,
		1914	struct drm_display_mode *other_mode)
		1915	{
5078	serge	1916	u32 tmp, buffer_alloc, i;
		1917	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
2997	Serge	1918	/*
		1919	* Line Buffer Setup
		1920	* There are 3 line buffers, each one shared by 2 display controllers.
		1921	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
		1922	* the display controllers. The paritioning is done via one of four
		1923	* preset allocations specified in bits 21:20:
		1924	* 0 - half lb
		1925	* 2 - whole lb, other crtc must be disabled
		1926	*/
		1927	/* this can get tricky if we have two large displays on a paired group
		1928	* of crtcs. Ideally for multiple large displays we'd assign them to
		1929	* non-linked crtcs for maximum line buffer allocation.
		1930	*/
		1931	if (radeon_crtc->base.enabled && mode) {
5078	serge	1932	if (other_mode) {
2997	Serge	1933	tmp = 0; /* 1/2 */
5078	serge	1934	buffer_alloc = 1;
		1935	} else {
2997	Serge	1936	tmp = 2; /* whole */
5078	serge	1937	buffer_alloc = 2;
		1938	}
		1939	} else {
2997	Serge	1940	tmp = 0;
5078	serge	1941	buffer_alloc = 0;
		1942	}
2997	Serge	1943
		1944	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
		1945	DC_LB_MEMORY_CONFIG(tmp));
		1946
5078	serge	1947	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
		1948	DMIF_BUFFERS_ALLOCATED(buffer_alloc));
		1949	for (i = 0; i < rdev->usec_timeout; i++) {
		1950	if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
		1951	DMIF_BUFFERS_ALLOCATED_COMPLETED)
		1952	break;
		1953	udelay(1);
		1954	}
		1955
2997	Serge	1956	if (radeon_crtc->base.enabled && mode) {
		1957	switch (tmp) {
		1958	case 0:
		1959	default:
		1960	return 4096 * 2;
		1961	case 2:
		1962	return 8192 * 2;
		1963	}
		1964	}
		1965
		1966	/* controller not enabled, so no lb used */
		1967	return 0;
		1968	}
		1969
		1970	static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
		1971	{
		1972	u32 tmp = RREG32(MC_SHARED_CHMAP);
		1973
		1974	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		1975	case 0:
		1976	default:
		1977	return 1;
		1978	case 1:
		1979	return 2;
		1980	case 2:
		1981	return 4;
		1982	case 3:
		1983	return 8;
		1984	case 4:
		1985	return 3;
		1986	case 5:
		1987	return 6;
		1988	case 6:
		1989	return 10;
		1990	case 7:
		1991	return 12;
		1992	case 8:
		1993	return 16;
		1994	}
		1995	}
		1996
		1997	struct dce6_wm_params {
		1998	u32 dram_channels; /* number of dram channels */
		1999	u32 yclk; /* bandwidth per dram data pin in kHz */
		2000	u32 sclk; /* engine clock in kHz */
		2001	u32 disp_clk; /* display clock in kHz */
		2002	u32 src_width; /* viewport width */
		2003	u32 active_time; /* active display time in ns */
		2004	u32 blank_time; /* blank time in ns */
		2005	bool interlaced; /* mode is interlaced */
		2006	fixed20_12 vsc; /* vertical scale ratio */
		2007	u32 num_heads; /* number of active crtcs */
		2008	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
		2009	u32 lb_size; /* line buffer allocated to pipe */
		2010	u32 vtaps; /* vertical scaler taps */
		2011	};
		2012
		2013	static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
		2014	{
		2015	/* Calculate raw DRAM Bandwidth */
		2016	fixed20_12 dram_efficiency; /* 0.7 */
		2017	fixed20_12 yclk, dram_channels, bandwidth;
		2018	fixed20_12 a;
		2019
		2020	a.full = dfixed_const(1000);
		2021	yclk.full = dfixed_const(wm->yclk);
		2022	yclk.full = dfixed_div(yclk, a);
		2023	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		2024	a.full = dfixed_const(10);
		2025	dram_efficiency.full = dfixed_const(7);
		2026	dram_efficiency.full = dfixed_div(dram_efficiency, a);
		2027	bandwidth.full = dfixed_mul(dram_channels, yclk);
		2028	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
		2029
		2030	return dfixed_trunc(bandwidth);
		2031	}
		2032
		2033	static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
		2034	{
		2035	/* Calculate DRAM Bandwidth and the part allocated to display. */
		2036	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
		2037	fixed20_12 yclk, dram_channels, bandwidth;
		2038	fixed20_12 a;
		2039
		2040	a.full = dfixed_const(1000);
		2041	yclk.full = dfixed_const(wm->yclk);
		2042	yclk.full = dfixed_div(yclk, a);
		2043	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		2044	a.full = dfixed_const(10);
		2045	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
		2046	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
		2047	bandwidth.full = dfixed_mul(dram_channels, yclk);
		2048	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
		2049
		2050	return dfixed_trunc(bandwidth);
		2051	}
		2052
		2053	static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
		2054	{
		2055	/* Calculate the display Data return Bandwidth */
		2056	fixed20_12 return_efficiency; /* 0.8 */
		2057	fixed20_12 sclk, bandwidth;
		2058	fixed20_12 a;
		2059
		2060	a.full = dfixed_const(1000);
		2061	sclk.full = dfixed_const(wm->sclk);
		2062	sclk.full = dfixed_div(sclk, a);
		2063	a.full = dfixed_const(10);
		2064	return_efficiency.full = dfixed_const(8);
		2065	return_efficiency.full = dfixed_div(return_efficiency, a);
		2066	a.full = dfixed_const(32);
		2067	bandwidth.full = dfixed_mul(a, sclk);
		2068	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
		2069
		2070	return dfixed_trunc(bandwidth);
		2071	}
		2072
		2073	static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
		2074	{
		2075	return 32;
		2076	}
		2077
		2078	static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
		2079	{
		2080	/* Calculate the DMIF Request Bandwidth */
		2081	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
		2082	fixed20_12 disp_clk, sclk, bandwidth;
		2083	fixed20_12 a, b1, b2;
		2084	u32 min_bandwidth;
		2085
		2086	a.full = dfixed_const(1000);
		2087	disp_clk.full = dfixed_const(wm->disp_clk);
		2088	disp_clk.full = dfixed_div(disp_clk, a);
		2089	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
		2090	b1.full = dfixed_mul(a, disp_clk);
		2091
		2092	a.full = dfixed_const(1000);
		2093	sclk.full = dfixed_const(wm->sclk);
		2094	sclk.full = dfixed_div(sclk, a);
		2095	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
		2096	b2.full = dfixed_mul(a, sclk);
		2097
		2098	a.full = dfixed_const(10);
		2099	disp_clk_request_efficiency.full = dfixed_const(8);
		2100	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
		2101
		2102	min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
		2103
		2104	a.full = dfixed_const(min_bandwidth);
		2105	bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
		2106
		2107	return dfixed_trunc(bandwidth);
		2108	}
		2109
		2110	static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
		2111	{
		2112	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
		2113	u32 dram_bandwidth = dce6_dram_bandwidth(wm);
		2114	u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
		2115	u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
		2116
		2117	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
		2118	}
		2119
		2120	static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
		2121	{
		2122	/* Calculate the display mode Average Bandwidth
		2123	* DisplayMode should contain the source and destination dimensions,
		2124	* timing, etc.
		2125	*/
		2126	fixed20_12 bpp;
		2127	fixed20_12 line_time;
		2128	fixed20_12 src_width;
		2129	fixed20_12 bandwidth;
		2130	fixed20_12 a;
		2131
		2132	a.full = dfixed_const(1000);
		2133	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
		2134	line_time.full = dfixed_div(line_time, a);
		2135	bpp.full = dfixed_const(wm->bytes_per_pixel);
		2136	src_width.full = dfixed_const(wm->src_width);
		2137	bandwidth.full = dfixed_mul(src_width, bpp);
		2138	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
		2139	bandwidth.full = dfixed_div(bandwidth, line_time);
		2140
		2141	return dfixed_trunc(bandwidth);
		2142	}
		2143
		2144	static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
		2145	{
		2146	/* First calcualte the latency in ns */
		2147	u32 mc_latency = 2000; /* 2000 ns. */
		2148	u32 available_bandwidth = dce6_available_bandwidth(wm);
		2149	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
		2150	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
		2151	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
		2152	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
		2153	(wm->num_heads * cursor_line_pair_return_time);
		2154	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
		2155	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
		2156	u32 tmp, dmif_size = 12288;
		2157	fixed20_12 a, b, c;
		2158
		2159	if (wm->num_heads == 0)
		2160	return 0;
		2161
		2162	a.full = dfixed_const(2);
		2163	b.full = dfixed_const(1);
		2164	if ((wm->vsc.full > a.full) \|\|
		2165	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) \|\|
		2166	(wm->vtaps >= 5) \|\|
		2167	((wm->vsc.full >= a.full) && wm->interlaced))
		2168	max_src_lines_per_dst_line = 4;
		2169	else
		2170	max_src_lines_per_dst_line = 2;
		2171
		2172	a.full = dfixed_const(available_bandwidth);
		2173	b.full = dfixed_const(wm->num_heads);
		2174	a.full = dfixed_div(a, b);
		2175
		2176	b.full = dfixed_const(mc_latency + 512);
		2177	c.full = dfixed_const(wm->disp_clk);
		2178	b.full = dfixed_div(b, c);
		2179
		2180	c.full = dfixed_const(dmif_size);
		2181	b.full = dfixed_div(c, b);
		2182
		2183	tmp = min(dfixed_trunc(a), dfixed_trunc(b));
		2184
		2185	b.full = dfixed_const(1000);
		2186	c.full = dfixed_const(wm->disp_clk);
		2187	b.full = dfixed_div(c, b);
		2188	c.full = dfixed_const(wm->bytes_per_pixel);
		2189	b.full = dfixed_mul(b, c);
		2190
		2191	lb_fill_bw = min(tmp, dfixed_trunc(b));
		2192
		2193	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
		2194	b.full = dfixed_const(1000);
		2195	c.full = dfixed_const(lb_fill_bw);
		2196	b.full = dfixed_div(c, b);
		2197	a.full = dfixed_div(a, b);
		2198	line_fill_time = dfixed_trunc(a);
		2199
		2200	if (line_fill_time < wm->active_time)
		2201	return latency;
		2202	else
		2203	return latency + (line_fill_time - wm->active_time);
		2204
		2205	}
		2206
		2207	static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
		2208	{
		2209	if (dce6_average_bandwidth(wm) <=
		2210	(dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
		2211	return true;
		2212	else
		2213	return false;
		2214	};
		2215
		2216	static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
		2217	{
		2218	if (dce6_average_bandwidth(wm) <=
		2219	(dce6_available_bandwidth(wm) / wm->num_heads))
		2220	return true;
		2221	else
		2222	return false;
		2223	};
		2224
		2225	static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
		2226	{
		2227	u32 lb_partitions = wm->lb_size / wm->src_width;
		2228	u32 line_time = wm->active_time + wm->blank_time;
		2229	u32 latency_tolerant_lines;
		2230	u32 latency_hiding;
		2231	fixed20_12 a;
		2232
		2233	a.full = dfixed_const(1);
		2234	if (wm->vsc.full > a.full)
		2235	latency_tolerant_lines = 1;
		2236	else {
		2237	if (lb_partitions <= (wm->vtaps + 1))
		2238	latency_tolerant_lines = 1;
		2239	else
		2240	latency_tolerant_lines = 2;
		2241	}
		2242
		2243	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
		2244
		2245	if (dce6_latency_watermark(wm) <= latency_hiding)
		2246	return true;
		2247	else
		2248	return false;
		2249	}
		2250
		2251	static void dce6_program_watermarks(struct radeon_device *rdev,
		2252	struct radeon_crtc *radeon_crtc,
		2253	u32 lb_size, u32 num_heads)
		2254	{
		2255	struct drm_display_mode *mode = &radeon_crtc->base.mode;
5078	serge	2256	struct dce6_wm_params wm_low, wm_high;
		2257	u32 dram_channels;
2997	Serge	2258	u32 pixel_period;
		2259	u32 line_time = 0;
		2260	u32 latency_watermark_a = 0, latency_watermark_b = 0;
		2261	u32 priority_a_mark = 0, priority_b_mark = 0;
		2262	u32 priority_a_cnt = PRIORITY_OFF;
		2263	u32 priority_b_cnt = PRIORITY_OFF;
		2264	u32 tmp, arb_control3;
		2265	fixed20_12 a, b, c;
		2266
		2267	if (radeon_crtc->base.enabled && num_heads && mode) {
		2268	pixel_period = 1000000 / (u32)mode->clock;
		2269	line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
		2270	priority_a_cnt = 0;
		2271	priority_b_cnt = 0;
		2272
		2273	if (rdev->family == CHIP_ARUBA)
5078	serge	2274	dram_channels = evergreen_get_number_of_dram_channels(rdev);
2997	Serge	2275	else
5078	serge	2276	dram_channels = si_get_number_of_dram_channels(rdev);
2997	Serge	2277
5078	serge	2278	/* watermark for high clocks */
		2279	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
		2280	wm_high.yclk =
		2281	radeon_dpm_get_mclk(rdev, false) * 10;
		2282	wm_high.sclk =
		2283	radeon_dpm_get_sclk(rdev, false) * 10;
		2284	} else {
		2285	wm_high.yclk = rdev->pm.current_mclk * 10;
		2286	wm_high.sclk = rdev->pm.current_sclk * 10;
		2287	}
		2288
		2289	wm_high.disp_clk = mode->clock;
		2290	wm_high.src_width = mode->crtc_hdisplay;
		2291	wm_high.active_time = mode->crtc_hdisplay * pixel_period;
		2292	wm_high.blank_time = line_time - wm_high.active_time;
		2293	wm_high.interlaced = false;
		2294	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		2295	wm_high.interlaced = true;
		2296	wm_high.vsc = radeon_crtc->vsc;
		2297	wm_high.vtaps = 1;
		2298	if (radeon_crtc->rmx_type != RMX_OFF)
		2299	wm_high.vtaps = 2;
		2300	wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
		2301	wm_high.lb_size = lb_size;
		2302	wm_high.dram_channels = dram_channels;
		2303	wm_high.num_heads = num_heads;
		2304
		2305	/* watermark for low clocks */
		2306	if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
		2307	wm_low.yclk =
		2308	radeon_dpm_get_mclk(rdev, true) * 10;
		2309	wm_low.sclk =
		2310	radeon_dpm_get_sclk(rdev, true) * 10;
		2311	} else {
		2312	wm_low.yclk = rdev->pm.current_mclk * 10;
		2313	wm_low.sclk = rdev->pm.current_sclk * 10;
		2314	}
		2315
		2316	wm_low.disp_clk = mode->clock;
		2317	wm_low.src_width = mode->crtc_hdisplay;
		2318	wm_low.active_time = mode->crtc_hdisplay * pixel_period;
		2319	wm_low.blank_time = line_time - wm_low.active_time;
		2320	wm_low.interlaced = false;
		2321	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		2322	wm_low.interlaced = true;
		2323	wm_low.vsc = radeon_crtc->vsc;
		2324	wm_low.vtaps = 1;
		2325	if (radeon_crtc->rmx_type != RMX_OFF)
		2326	wm_low.vtaps = 2;
		2327	wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
		2328	wm_low.lb_size = lb_size;
		2329	wm_low.dram_channels = dram_channels;
		2330	wm_low.num_heads = num_heads;
		2331
2997	Serge	2332	/* set for high clocks */
5078	serge	2333	latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2997	Serge	2334	/* set for low clocks */
5078	serge	2335	latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2997	Serge	2336
		2337	/* possibly force display priority to high */
		2338	/* should really do this at mode validation time... */
5078	serge	2339	if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) \|\|
		2340	!dce6_average_bandwidth_vs_available_bandwidth(&wm_high) \|\|
		2341	!dce6_check_latency_hiding(&wm_high) \|\|
2997	Serge	2342	(rdev->disp_priority == 2)) {
		2343	DRM_DEBUG_KMS("force priority to high\n");
		2344	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		2345	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		2346	}
5078	serge	2347	if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) \|\|
		2348	!dce6_average_bandwidth_vs_available_bandwidth(&wm_low) \|\|
		2349	!dce6_check_latency_hiding(&wm_low) \|\|
		2350	(rdev->disp_priority == 2)) {
		2351	DRM_DEBUG_KMS("force priority to high\n");
		2352	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		2353	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		2354	}
2997	Serge	2355
		2356	a.full = dfixed_const(1000);
		2357	b.full = dfixed_const(mode->clock);
		2358	b.full = dfixed_div(b, a);
		2359	c.full = dfixed_const(latency_watermark_a);
		2360	c.full = dfixed_mul(c, b);
		2361	c.full = dfixed_mul(c, radeon_crtc->hsc);
		2362	c.full = dfixed_div(c, a);
		2363	a.full = dfixed_const(16);
		2364	c.full = dfixed_div(c, a);
		2365	priority_a_mark = dfixed_trunc(c);
		2366	priority_a_cnt \|= priority_a_mark & PRIORITY_MARK_MASK;
		2367
		2368	a.full = dfixed_const(1000);
		2369	b.full = dfixed_const(mode->clock);
		2370	b.full = dfixed_div(b, a);
		2371	c.full = dfixed_const(latency_watermark_b);
		2372	c.full = dfixed_mul(c, b);
		2373	c.full = dfixed_mul(c, radeon_crtc->hsc);
		2374	c.full = dfixed_div(c, a);
		2375	a.full = dfixed_const(16);
		2376	c.full = dfixed_div(c, a);
		2377	priority_b_mark = dfixed_trunc(c);
		2378	priority_b_cnt \|= priority_b_mark & PRIORITY_MARK_MASK;
6104	serge	2379
		2380	/* Save number of lines the linebuffer leads before the scanout */
		2381	radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2997	Serge	2382	}
		2383
		2384	/* select wm A */
		2385	arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
		2386	tmp = arb_control3;
		2387	tmp &= ~LATENCY_WATERMARK_MASK(3);
		2388	tmp \|= LATENCY_WATERMARK_MASK(1);
		2389	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
		2390	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
		2391	(LATENCY_LOW_WATERMARK(latency_watermark_a) \|
		2392	LATENCY_HIGH_WATERMARK(line_time)));
		2393	/* select wm B */
		2394	tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
		2395	tmp &= ~LATENCY_WATERMARK_MASK(3);
		2396	tmp \|= LATENCY_WATERMARK_MASK(2);
		2397	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
		2398	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
		2399	(LATENCY_LOW_WATERMARK(latency_watermark_b) \|
		2400	LATENCY_HIGH_WATERMARK(line_time)));
		2401	/* restore original selection */
		2402	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
		2403
		2404	/* write the priority marks */
		2405	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
		2406	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
		2407
5078	serge	2408	/* save values for DPM */
		2409	radeon_crtc->line_time = line_time;
		2410	radeon_crtc->wm_high = latency_watermark_a;
		2411	radeon_crtc->wm_low = latency_watermark_b;
2997	Serge	2412	}
		2413
		2414	void dce6_bandwidth_update(struct radeon_device *rdev)
		2415	{
		2416	struct drm_display_mode *mode0 = NULL;
		2417	struct drm_display_mode *mode1 = NULL;
		2418	u32 num_heads = 0, lb_size;
		2419	int i;
		2420
5271	serge	2421	if (!rdev->mode_info.mode_config_initialized)
		2422	return;
		2423
2997	Serge	2424	radeon_update_display_priority(rdev);
		2425
		2426	for (i = 0; i < rdev->num_crtc; i++) {
		2427	if (rdev->mode_info.crtcs[i]->base.enabled)
		2428	num_heads++;
		2429	}
		2430	for (i = 0; i < rdev->num_crtc; i += 2) {
		2431	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
		2432	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
		2433	lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
		2434	dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
		2435	lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
		2436	dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
		2437	}
		2438	}
		2439
		2440	/*
		2441	* Core functions
		2442	*/
		2443	static void si_tiling_mode_table_init(struct radeon_device *rdev)
		2444	{
7146	serge	2445	u32 *tile = rdev->config.si.tile_mode_array;
		2446	const u32 num_tile_mode_states =
		2447	ARRAY_SIZE(rdev->config.si.tile_mode_array);
		2448	u32 reg_offset, split_equal_to_row_size;
2997	Serge	2449
		2450	switch (rdev->config.si.mem_row_size_in_kb) {
		2451	case 1:
		2452	split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
		2453	break;
		2454	case 2:
		2455	default:
		2456	split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
		2457	break;
		2458	case 4:
		2459	split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
		2460	break;
		2461	}
		2462
7146	serge	2463	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
		2464	tile[reg_offset] = 0;
		2465
		2466	switch(rdev->family) {
		2467	case CHIP_TAHITI:
		2468	case CHIP_PITCAIRN:
		2469	/* non-AA compressed depth or any compressed stencil */
		2470	tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2471	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2472	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2473	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2474	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2475	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2476	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2477	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2478	/* 2xAA/4xAA compressed depth only */
		2479	tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2480	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2481	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2482	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		2483	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2484	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2485	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2486	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2487	/* 8xAA compressed depth only */
		2488	tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2489	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2490	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2491	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2492	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2493	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2494	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2495	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2496	/* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
		2497	tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2498	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2499	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2500	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		2501	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2502	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2503	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2504	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2505	/* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
		2506	tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2507	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2508	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2509	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2510	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2511	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2512	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2513	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2514	/* Uncompressed 16bpp depth - and stencil buffer allocated with it */
		2515	tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2516	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2517	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2518	TILE_SPLIT(split_equal_to_row_size) \|
		2519	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2520	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2521	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2522	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2523	/* Uncompressed 32bpp depth - and stencil buffer allocated with it */
		2524	tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2525	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2526	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2527	TILE_SPLIT(split_equal_to_row_size) \|
		2528	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2529	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2530	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2531	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2532	/* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
		2533	tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2534	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2535	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2536	TILE_SPLIT(split_equal_to_row_size) \|
		2537	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2538	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2539	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2540	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2541	/* 1D and 1D Array Surfaces */
		2542	tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) \|
		2543	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2544	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2545	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2546	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2547	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2548	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2549	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2550	/* Displayable maps. */
		2551	tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2552	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2553	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2554	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2555	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2556	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2557	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2558	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2559	/* Display 8bpp. */
		2560	tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2561	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2562	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2563	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2564	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2565	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2566	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2567	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2568	/* Display 16bpp. */
		2569	tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2570	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2571	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2572	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2573	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2574	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2575	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2576	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2577	/* Display 32bpp. */
		2578	tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2579	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2580	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2581	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2582	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2583	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2584	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2585	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2586	/* Thin. */
		2587	tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2588	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2589	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2590	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2591	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2592	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2593	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2594	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2595	/* Thin 8 bpp. */
		2596	tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2597	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2598	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2599	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2600	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2601	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2602	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2603	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2604	/* Thin 16 bpp. */
		2605	tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2606	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2607	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2608	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2609	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2610	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2611	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2612	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2613	/* Thin 32 bpp. */
		2614	tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2615	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2616	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2617	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2618	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2619	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2620	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2621	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2622	/* Thin 64 bpp. */
		2623	tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2624	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2625	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2626	TILE_SPLIT(split_equal_to_row_size) \|
		2627	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2628	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2629	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2630	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2631	/* 8 bpp PRT. */
		2632	tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2633	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2634	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2635	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2636	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2637	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) \|
		2638	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2639	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2640	/* 16 bpp PRT */
		2641	tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2642	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2643	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2644	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2645	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2646	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2647	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2648	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2649	/* 32 bpp PRT */
		2650	tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2651	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2652	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2653	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2654	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2655	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2656	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2657	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2658	/* 64 bpp PRT */
		2659	tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2660	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2661	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2662	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2663	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2664	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2665	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2666	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2667	/* 128 bpp PRT */
		2668	tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2669	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2670	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2671	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) \|
		2672	NUM_BANKS(ADDR_SURF_8_BANK) \|
		2673	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2674	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2675	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2676
		2677	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
		2678	WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
		2679	break;
		2680
		2681	case CHIP_VERDE:
		2682	case CHIP_OLAND:
		2683	case CHIP_HAINAN:
		2684	/* non-AA compressed depth or any compressed stencil */
		2685	tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2686	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2687	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2688	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2689	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2690	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2691	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2692	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2693	/* 2xAA/4xAA compressed depth only */
		2694	tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2695	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2696	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2697	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		2698	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2699	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2700	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2701	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2702	/* 8xAA compressed depth only */
		2703	tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2704	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2705	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2706	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2707	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2708	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2709	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2710	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2711	/* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
		2712	tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2713	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2714	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2715	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		2716	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2717	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2718	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2719	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2720	/* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
		2721	tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2722	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2723	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2724	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2725	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2726	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2727	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2728	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2729	/* Uncompressed 16bpp depth - and stencil buffer allocated with it */
		2730	tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2731	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2732	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2733	TILE_SPLIT(split_equal_to_row_size) \|
		2734	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2735	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2736	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2737	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2738	/* Uncompressed 32bpp depth - and stencil buffer allocated with it */
		2739	tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2740	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2741	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2742	TILE_SPLIT(split_equal_to_row_size) \|
		2743	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2744	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2745	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2746	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2747	/* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
		2748	tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2749	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2750	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2751	TILE_SPLIT(split_equal_to_row_size) \|
		2752	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2753	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2754	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2755	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2756	/* 1D and 1D Array Surfaces */
		2757	tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) \|
		2758	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2759	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2760	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2761	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2762	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2763	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2764	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2765	/* Displayable maps. */
		2766	tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2767	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2768	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2769	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2770	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2771	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2772	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2773	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2774	/* Display 8bpp. */
		2775	tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2776	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2777	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2778	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2779	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2780	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2781	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2782	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2783	/* Display 16bpp. */
		2784	tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2785	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2786	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2787	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2788	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2789	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2790	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2791	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2792	/* Display 32bpp. */
		2793	tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2794	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2795	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2796	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2797	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2798	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2799	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2800	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2801	/* Thin. */
		2802	tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2803	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2804	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2805	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2806	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2807	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2808	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2809	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2810	/* Thin 8 bpp. */
		2811	tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2812	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2813	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2814	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2815	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2816	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2817	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2818	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2819	/* Thin 16 bpp. */
		2820	tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2821	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2822	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2823	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2824	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2825	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2826	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2827	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2828	/* Thin 32 bpp. */
		2829	tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2830	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2831	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2832	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2833	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2834	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2835	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2836	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2837	/* Thin 64 bpp. */
		2838	tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2839	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2840	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2841	TILE_SPLIT(split_equal_to_row_size) \|
		2842	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2843	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2844	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2845	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2846	/* 8 bpp PRT. */
		2847	tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2848	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2849	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2850	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2851	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2852	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) \|
		2853	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2854	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2855	/* 16 bpp PRT */
		2856	tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2857	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2858	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2859	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2860	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2861	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2862	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2863	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2864	/* 32 bpp PRT */
		2865	tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2866	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2867	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2868	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2869	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2870	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2871	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2872	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2873	/* 64 bpp PRT */
		2874	tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2875	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2876	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2877	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2878	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2879	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2880	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2881	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2882	/* 128 bpp PRT */
		2883	tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2884	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2885	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2886	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) \|
		2887	NUM_BANKS(ADDR_SURF_8_BANK) \|
		2888	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2889	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2890	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2891
		2892	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
		2893	WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
		2894	break;
		2895
		2896	default:
2997	Serge	2897	DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
7146	serge	2898	}
2997	Serge	2899	}
		2900
		2901	static void si_select_se_sh(struct radeon_device *rdev,
		2902	u32 se_num, u32 sh_num)
		2903	{
		2904	u32 data = INSTANCE_BROADCAST_WRITES;
		2905
		2906	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3764	Serge	2907	data \|= SH_BROADCAST_WRITES \| SE_BROADCAST_WRITES;
2997	Serge	2908	else if (se_num == 0xffffffff)
		2909	data \|= SE_BROADCAST_WRITES \| SH_INDEX(sh_num);
		2910	else if (sh_num == 0xffffffff)
		2911	data \|= SH_BROADCAST_WRITES \| SE_INDEX(se_num);
		2912	else
		2913	data \|= SH_INDEX(sh_num) \| SE_INDEX(se_num);
		2914	WREG32(GRBM_GFX_INDEX, data);
		2915	}
		2916
		2917	static u32 si_create_bitmask(u32 bit_width)
		2918	{
		2919	u32 i, mask = 0;
		2920
		2921	for (i = 0; i < bit_width; i++) {
		2922	mask <<= 1;
		2923	mask \|= 1;
		2924	}
		2925	return mask;
		2926	}
		2927
		2928	static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
		2929	{
		2930	u32 data, mask;
		2931
		2932	data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
		2933	if (data & 1)
		2934	data &= INACTIVE_CUS_MASK;
		2935	else
		2936	data = 0;
		2937	data \|= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
		2938
		2939	data >>= INACTIVE_CUS_SHIFT;
		2940
		2941	mask = si_create_bitmask(cu_per_sh);
		2942
		2943	return ~data & mask;
		2944	}
		2945
		2946	static void si_setup_spi(struct radeon_device *rdev,
		2947	u32 se_num, u32 sh_per_se,
		2948	u32 cu_per_sh)
		2949	{
		2950	int i, j, k;
		2951	u32 data, mask, active_cu;
		2952
		2953	for (i = 0; i < se_num; i++) {
		2954	for (j = 0; j < sh_per_se; j++) {
		2955	si_select_se_sh(rdev, i, j);
		2956	data = RREG32(SPI_STATIC_THREAD_MGMT_3);
		2957	active_cu = si_get_cu_enabled(rdev, cu_per_sh);
		2958
		2959	mask = 1;
		2960	for (k = 0; k < 16; k++) {
		2961	mask <<= k;
		2962	if (active_cu & mask) {
		2963	data &= ~mask;
		2964	WREG32(SPI_STATIC_THREAD_MGMT_3, data);
		2965	break;
		2966	}
		2967	}
		2968	}
		2969	}
		2970	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		2971	}
		2972
		2973	static u32 si_get_rb_disabled(struct radeon_device *rdev,
5078	serge	2974	u32 max_rb_num_per_se,
2997	Serge	2975	u32 sh_per_se)
		2976	{
		2977	u32 data, mask;
		2978
		2979	data = RREG32(CC_RB_BACKEND_DISABLE);
		2980	if (data & 1)
		2981	data &= BACKEND_DISABLE_MASK;
		2982	else
		2983	data = 0;
		2984	data \|= RREG32(GC_USER_RB_BACKEND_DISABLE);
		2985
		2986	data >>= BACKEND_DISABLE_SHIFT;
		2987
5078	serge	2988	mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2997	Serge	2989
		2990	return data & mask;
		2991	}
		2992
		2993	static void si_setup_rb(struct radeon_device *rdev,
		2994	u32 se_num, u32 sh_per_se,
5078	serge	2995	u32 max_rb_num_per_se)
2997	Serge	2996	{
		2997	int i, j;
		2998	u32 data, mask;
		2999	u32 disabled_rbs = 0;
		3000	u32 enabled_rbs = 0;
		3001
		3002	for (i = 0; i < se_num; i++) {
		3003	for (j = 0; j < sh_per_se; j++) {
		3004	si_select_se_sh(rdev, i, j);
5078	serge	3005	data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
2997	Serge	3006	disabled_rbs \|= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
		3007	}
		3008	}
		3009	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		3010
		3011	mask = 1;
5078	serge	3012	for (i = 0; i < max_rb_num_per_se * se_num; i++) {
2997	Serge	3013	if (!(disabled_rbs & mask))
		3014	enabled_rbs \|= mask;
		3015	mask <<= 1;
		3016	}
		3017
5078	serge	3018	rdev->config.si.backend_enable_mask = enabled_rbs;
		3019
2997	Serge	3020	for (i = 0; i < se_num; i++) {
		3021	si_select_se_sh(rdev, i, 0xffffffff);
		3022	data = 0;
		3023	for (j = 0; j < sh_per_se; j++) {
		3024	switch (enabled_rbs & 3) {
		3025	case 1:
		3026	data \|= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
		3027	break;
		3028	case 2:
		3029	data \|= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
		3030	break;
		3031	case 3:
		3032	default:
		3033	data \|= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
		3034	break;
		3035	}
		3036	enabled_rbs >>= 2;
		3037	}
		3038	WREG32(PA_SC_RASTER_CONFIG, data);
		3039	}
		3040	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		3041	}
		3042
		3043	static void si_gpu_init(struct radeon_device *rdev)
		3044	{
		3045	u32 gb_addr_config = 0;
		3046	u32 mc_shared_chmap, mc_arb_ramcfg;
		3047	u32 sx_debug_1;
		3048	u32 hdp_host_path_cntl;
		3049	u32 tmp;
		3050	int i, j;
		3051
		3052	switch (rdev->family) {
		3053	case CHIP_TAHITI:
		3054	rdev->config.si.max_shader_engines = 2;
		3055	rdev->config.si.max_tile_pipes = 12;
		3056	rdev->config.si.max_cu_per_sh = 8;
		3057	rdev->config.si.max_sh_per_se = 2;
		3058	rdev->config.si.max_backends_per_se = 4;
		3059	rdev->config.si.max_texture_channel_caches = 12;
		3060	rdev->config.si.max_gprs = 256;
		3061	rdev->config.si.max_gs_threads = 32;
		3062	rdev->config.si.max_hw_contexts = 8;
		3063
		3064	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		3065	rdev->config.si.sc_prim_fifo_size_backend = 0x100;
		3066	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		3067	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		3068	gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
		3069	break;
		3070	case CHIP_PITCAIRN:
		3071	rdev->config.si.max_shader_engines = 2;
		3072	rdev->config.si.max_tile_pipes = 8;
		3073	rdev->config.si.max_cu_per_sh = 5;
		3074	rdev->config.si.max_sh_per_se = 2;
		3075	rdev->config.si.max_backends_per_se = 4;
		3076	rdev->config.si.max_texture_channel_caches = 8;
		3077	rdev->config.si.max_gprs = 256;
		3078	rdev->config.si.max_gs_threads = 32;
		3079	rdev->config.si.max_hw_contexts = 8;
		3080
		3081	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		3082	rdev->config.si.sc_prim_fifo_size_backend = 0x100;
		3083	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		3084	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		3085	gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
		3086	break;
		3087	case CHIP_VERDE:
		3088	default:
		3089	rdev->config.si.max_shader_engines = 1;
		3090	rdev->config.si.max_tile_pipes = 4;
3764	Serge	3091	rdev->config.si.max_cu_per_sh = 5;
2997	Serge	3092	rdev->config.si.max_sh_per_se = 2;
		3093	rdev->config.si.max_backends_per_se = 4;
		3094	rdev->config.si.max_texture_channel_caches = 4;
		3095	rdev->config.si.max_gprs = 256;
		3096	rdev->config.si.max_gs_threads = 32;
		3097	rdev->config.si.max_hw_contexts = 8;
		3098
		3099	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		3100	rdev->config.si.sc_prim_fifo_size_backend = 0x40;
		3101	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		3102	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		3103	gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
		3104	break;
3764	Serge	3105	case CHIP_OLAND:
		3106	rdev->config.si.max_shader_engines = 1;
		3107	rdev->config.si.max_tile_pipes = 4;
		3108	rdev->config.si.max_cu_per_sh = 6;
		3109	rdev->config.si.max_sh_per_se = 1;
		3110	rdev->config.si.max_backends_per_se = 2;
		3111	rdev->config.si.max_texture_channel_caches = 4;
		3112	rdev->config.si.max_gprs = 256;
		3113	rdev->config.si.max_gs_threads = 16;
		3114	rdev->config.si.max_hw_contexts = 8;
		3115
		3116	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		3117	rdev->config.si.sc_prim_fifo_size_backend = 0x40;
		3118	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		3119	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		3120	gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
		3121	break;
		3122	case CHIP_HAINAN:
		3123	rdev->config.si.max_shader_engines = 1;
		3124	rdev->config.si.max_tile_pipes = 4;
		3125	rdev->config.si.max_cu_per_sh = 5;
		3126	rdev->config.si.max_sh_per_se = 1;
		3127	rdev->config.si.max_backends_per_se = 1;
		3128	rdev->config.si.max_texture_channel_caches = 2;
		3129	rdev->config.si.max_gprs = 256;
		3130	rdev->config.si.max_gs_threads = 16;
		3131	rdev->config.si.max_hw_contexts = 8;
		3132
		3133	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		3134	rdev->config.si.sc_prim_fifo_size_backend = 0x40;
		3135	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		3136	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		3137	gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
		3138	break;
2997	Serge	3139	}
		3140
		3141	/* Initialize HDP */
		3142	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		3143	WREG32((0x2c14 + j), 0x00000000);
		3144	WREG32((0x2c18 + j), 0x00000000);
		3145	WREG32((0x2c1c + j), 0x00000000);
		3146	WREG32((0x2c20 + j), 0x00000000);
		3147	WREG32((0x2c24 + j), 0x00000000);
		3148	}
		3149
		3150	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
6104	serge	3151	WREG32(SRBM_INT_CNTL, 1);
		3152	WREG32(SRBM_INT_ACK, 1);
2997	Serge	3153
		3154	evergreen_fix_pci_max_read_req_size(rdev);
		3155
		3156	WREG32(BIF_FB_EN, FB_READ_EN \| FB_WRITE_EN);
		3157
		3158	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
		3159	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
		3160
		3161	rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
		3162	rdev->config.si.mem_max_burst_length_bytes = 256;
		3163	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
		3164	rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
		3165	if (rdev->config.si.mem_row_size_in_kb > 4)
		3166	rdev->config.si.mem_row_size_in_kb = 4;
		3167	/* XXX use MC settings? */
		3168	rdev->config.si.shader_engine_tile_size = 32;
		3169	rdev->config.si.num_gpus = 1;
		3170	rdev->config.si.multi_gpu_tile_size = 64;
		3171
		3172	/* fix up row size */
		3173	gb_addr_config &= ~ROW_SIZE_MASK;
		3174	switch (rdev->config.si.mem_row_size_in_kb) {
		3175	case 1:
		3176	default:
		3177	gb_addr_config \|= ROW_SIZE(0);
		3178	break;
		3179	case 2:
		3180	gb_addr_config \|= ROW_SIZE(1);
		3181	break;
		3182	case 4:
		3183	gb_addr_config \|= ROW_SIZE(2);
		3184	break;
		3185	}
		3186
		3187	/* setup tiling info dword. gb_addr_config is not adequate since it does
		3188	* not have bank info, so create a custom tiling dword.
		3189	* bits 3:0 num_pipes
		3190	* bits 7:4 num_banks
		3191	* bits 11:8 group_size
		3192	* bits 15:12 row_size
		3193	*/
		3194	rdev->config.si.tile_config = 0;
		3195	switch (rdev->config.si.num_tile_pipes) {
		3196	case 1:
		3197	rdev->config.si.tile_config \|= (0 << 0);
		3198	break;
		3199	case 2:
		3200	rdev->config.si.tile_config \|= (1 << 0);
		3201	break;
		3202	case 4:
		3203	rdev->config.si.tile_config \|= (2 << 0);
		3204	break;
		3205	case 8:
		3206	default:
		3207	/* XXX what about 12? */
		3208	rdev->config.si.tile_config \|= (3 << 0);
		3209	break;
6104	serge	3210	}
2997	Serge	3211	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
		3212	case 0: /* four banks */
		3213	rdev->config.si.tile_config \|= 0 << 4;
		3214	break;
		3215	case 1: /* eight banks */
		3216	rdev->config.si.tile_config \|= 1 << 4;
		3217	break;
		3218	case 2: /* sixteen banks */
		3219	default:
		3220	rdev->config.si.tile_config \|= 2 << 4;
		3221	break;
		3222	}
		3223	rdev->config.si.tile_config \|=
		3224	((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
		3225	rdev->config.si.tile_config \|=
		3226	((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
		3227
		3228	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		3229	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3764	Serge	3230	WREG32(DMIF_ADDR_CALC, gb_addr_config);
2997	Serge	3231	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3192	Serge	3232	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
		3233	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3764	Serge	3234	if (rdev->has_uvd) {
		3235	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
		3236	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
		3237	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
		3238	}
2997	Serge	3239
		3240	si_tiling_mode_table_init(rdev);
		3241
		3242	si_setup_rb(rdev, rdev->config.si.max_shader_engines,
		3243	rdev->config.si.max_sh_per_se,
		3244	rdev->config.si.max_backends_per_se);
		3245
		3246	si_setup_spi(rdev, rdev->config.si.max_shader_engines,
		3247	rdev->config.si.max_sh_per_se,
		3248	rdev->config.si.max_cu_per_sh);
		3249
5078	serge	3250	rdev->config.si.active_cus = 0;
		3251	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
		3252	for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
6104	serge	3253	rdev->config.si.active_cus +=
		3254	hweight32(si_get_cu_active_bitmap(rdev, i, j));
5078	serge	3255	}
6104	serge	3256	}
2997	Serge	3257
		3258	/* set HW defaults for 3D engine */
		3259	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \|
		3260	ROQ_IB2_START(0x2b)));
		3261	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) \| MEQ2_START(0x60));
		3262
		3263	sx_debug_1 = RREG32(SX_DEBUG_1);
		3264	WREG32(SX_DEBUG_1, sx_debug_1);
		3265
		3266	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
		3267
		3268	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) \|
		3269	SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) \|
		3270	SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) \|
		3271	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
		3272
		3273	WREG32(VGT_NUM_INSTANCES, 1);
		3274
		3275	WREG32(CP_PERFMON_CNTL, 0);
		3276
		3277	WREG32(SQ_CONFIG, 0);
		3278
		3279	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		3280	FORCE_EOV_MAX_REZ_CNT(255)));
		3281
		3282	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) \|
		3283	AUTO_INVLD_EN(ES_AND_GS_AUTO));
		3284
		3285	WREG32(VGT_GS_VERTEX_REUSE, 16);
		3286	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		3287
		3288	WREG32(CB_PERFCOUNTER0_SELECT0, 0);
		3289	WREG32(CB_PERFCOUNTER0_SELECT1, 0);
		3290	WREG32(CB_PERFCOUNTER1_SELECT0, 0);
		3291	WREG32(CB_PERFCOUNTER1_SELECT1, 0);
		3292	WREG32(CB_PERFCOUNTER2_SELECT0, 0);
		3293	WREG32(CB_PERFCOUNTER2_SELECT1, 0);
		3294	WREG32(CB_PERFCOUNTER3_SELECT0, 0);
		3295	WREG32(CB_PERFCOUNTER3_SELECT1, 0);
		3296
		3297	tmp = RREG32(HDP_MISC_CNTL);
		3298	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		3299	WREG32(HDP_MISC_CNTL, tmp);
		3300
		3301	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		3302	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		3303
		3304	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		3305
		3306	udelay(50);
		3307	}
		3308
		3309	/*
		3310	* GPU scratch registers helpers function.
		3311	*/
		3312	static void si_scratch_init(struct radeon_device *rdev)
		3313	{
		3314	int i;
		3315
		3316	rdev->scratch.num_reg = 7;
		3317	rdev->scratch.reg_base = SCRATCH_REG0;
		3318	for (i = 0; i < rdev->scratch.num_reg; i++) {
		3319	rdev->scratch.free[i] = true;
		3320	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
		3321	}
		3322	}
		3323
		3324	void si_fence_ring_emit(struct radeon_device *rdev,
		3325	struct radeon_fence *fence)
		3326	{
		3327	struct radeon_ring *ring = &rdev->ring[fence->ring];
		3328	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		3329
		3330	/* flush read cache over gart */
		3331	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		3332	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
		3333	radeon_ring_write(ring, 0);
		3334	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		3335	radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA \|
		3336	PACKET3_TC_ACTION_ENA \|
		3337	PACKET3_SH_KCACHE_ACTION_ENA \|
		3338	PACKET3_SH_ICACHE_ACTION_ENA);
		3339	radeon_ring_write(ring, 0xFFFFFFFF);
		3340	radeon_ring_write(ring, 0);
		3341	radeon_ring_write(ring, 10); /* poll interval */
		3342	/* EVENT_WRITE_EOP - flush caches, send int */
		3343	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		3344	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) \| EVENT_INDEX(5));
5078	serge	3345	radeon_ring_write(ring, lower_32_bits(addr));
2997	Serge	3346	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		3347	radeon_ring_write(ring, fence->seq);
		3348	radeon_ring_write(ring, 0);
		3349	}
		3350
		3351	/*
		3352	* IB stuff
		3353	*/
		3354	void si_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		3355	{
		3356	struct radeon_ring *ring = &rdev->ring[ib->ring];
5271	serge	3357	unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2997	Serge	3358	u32 header;
		3359
		3360	if (ib->is_const_ib) {
		3361	/* set switch buffer packet before const IB */
		3362	radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
		3363	radeon_ring_write(ring, 0);
		3364
		3365	header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
		3366	} else {
		3367	u32 next_rptr;
		3368	if (ring->rptr_save_reg) {
		3369	next_rptr = ring->wptr + 3 + 4 + 8;
		3370	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		3371	radeon_ring_write(ring, ((ring->rptr_save_reg -
		3372	PACKET3_SET_CONFIG_REG_START) >> 2));
		3373	radeon_ring_write(ring, next_rptr);
		3374	} else if (rdev->wb.enabled) {
		3375	next_rptr = ring->wptr + 5 + 4 + 8;
		3376	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
		3377	radeon_ring_write(ring, (1 << 8));
		3378	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5078	serge	3379	radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2997	Serge	3380	radeon_ring_write(ring, next_rptr);
		3381	}
		3382
		3383	header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
		3384	}
		3385
		3386	radeon_ring_write(ring, header);
		3387	radeon_ring_write(ring,
		3388	#ifdef __BIG_ENDIAN
		3389	(2 << 0) \|
		3390	#endif
		3391	(ib->gpu_addr & 0xFFFFFFFC));
		3392	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
5271	serge	3393	radeon_ring_write(ring, ib->length_dw \| (vm_id << 24));
2997	Serge	3394
		3395	if (!ib->is_const_ib) {
		3396	/* flush read cache over gart for this vmid */
		3397	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		3398	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
5271	serge	3399	radeon_ring_write(ring, vm_id);
2997	Serge	3400	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		3401	radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA \|
		3402	PACKET3_TC_ACTION_ENA \|
		3403	PACKET3_SH_KCACHE_ACTION_ENA \|
		3404	PACKET3_SH_ICACHE_ACTION_ENA);
		3405	radeon_ring_write(ring, 0xFFFFFFFF);
		3406	radeon_ring_write(ring, 0);
		3407	radeon_ring_write(ring, 10); /* poll interval */
		3408	}
		3409	}
		3410
		3411	/*
		3412	* CP.
		3413	*/
		3414	static void si_cp_enable(struct radeon_device *rdev, bool enable)
		3415	{
		3416	if (enable)
		3417	WREG32(CP_ME_CNTL, 0);
		3418	else {
5078	serge	3419	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
6104	serge	3420	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2997	Serge	3421	WREG32(CP_ME_CNTL, (CP_ME_HALT \| CP_PFP_HALT \| CP_CE_HALT));
		3422	WREG32(SCRATCH_UMSK, 0);
3192	Serge	3423	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
		3424	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		3425	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2997	Serge	3426	}
		3427	udelay(50);
		3428	}
		3429
		3430	static int si_cp_load_microcode(struct radeon_device *rdev)
		3431	{
		3432	int i;
		3433
5078	serge	3434	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->ce_fw)
2997	Serge	3435	return -EINVAL;
		3436
		3437	si_cp_enable(rdev, false);
		3438
5078	serge	3439	if (rdev->new_fw) {
		3440	const struct gfx_firmware_header_v1_0 *pfp_hdr =
		3441	(const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
		3442	const struct gfx_firmware_header_v1_0 *ce_hdr =
		3443	(const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
		3444	const struct gfx_firmware_header_v1_0 *me_hdr =
		3445	(const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
		3446	const __le32 *fw_data;
		3447	u32 fw_size;
		3448
		3449	radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
		3450	radeon_ucode_print_gfx_hdr(&ce_hdr->header);
		3451	radeon_ucode_print_gfx_hdr(&me_hdr->header);
		3452
		3453	/* PFP */
		3454	fw_data = (const __le32 *)
		3455	(rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
		3456	fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
		3457	WREG32(CP_PFP_UCODE_ADDR, 0);
		3458	for (i = 0; i < fw_size; i++)
		3459	WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
		3460	WREG32(CP_PFP_UCODE_ADDR, 0);
		3461
		3462	/* CE */
		3463	fw_data = (const __le32 *)
		3464	(rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
		3465	fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
		3466	WREG32(CP_CE_UCODE_ADDR, 0);
		3467	for (i = 0; i < fw_size; i++)
		3468	WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
		3469	WREG32(CP_CE_UCODE_ADDR, 0);
		3470
		3471	/* ME */
		3472	fw_data = (const __be32 *)
		3473	(rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
		3474	fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
		3475	WREG32(CP_ME_RAM_WADDR, 0);
		3476	for (i = 0; i < fw_size; i++)
		3477	WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
		3478	WREG32(CP_ME_RAM_WADDR, 0);
		3479	} else {
		3480	const __be32 *fw_data;
		3481
6104	serge	3482	/* PFP */
		3483	fw_data = (const __be32 *)rdev->pfp_fw->data;
		3484	WREG32(CP_PFP_UCODE_ADDR, 0);
		3485	for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
		3486	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		3487	WREG32(CP_PFP_UCODE_ADDR, 0);
2997	Serge	3488
6104	serge	3489	/* CE */
		3490	fw_data = (const __be32 *)rdev->ce_fw->data;
		3491	WREG32(CP_CE_UCODE_ADDR, 0);
		3492	for (i = 0; i < SI_CE_UCODE_SIZE; i++)
		3493	WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
		3494	WREG32(CP_CE_UCODE_ADDR, 0);
2997	Serge	3495
6104	serge	3496	/* ME */
		3497	fw_data = (const __be32 *)rdev->me_fw->data;
		3498	WREG32(CP_ME_RAM_WADDR, 0);
		3499	for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
		3500	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		3501	WREG32(CP_ME_RAM_WADDR, 0);
5078	serge	3502	}
2997	Serge	3503
		3504	WREG32(CP_PFP_UCODE_ADDR, 0);
		3505	WREG32(CP_CE_UCODE_ADDR, 0);
		3506	WREG32(CP_ME_RAM_WADDR, 0);
		3507	WREG32(CP_ME_RAM_RADDR, 0);
		3508	return 0;
		3509	}
		3510
		3511	static int si_cp_start(struct radeon_device *rdev)
		3512	{
		3513	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		3514	int r, i;
		3515
		3516	r = radeon_ring_lock(rdev, ring, 7 + 4);
		3517	if (r) {
		3518	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		3519	return r;
		3520	}
		3521	/* init the CP */
		3522	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		3523	radeon_ring_write(ring, 0x1);
		3524	radeon_ring_write(ring, 0x0);
		3525	radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
		3526	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		3527	radeon_ring_write(ring, 0);
		3528	radeon_ring_write(ring, 0);
		3529
		3530	/* init the CE partitions */
		3531	radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
		3532	radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
		3533	radeon_ring_write(ring, 0xc000);
		3534	radeon_ring_write(ring, 0xe000);
5078	serge	3535	radeon_ring_unlock_commit(rdev, ring, false);
2997	Serge	3536
		3537	si_cp_enable(rdev, true);
		3538
		3539	r = radeon_ring_lock(rdev, ring, si_default_size + 10);
		3540	if (r) {
		3541	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		3542	return r;
		3543	}
		3544
		3545	/* setup clear context state */
		3546	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		3547	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
		3548
		3549	for (i = 0; i < si_default_size; i++)
		3550	radeon_ring_write(ring, si_default_state[i]);
		3551
		3552	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		3553	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
		3554
		3555	/* set clear context state */
		3556	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
		3557	radeon_ring_write(ring, 0);
		3558
		3559	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
		3560	radeon_ring_write(ring, 0x00000316);
		3561	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		3562	radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
		3563
5078	serge	3564	radeon_ring_unlock_commit(rdev, ring, false);
2997	Serge	3565
		3566	for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
		3567	ring = &rdev->ring[i];
		3568	r = radeon_ring_lock(rdev, ring, 2);
		3569
		3570	/* clear the compute context state */
		3571	radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
		3572	radeon_ring_write(ring, 0);
		3573
5078	serge	3574	radeon_ring_unlock_commit(rdev, ring, false);
2997	Serge	3575	}
		3576
		3577	return 0;
		3578	}
		3579
		3580	static void si_cp_fini(struct radeon_device *rdev)
		3581	{
		3582	struct radeon_ring *ring;
		3583	si_cp_enable(rdev, false);
		3584
5078	serge	3585	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		3586	radeon_ring_fini(rdev, ring);
		3587	radeon_scratch_free(rdev, ring->rptr_save_reg);
2997	Serge	3588
5078	serge	3589	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
		3590	radeon_ring_fini(rdev, ring);
		3591	radeon_scratch_free(rdev, ring->rptr_save_reg);
2997	Serge	3592
5078	serge	3593	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
		3594	radeon_ring_fini(rdev, ring);
		3595	radeon_scratch_free(rdev, ring->rptr_save_reg);
2997	Serge	3596	}
		3597
		3598	static int si_cp_resume(struct radeon_device *rdev)
		3599	{
		3600	struct radeon_ring *ring;
		3601	u32 tmp;
		3602	u32 rb_bufsz;
		3603	int r;
		3604
5078	serge	3605	si_enable_gui_idle_interrupt(rdev, false);
2997	Serge	3606
		3607	WREG32(CP_SEM_WAIT_TIMER, 0x0);
		3608	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
		3609
		3610	/* Set the write pointer delay */
		3611	WREG32(CP_RB_WPTR_DELAY, 0);
		3612
		3613	WREG32(CP_DEBUG, 0);
		3614	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		3615
		3616	/* ring 0 - compute and gfx */
		3617	/* Set ring buffer size */
		3618	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5078	serge	3619	rb_bufsz = order_base_2(ring->ring_size / 8);
		3620	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
2997	Serge	3621	#ifdef __BIG_ENDIAN
		3622	tmp \|= BUF_SWAP_32BIT;
		3623	#endif
		3624	WREG32(CP_RB0_CNTL, tmp);
		3625
		3626	/* Initialize the ring buffer's read and write pointers */
		3627	WREG32(CP_RB0_CNTL, tmp \| RB_RPTR_WR_ENA);
		3628	ring->wptr = 0;
		3629	WREG32(CP_RB0_WPTR, ring->wptr);
		3630
3120	serge	3631	/* set the wb address whether it's enabled or not */
2997	Serge	3632	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
		3633	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		3634
		3635	if (rdev->wb.enabled)
		3636	WREG32(SCRATCH_UMSK, 0xff);
		3637	else {
		3638	tmp \|= RB_NO_UPDATE;
		3639	WREG32(SCRATCH_UMSK, 0);
		3640	}
		3641
		3642	mdelay(1);
		3643	WREG32(CP_RB0_CNTL, tmp);
		3644
		3645	WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
		3646
		3647	/* ring1 - compute only */
		3648	/* Set ring buffer size */
		3649	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5078	serge	3650	rb_bufsz = order_base_2(ring->ring_size / 8);
		3651	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
2997	Serge	3652	#ifdef __BIG_ENDIAN
		3653	tmp \|= BUF_SWAP_32BIT;
		3654	#endif
		3655	WREG32(CP_RB1_CNTL, tmp);
		3656
		3657	/* Initialize the ring buffer's read and write pointers */
		3658	WREG32(CP_RB1_CNTL, tmp \| RB_RPTR_WR_ENA);
		3659	ring->wptr = 0;
		3660	WREG32(CP_RB1_WPTR, ring->wptr);
		3661
3120	serge	3662	/* set the wb address whether it's enabled or not */
2997	Serge	3663	WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
		3664	WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
		3665
		3666	mdelay(1);
		3667	WREG32(CP_RB1_CNTL, tmp);
		3668
		3669	WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
		3670
		3671	/* ring2 - compute only */
		3672	/* Set ring buffer size */
		3673	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5078	serge	3674	rb_bufsz = order_base_2(ring->ring_size / 8);
		3675	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
2997	Serge	3676	#ifdef __BIG_ENDIAN
		3677	tmp \|= BUF_SWAP_32BIT;
		3678	#endif
		3679	WREG32(CP_RB2_CNTL, tmp);
		3680
		3681	/* Initialize the ring buffer's read and write pointers */
		3682	WREG32(CP_RB2_CNTL, tmp \| RB_RPTR_WR_ENA);
		3683	ring->wptr = 0;
		3684	WREG32(CP_RB2_WPTR, ring->wptr);
		3685
3120	serge	3686	/* set the wb address whether it's enabled or not */
2997	Serge	3687	WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
		3688	WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
		3689
		3690	mdelay(1);
		3691	WREG32(CP_RB2_CNTL, tmp);
		3692
		3693	WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
		3694
		3695	/* start the rings */
		3696	si_cp_start(rdev);
		3697	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
		3698	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
		3699	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
		3700	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
		3701	if (r) {
		3702	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
		3703	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		3704	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
		3705	return r;
		3706	}
		3707	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
		3708	if (r) {
		3709	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		3710	}
		3711	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
		3712	if (r) {
		3713	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
		3714	}
		3715
5078	serge	3716	si_enable_gui_idle_interrupt(rdev, true);
		3717
		3718	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
		3719	radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
		3720
2997	Serge	3721	return 0;
		3722	}
		3723
5078	serge	3724	u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
2997	Serge	3725	{
3764	Serge	3726	u32 reset_mask = 0;
		3727	u32 tmp;
2997	Serge	3728
3764	Serge	3729	/* GRBM_STATUS */
		3730	tmp = RREG32(GRBM_STATUS);
		3731	if (tmp & (PA_BUSY \| SC_BUSY \|
		3732	BCI_BUSY \| SX_BUSY \|
		3733	TA_BUSY \| VGT_BUSY \|
		3734	DB_BUSY \| CB_BUSY \|
		3735	GDS_BUSY \| SPI_BUSY \|
		3736	IA_BUSY \| IA_BUSY_NO_DMA))
		3737	reset_mask \|= RADEON_RESET_GFX;
		3738
		3739	if (tmp & (CF_RQ_PENDING \| PF_RQ_PENDING \|
		3740	CP_BUSY \| CP_COHERENCY_BUSY))
		3741	reset_mask \|= RADEON_RESET_CP;
		3742
		3743	if (tmp & GRBM_EE_BUSY)
		3744	reset_mask \|= RADEON_RESET_GRBM \| RADEON_RESET_GFX \| RADEON_RESET_CP;
		3745
		3746	/* GRBM_STATUS2 */
		3747	tmp = RREG32(GRBM_STATUS2);
		3748	if (tmp & (RLC_RQ_PENDING \| RLC_BUSY))
		3749	reset_mask \|= RADEON_RESET_RLC;
		3750
		3751	/* DMA_STATUS_REG 0 */
		3752	tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
		3753	if (!(tmp & DMA_IDLE))
		3754	reset_mask \|= RADEON_RESET_DMA;
		3755
		3756	/* DMA_STATUS_REG 1 */
		3757	tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
		3758	if (!(tmp & DMA_IDLE))
		3759	reset_mask \|= RADEON_RESET_DMA1;
		3760
		3761	/* SRBM_STATUS2 */
		3762	tmp = RREG32(SRBM_STATUS2);
		3763	if (tmp & DMA_BUSY)
		3764	reset_mask \|= RADEON_RESET_DMA;
		3765
		3766	if (tmp & DMA1_BUSY)
		3767	reset_mask \|= RADEON_RESET_DMA1;
		3768
		3769	/* SRBM_STATUS */
		3770	tmp = RREG32(SRBM_STATUS);
		3771
		3772	if (tmp & IH_BUSY)
		3773	reset_mask \|= RADEON_RESET_IH;
		3774
		3775	if (tmp & SEM_BUSY)
		3776	reset_mask \|= RADEON_RESET_SEM;
		3777
		3778	if (tmp & GRBM_RQ_PENDING)
		3779	reset_mask \|= RADEON_RESET_GRBM;
		3780
		3781	if (tmp & VMC_BUSY)
		3782	reset_mask \|= RADEON_RESET_VMC;
		3783
		3784	if (tmp & (MCB_BUSY \| MCB_NON_DISPLAY_BUSY \|
		3785	MCC_BUSY \| MCD_BUSY))
		3786	reset_mask \|= RADEON_RESET_MC;
		3787
		3788	if (evergreen_is_display_hung(rdev))
		3789	reset_mask \|= RADEON_RESET_DISPLAY;
		3790
		3791	/* VM_L2_STATUS */
		3792	tmp = RREG32(VM_L2_STATUS);
		3793	if (tmp & L2_BUSY)
		3794	reset_mask \|= RADEON_RESET_VMC;
		3795
		3796	/* Skip MC reset as it's mostly likely not hung, just busy */
		3797	if (reset_mask & RADEON_RESET_MC) {
		3798	DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
		3799	reset_mask &= ~RADEON_RESET_MC;
2997	Serge	3800	}
3764	Serge	3801
		3802	return reset_mask;
2997	Serge	3803	}
		3804
3764	Serge	3805	static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
2997	Serge	3806	{
3764	Serge	3807	struct evergreen_mc_save save;
		3808	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
		3809	u32 tmp;
2997	Serge	3810
3764	Serge	3811	if (reset_mask == 0)
3192	Serge	3812	return;
2997	Serge	3813
3764	Serge	3814	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3192	Serge	3815
3764	Serge	3816	evergreen_print_gpu_status_regs(rdev);
		3817	dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
		3818	RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
		3819	dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
		3820	RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
		3821
5078	serge	3822	/* disable PG/CG */
		3823	si_fini_pg(rdev);
		3824	si_fini_cg(rdev);
		3825
		3826	/* stop the rlc */
		3827	si_rlc_stop(rdev);
		3828
2997	Serge	3829	/* Disable CP parsing/prefetching */
		3830	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT \| CP_CE_HALT);
		3831
3764	Serge	3832	if (reset_mask & RADEON_RESET_DMA) {
		3833	/* dma0 */
		3834	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
		3835	tmp &= ~DMA_RB_ENABLE;
		3836	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
		3837	}
		3838	if (reset_mask & RADEON_RESET_DMA1) {
		3839	/* dma1 */
		3840	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
		3841	tmp &= ~DMA_RB_ENABLE;
		3842	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
		3843	}
		3844
		3845	udelay(50);
		3846
		3847	evergreen_mc_stop(rdev, &save);
		3848	if (evergreen_mc_wait_for_idle(rdev)) {
		3849	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		3850	}
		3851
		3852	if (reset_mask & (RADEON_RESET_GFX \| RADEON_RESET_COMPUTE \| RADEON_RESET_CP)) {
		3853	grbm_soft_reset = SOFT_RESET_CB \|
6104	serge	3854	SOFT_RESET_DB \|
		3855	SOFT_RESET_GDS \|
		3856	SOFT_RESET_PA \|
		3857	SOFT_RESET_SC \|
		3858	SOFT_RESET_BCI \|
		3859	SOFT_RESET_SPI \|
		3860	SOFT_RESET_SX \|
		3861	SOFT_RESET_TC \|
		3862	SOFT_RESET_TA \|
		3863	SOFT_RESET_VGT \|
3764	Serge	3864	SOFT_RESET_IA;
		3865	}
2997	Serge	3866
3764	Serge	3867	if (reset_mask & RADEON_RESET_CP) {
		3868	grbm_soft_reset \|= SOFT_RESET_CP \| SOFT_RESET_VGT;
3192	Serge	3869
3764	Serge	3870	srbm_soft_reset \|= SOFT_RESET_GRBM;
		3871	}
3192	Serge	3872
3764	Serge	3873	if (reset_mask & RADEON_RESET_DMA)
		3874	srbm_soft_reset \|= SOFT_RESET_DMA;
3192	Serge	3875
3764	Serge	3876	if (reset_mask & RADEON_RESET_DMA1)
		3877	srbm_soft_reset \|= SOFT_RESET_DMA1;
3192	Serge	3878
3764	Serge	3879	if (reset_mask & RADEON_RESET_DISPLAY)
		3880	srbm_soft_reset \|= SOFT_RESET_DC;
3192	Serge	3881
3764	Serge	3882	if (reset_mask & RADEON_RESET_RLC)
		3883	grbm_soft_reset \|= SOFT_RESET_RLC;
3192	Serge	3884
3764	Serge	3885	if (reset_mask & RADEON_RESET_SEM)
		3886	srbm_soft_reset \|= SOFT_RESET_SEM;
3192	Serge	3887
3764	Serge	3888	if (reset_mask & RADEON_RESET_IH)
		3889	srbm_soft_reset \|= SOFT_RESET_IH;
3192	Serge	3890
3764	Serge	3891	if (reset_mask & RADEON_RESET_GRBM)
		3892	srbm_soft_reset \|= SOFT_RESET_GRBM;
3192	Serge	3893
3764	Serge	3894	if (reset_mask & RADEON_RESET_VMC)
		3895	srbm_soft_reset \|= SOFT_RESET_VMC;
3192	Serge	3896
3764	Serge	3897	if (reset_mask & RADEON_RESET_MC)
		3898	srbm_soft_reset \|= SOFT_RESET_MC;
3192	Serge	3899
3764	Serge	3900	if (grbm_soft_reset) {
		3901	tmp = RREG32(GRBM_SOFT_RESET);
		3902	tmp \|= grbm_soft_reset;
		3903	dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
		3904	WREG32(GRBM_SOFT_RESET, tmp);
		3905	tmp = RREG32(GRBM_SOFT_RESET);
3192	Serge	3906
3764	Serge	3907	udelay(50);
3192	Serge	3908
3764	Serge	3909	tmp &= ~grbm_soft_reset;
		3910	WREG32(GRBM_SOFT_RESET, tmp);
		3911	tmp = RREG32(GRBM_SOFT_RESET);
3192	Serge	3912	}
		3913
3764	Serge	3914	if (srbm_soft_reset) {
		3915	tmp = RREG32(SRBM_SOFT_RESET);
		3916	tmp \|= srbm_soft_reset;
		3917	dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
		3918	WREG32(SRBM_SOFT_RESET, tmp);
		3919	tmp = RREG32(SRBM_SOFT_RESET);
3192	Serge	3920
3764	Serge	3921	udelay(50);
3192	Serge	3922
3764	Serge	3923	tmp &= ~srbm_soft_reset;
		3924	WREG32(SRBM_SOFT_RESET, tmp);
		3925	tmp = RREG32(SRBM_SOFT_RESET);
		3926	}
		3927
3192	Serge	3928	/* Wait a little for things to settle down */
		3929	udelay(50);
		3930
2997	Serge	3931	evergreen_mc_resume(rdev, &save);
3764	Serge	3932	udelay(50);
		3933
		3934	evergreen_print_gpu_status_regs(rdev);
2997	Serge	3935	}
		3936
5078	serge	3937	static void si_set_clk_bypass_mode(struct radeon_device *rdev)
		3938	{
		3939	u32 tmp, i;
		3940
		3941	tmp = RREG32(CG_SPLL_FUNC_CNTL);
		3942	tmp \|= SPLL_BYPASS_EN;
		3943	WREG32(CG_SPLL_FUNC_CNTL, tmp);
		3944
		3945	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
		3946	tmp \|= SPLL_CTLREQ_CHG;
		3947	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
		3948
		3949	for (i = 0; i < rdev->usec_timeout; i++) {
		3950	if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
		3951	break;
		3952	udelay(1);
		3953	}
		3954
		3955	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
		3956	tmp &= ~(SPLL_CTLREQ_CHG \| SCLK_MUX_UPDATE);
		3957	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
		3958
		3959	tmp = RREG32(MPLL_CNTL_MODE);
		3960	tmp &= ~MPLL_MCLK_SEL;
		3961	WREG32(MPLL_CNTL_MODE, tmp);
		3962	}
		3963
		3964	static void si_spll_powerdown(struct radeon_device *rdev)
		3965	{
		3966	u32 tmp;
		3967
		3968	tmp = RREG32(SPLL_CNTL_MODE);
		3969	tmp \|= SPLL_SW_DIR_CONTROL;
		3970	WREG32(SPLL_CNTL_MODE, tmp);
		3971
		3972	tmp = RREG32(CG_SPLL_FUNC_CNTL);
		3973	tmp \|= SPLL_RESET;
		3974	WREG32(CG_SPLL_FUNC_CNTL, tmp);
		3975
		3976	tmp = RREG32(CG_SPLL_FUNC_CNTL);
		3977	tmp \|= SPLL_SLEEP;
		3978	WREG32(CG_SPLL_FUNC_CNTL, tmp);
		3979
		3980	tmp = RREG32(SPLL_CNTL_MODE);
		3981	tmp &= ~SPLL_SW_DIR_CONTROL;
		3982	WREG32(SPLL_CNTL_MODE, tmp);
		3983	}
		3984
		3985	static void si_gpu_pci_config_reset(struct radeon_device *rdev)
		3986	{
		3987	struct evergreen_mc_save save;
		3988	u32 tmp, i;
		3989
		3990	dev_info(rdev->dev, "GPU pci config reset\n");
		3991
		3992	/* disable dpm? */
		3993
		3994	/* disable cg/pg */
		3995	si_fini_pg(rdev);
		3996	si_fini_cg(rdev);
		3997
		3998	/* Disable CP parsing/prefetching */
		3999	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT \| CP_CE_HALT);
		4000	/* dma0 */
		4001	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
		4002	tmp &= ~DMA_RB_ENABLE;
		4003	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
		4004	/* dma1 */
		4005	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
		4006	tmp &= ~DMA_RB_ENABLE;
		4007	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
		4008	/* XXX other engines? */
		4009
		4010	/* halt the rlc, disable cp internal ints */
		4011	si_rlc_stop(rdev);
		4012
		4013	udelay(50);
		4014
		4015	/* disable mem access */
		4016	evergreen_mc_stop(rdev, &save);
		4017	if (evergreen_mc_wait_for_idle(rdev)) {
		4018	dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
		4019	}
		4020
		4021	/* set mclk/sclk to bypass */
		4022	si_set_clk_bypass_mode(rdev);
		4023	/* powerdown spll */
		4024	si_spll_powerdown(rdev);
		4025	/* disable BM */
		4026	pci_clear_master(rdev->pdev);
		4027	/* reset */
		4028	radeon_pci_config_reset(rdev);
		4029	/* wait for asic to come out of reset */
		4030	for (i = 0; i < rdev->usec_timeout; i++) {
		4031	if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
		4032	break;
		4033	udelay(1);
		4034	}
		4035	}
		4036
2997	Serge	4037	int si_asic_reset(struct radeon_device *rdev)
		4038	{
3764	Serge	4039	u32 reset_mask;
		4040
		4041	reset_mask = si_gpu_check_soft_reset(rdev);
		4042
		4043	if (reset_mask)
		4044	r600_set_bios_scratch_engine_hung(rdev, true);
		4045
5078	serge	4046	/* try soft reset */
3764	Serge	4047	si_gpu_soft_reset(rdev, reset_mask);
		4048
		4049	reset_mask = si_gpu_check_soft_reset(rdev);
		4050
5078	serge	4051	/* try pci config reset */
		4052	if (reset_mask && radeon_hard_reset)
		4053	si_gpu_pci_config_reset(rdev);
		4054
		4055	reset_mask = si_gpu_check_soft_reset(rdev);
		4056
3764	Serge	4057	if (!reset_mask)
		4058	r600_set_bios_scratch_engine_hung(rdev, false);
		4059
		4060	return 0;
		4061	}
		4062
		4063	/**
		4064	* si_gfx_is_lockup - Check if the GFX engine is locked up
		4065	*
		4066	* @rdev: radeon_device pointer
		4067	* @ring: radeon_ring structure holding ring information
		4068	*
		4069	* Check if the GFX engine is locked up.
		4070	* Returns true if the engine appears to be locked up, false if not.
		4071	*/
		4072	bool si_gfx_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
		4073	{
		4074	u32 reset_mask = si_gpu_check_soft_reset(rdev);
		4075
		4076	if (!(reset_mask & (RADEON_RESET_GFX \|
6104	serge	4077	RADEON_RESET_COMPUTE \|
3764	Serge	4078	RADEON_RESET_CP))) {
5078	serge	4079	radeon_ring_lockup_update(rdev, ring);
3764	Serge	4080	return false;
		4081	}
		4082	return radeon_ring_test_lockup(rdev, ring);
2997	Serge	4083	}
		4084
		4085	/* MC */
		4086	static void si_mc_program(struct radeon_device *rdev)
		4087	{
		4088	struct evergreen_mc_save save;
		4089	u32 tmp;
		4090	int i, j;
		4091
		4092	/* Initialize HDP */
		4093	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		4094	WREG32((0x2c14 + j), 0x00000000);
		4095	WREG32((0x2c18 + j), 0x00000000);
		4096	WREG32((0x2c1c + j), 0x00000000);
		4097	WREG32((0x2c20 + j), 0x00000000);
		4098	WREG32((0x2c24 + j), 0x00000000);
		4099	}
		4100	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		4101
		4102	evergreen_mc_stop(rdev, &save);
		4103	if (radeon_mc_wait_for_idle(rdev)) {
		4104	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		4105	}
3764	Serge	4106	if (!ASIC_IS_NODCE(rdev))
6104	serge	4107	/* Lockout access through VGA aperture*/
		4108	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2997	Serge	4109	/* Update configuration */
		4110	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		4111	rdev->mc.vram_start >> 12);
		4112	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		4113	rdev->mc.vram_end >> 12);
		4114	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
		4115	rdev->vram_scratch.gpu_addr >> 12);
		4116	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		4117	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		4118	WREG32(MC_VM_FB_LOCATION, tmp);
		4119	/* XXX double check these! */
		4120	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
		4121	WREG32(HDP_NONSURFACE_INFO, (2 << 7) \| (1 << 30));
		4122	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
		4123	WREG32(MC_VM_AGP_BASE, 0);
		4124	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		4125	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		4126	if (radeon_mc_wait_for_idle(rdev)) {
		4127	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		4128	}
		4129	evergreen_mc_resume(rdev, &save);
3764	Serge	4130	if (!ASIC_IS_NODCE(rdev)) {
6104	serge	4131	/* we need to own VRAM, so turn off the VGA renderer here
		4132	* to stop it overwriting our objects */
		4133	rv515_vga_render_disable(rdev);
		4134	}
2997	Serge	4135	}
		4136
5078	serge	4137	void si_vram_gtt_location(struct radeon_device *rdev,
6104	serge	4138	struct radeon_mc *mc)
2997	Serge	4139	{
		4140	if (mc->mc_vram_size > 0xFFC0000000ULL) {
		4141	/* leave room for at least 1024M GTT */
		4142	dev_warn(rdev->dev, "limiting VRAM\n");
		4143	mc->real_vram_size = 0xFFC0000000ULL;
		4144	mc->mc_vram_size = 0xFFC0000000ULL;
		4145	}
3764	Serge	4146	radeon_vram_location(rdev, &rdev->mc, 0);
2997	Serge	4147	rdev->mc.gtt_base_align = 0;
3764	Serge	4148	radeon_gtt_location(rdev, mc);
2997	Serge	4149	}
		4150
		4151	static int si_mc_init(struct radeon_device *rdev)
		4152	{
		4153	u32 tmp;
		4154	int chansize, numchan;
		4155
		4156	/* Get VRAM informations */
		4157	rdev->mc.vram_is_ddr = true;
		4158	tmp = RREG32(MC_ARB_RAMCFG);
		4159	if (tmp & CHANSIZE_OVERRIDE) {
		4160	chansize = 16;
		4161	} else if (tmp & CHANSIZE_MASK) {
		4162	chansize = 64;
		4163	} else {
		4164	chansize = 32;
		4165	}
		4166	tmp = RREG32(MC_SHARED_CHMAP);
		4167	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		4168	case 0:
		4169	default:
		4170	numchan = 1;
		4171	break;
		4172	case 1:
		4173	numchan = 2;
		4174	break;
		4175	case 2:
		4176	numchan = 4;
		4177	break;
		4178	case 3:
		4179	numchan = 8;
		4180	break;
		4181	case 4:
		4182	numchan = 3;
		4183	break;
		4184	case 5:
		4185	numchan = 6;
		4186	break;
		4187	case 6:
		4188	numchan = 10;
		4189	break;
		4190	case 7:
		4191	numchan = 12;
		4192	break;
		4193	case 8:
		4194	numchan = 16;
		4195	break;
		4196	}
		4197	rdev->mc.vram_width = numchan * chansize;
		4198	/* Could aper size report 0 ? */
		4199	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		4200	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
		4201	/* size in MB on si */
5078	serge	4202	tmp = RREG32(CONFIG_MEMSIZE);
		4203	/* some boards may have garbage in the upper 16 bits */
		4204	if (tmp & 0xffff0000) {
		4205	DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
		4206	if (tmp & 0xffff)
		4207	tmp &= 0xffff;
		4208	}
		4209	rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
		4210	rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
2997	Serge	4211	rdev->mc.visible_vram_size = rdev->mc.aper_size;
		4212	si_vram_gtt_location(rdev, &rdev->mc);
		4213	radeon_update_bandwidth_info(rdev);
		4214
		4215	return 0;
		4216	}
		4217
		4218	/*
		4219	* GART
		4220	*/
		4221	void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
		4222	{
		4223	/* flush hdp cache */
		4224	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		4225
		4226	/* bits 0-15 are the VM contexts0-15 */
		4227	WREG32(VM_INVALIDATE_REQUEST, 1);
		4228	}
		4229
		4230	static int si_pcie_gart_enable(struct radeon_device *rdev)
		4231	{
		4232	int r, i;
		4233
		4234	if (rdev->gart.robj == NULL) {
		4235	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		4236	return -EINVAL;
		4237	}
		4238	r = radeon_gart_table_vram_pin(rdev);
		4239	if (r)
		4240	return r;
		4241	/* Setup TLB control */
		4242	WREG32(MC_VM_MX_L1_TLB_CNTL,
		4243	(0xA << 7) \|
		4244	ENABLE_L1_TLB \|
5078	serge	4245	ENABLE_L1_FRAGMENT_PROCESSING \|
2997	Serge	4246	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		4247	ENABLE_ADVANCED_DRIVER_MODEL \|
		4248	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
		4249	/* Setup L2 cache */
		4250	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \|
5078	serge	4251	ENABLE_L2_FRAGMENT_PROCESSING \|
2997	Serge	4252	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		4253	ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE \|
		4254	EFFECTIVE_L2_QUEUE_SIZE(7) \|
		4255	CONTEXT1_IDENTITY_ACCESS_MODE(1));
		4256	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS \| INVALIDATE_L2_CACHE);
		4257	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY \|
5078	serge	4258	BANK_SELECT(4) \|
		4259	L2_CACHE_BIGK_FRAGMENT_SIZE(4));
2997	Serge	4260	/* setup context0 */
		4261	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		4262	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		4263	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		4264	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		4265	(u32)(rdev->dummy_page.addr >> 12));
		4266	WREG32(VM_CONTEXT0_CNTL2, 0);
		4267	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		4268	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
		4269
		4270	WREG32(0x15D4, 0);
		4271	WREG32(0x15D8, 0);
		4272	WREG32(0x15DC, 0);
		4273
		4274	/* empty context1-15 */
		4275	/* set vm size, must be a multiple of 4 */
		4276	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
6104	serge	4277	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
2997	Serge	4278	/* Assign the pt base to something valid for now; the pts used for
		4279	* the VMs are determined by the application and setup and assigned
		4280	* on the fly in the vm part of radeon_gart.c
		4281	*/
		4282	for (i = 1; i < 16; i++) {
		4283	if (i < 8)
		4284	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5078	serge	4285	rdev->vm_manager.saved_table_addr[i]);
2997	Serge	4286	else
		4287	WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5078	serge	4288	rdev->vm_manager.saved_table_addr[i]);
2997	Serge	4289	}
		4290
		4291	/* enable context1-15 */
		4292	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
		4293	(u32)(rdev->dummy_page.addr >> 12));
3192	Serge	4294	WREG32(VM_CONTEXT1_CNTL2, 4);
2997	Serge	4295	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(1) \|
5078	serge	4296	PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) \|
3192	Serge	4297	RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4298	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT \|
		4299	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4300	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT \|
		4301	PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4302	PDE0_PROTECTION_FAULT_ENABLE_DEFAULT \|
		4303	VALID_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4304	VALID_PROTECTION_FAULT_ENABLE_DEFAULT \|
		4305	READ_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4306	READ_PROTECTION_FAULT_ENABLE_DEFAULT \|
		4307	WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		4308	WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
2997	Serge	4309
		4310	si_pcie_gart_tlb_flush(rdev);
		4311	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		4312	(unsigned)(rdev->mc.gtt_size >> 20),
		4313	(unsigned long long)rdev->gart.table_addr);
		4314	rdev->gart.ready = true;
		4315	return 0;
		4316	}
		4317
		4318	static void si_pcie_gart_disable(struct radeon_device *rdev)
		4319	{
5078	serge	4320	unsigned i;
		4321
		4322	for (i = 1; i < 16; ++i) {
		4323	uint32_t reg;
		4324	if (i < 8)
		4325	reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
		4326	else
		4327	reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
		4328	rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
		4329	}
		4330
2997	Serge	4331	/* Disable all tables */
		4332	WREG32(VM_CONTEXT0_CNTL, 0);
		4333	WREG32(VM_CONTEXT1_CNTL, 0);
		4334	/* Setup TLB control */
		4335	WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		4336	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
		4337	/* Setup L2 cache */
		4338	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		4339	ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE \|
		4340	EFFECTIVE_L2_QUEUE_SIZE(7) \|
		4341	CONTEXT1_IDENTITY_ACCESS_MODE(1));
		4342	WREG32(VM_L2_CNTL2, 0);
		4343	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY \|
		4344	L2_CACHE_BIGK_FRAGMENT_SIZE(0));
		4345	radeon_gart_table_vram_unpin(rdev);
		4346	}
		4347
		4348	static void si_pcie_gart_fini(struct radeon_device *rdev)
		4349	{
		4350	si_pcie_gart_disable(rdev);
		4351	radeon_gart_table_vram_free(rdev);
6104	serge	4352	radeon_gart_fini(rdev);
2997	Serge	4353	}
		4354
		4355	/* vm parser */
		4356	static bool si_vm_reg_valid(u32 reg)
		4357	{
		4358	/* context regs are fine */
		4359	if (reg >= 0x28000)
		4360	return true;
		4361
		4362	/* check config regs */
		4363	switch (reg) {
		4364	case GRBM_GFX_INDEX:
3031	serge	4365	case CP_STRMOUT_CNTL:
2997	Serge	4366	case VGT_VTX_VECT_EJECT_REG:
		4367	case VGT_CACHE_INVALIDATION:
		4368	case VGT_ESGS_RING_SIZE:
		4369	case VGT_GSVS_RING_SIZE:
		4370	case VGT_GS_VERTEX_REUSE:
		4371	case VGT_PRIMITIVE_TYPE:
		4372	case VGT_INDEX_TYPE:
		4373	case VGT_NUM_INDICES:
		4374	case VGT_NUM_INSTANCES:
		4375	case VGT_TF_RING_SIZE:
		4376	case VGT_HS_OFFCHIP_PARAM:
		4377	case VGT_TF_MEMORY_BASE:
		4378	case PA_CL_ENHANCE:
		4379	case PA_SU_LINE_STIPPLE_VALUE:
		4380	case PA_SC_LINE_STIPPLE_STATE:
		4381	case PA_SC_ENHANCE:
		4382	case SQC_CACHES:
		4383	case SPI_STATIC_THREAD_MGMT_1:
		4384	case SPI_STATIC_THREAD_MGMT_2:
		4385	case SPI_STATIC_THREAD_MGMT_3:
		4386	case SPI_PS_MAX_WAVE_ID:
		4387	case SPI_CONFIG_CNTL:
		4388	case SPI_CONFIG_CNTL_1:
		4389	case TA_CNTL_AUX:
		4390	return true;
		4391	default:
		4392	DRM_ERROR("Invalid register 0x%x in CS\n", reg);
		4393	return false;
		4394	}
		4395	}
		4396
		4397	static int si_vm_packet3_ce_check(struct radeon_device *rdev,
		4398	u32 ib, struct radeon_cs_packet pkt)
		4399	{
		4400	switch (pkt->opcode) {
		4401	case PACKET3_NOP:
		4402	case PACKET3_SET_BASE:
		4403	case PACKET3_SET_CE_DE_COUNTERS:
		4404	case PACKET3_LOAD_CONST_RAM:
		4405	case PACKET3_WRITE_CONST_RAM:
		4406	case PACKET3_WRITE_CONST_RAM_OFFSET:
		4407	case PACKET3_DUMP_CONST_RAM:
		4408	case PACKET3_INCREMENT_CE_COUNTER:
		4409	case PACKET3_WAIT_ON_DE_COUNTER:
		4410	case PACKET3_CE_WRITE:
		4411	break;
		4412	default:
		4413	DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
		4414	return -EINVAL;
		4415	}
		4416	return 0;
		4417	}
		4418
5078	serge	4419	static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
		4420	{
		4421	u32 start_reg, reg, i;
		4422	u32 command = ib[idx + 4];
		4423	u32 info = ib[idx + 1];
		4424	u32 idx_value = ib[idx];
		4425	if (command & PACKET3_CP_DMA_CMD_SAS) {
		4426	/* src address space is register */
		4427	if (((info & 0x60000000) >> 29) == 0) {
		4428	start_reg = idx_value << 2;
		4429	if (command & PACKET3_CP_DMA_CMD_SAIC) {
		4430	reg = start_reg;
		4431	if (!si_vm_reg_valid(reg)) {
		4432	DRM_ERROR("CP DMA Bad SRC register\n");
		4433	return -EINVAL;
		4434	}
		4435	} else {
		4436	for (i = 0; i < (command & 0x1fffff); i++) {
		4437	reg = start_reg + (4 * i);
		4438	if (!si_vm_reg_valid(reg)) {
		4439	DRM_ERROR("CP DMA Bad SRC register\n");
		4440	return -EINVAL;
		4441	}
		4442	}
		4443	}
		4444	}
		4445	}
		4446	if (command & PACKET3_CP_DMA_CMD_DAS) {
		4447	/* dst address space is register */
		4448	if (((info & 0x00300000) >> 20) == 0) {
		4449	start_reg = ib[idx + 2];
		4450	if (command & PACKET3_CP_DMA_CMD_DAIC) {
		4451	reg = start_reg;
		4452	if (!si_vm_reg_valid(reg)) {
		4453	DRM_ERROR("CP DMA Bad DST register\n");
		4454	return -EINVAL;
		4455	}
		4456	} else {
		4457	for (i = 0; i < (command & 0x1fffff); i++) {
		4458	reg = start_reg + (4 * i);
		4459	if (!si_vm_reg_valid(reg)) {
		4460	DRM_ERROR("CP DMA Bad DST register\n");
		4461	return -EINVAL;
		4462	}
		4463	}
		4464	}
		4465	}
		4466	}
		4467	return 0;
		4468	}
		4469
2997	Serge	4470	static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
		4471	u32 ib, struct radeon_cs_packet pkt)
		4472	{
5078	serge	4473	int r;
2997	Serge	4474	u32 idx = pkt->idx + 1;
		4475	u32 idx_value = ib[idx];
		4476	u32 start_reg, end_reg, reg, i;
		4477
		4478	switch (pkt->opcode) {
		4479	case PACKET3_NOP:
		4480	case PACKET3_SET_BASE:
		4481	case PACKET3_CLEAR_STATE:
		4482	case PACKET3_INDEX_BUFFER_SIZE:
		4483	case PACKET3_DISPATCH_DIRECT:
		4484	case PACKET3_DISPATCH_INDIRECT:
		4485	case PACKET3_ALLOC_GDS:
		4486	case PACKET3_WRITE_GDS_RAM:
		4487	case PACKET3_ATOMIC_GDS:
		4488	case PACKET3_ATOMIC:
		4489	case PACKET3_OCCLUSION_QUERY:
		4490	case PACKET3_SET_PREDICATION:
		4491	case PACKET3_COND_EXEC:
		4492	case PACKET3_PRED_EXEC:
		4493	case PACKET3_DRAW_INDIRECT:
		4494	case PACKET3_DRAW_INDEX_INDIRECT:
		4495	case PACKET3_INDEX_BASE:
		4496	case PACKET3_DRAW_INDEX_2:
		4497	case PACKET3_CONTEXT_CONTROL:
		4498	case PACKET3_INDEX_TYPE:
		4499	case PACKET3_DRAW_INDIRECT_MULTI:
		4500	case PACKET3_DRAW_INDEX_AUTO:
		4501	case PACKET3_DRAW_INDEX_IMMD:
		4502	case PACKET3_NUM_INSTANCES:
		4503	case PACKET3_DRAW_INDEX_MULTI_AUTO:
		4504	case PACKET3_STRMOUT_BUFFER_UPDATE:
		4505	case PACKET3_DRAW_INDEX_OFFSET_2:
		4506	case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
		4507	case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
		4508	case PACKET3_MPEG_INDEX:
		4509	case PACKET3_WAIT_REG_MEM:
		4510	case PACKET3_MEM_WRITE:
		4511	case PACKET3_PFP_SYNC_ME:
		4512	case PACKET3_SURFACE_SYNC:
		4513	case PACKET3_EVENT_WRITE:
		4514	case PACKET3_EVENT_WRITE_EOP:
		4515	case PACKET3_EVENT_WRITE_EOS:
		4516	case PACKET3_SET_CONTEXT_REG:
		4517	case PACKET3_SET_CONTEXT_REG_INDIRECT:
		4518	case PACKET3_SET_SH_REG:
		4519	case PACKET3_SET_SH_REG_OFFSET:
		4520	case PACKET3_INCREMENT_DE_COUNTER:
		4521	case PACKET3_WAIT_ON_CE_COUNTER:
		4522	case PACKET3_WAIT_ON_AVAIL_BUFFER:
		4523	case PACKET3_ME_WRITE:
		4524	break;
		4525	case PACKET3_COPY_DATA:
		4526	if ((idx_value & 0xf00) == 0) {
		4527	reg = ib[idx + 3] * 4;
		4528	if (!si_vm_reg_valid(reg))
		4529	return -EINVAL;
		4530	}
		4531	break;
		4532	case PACKET3_WRITE_DATA:
		4533	if ((idx_value & 0xf00) == 0) {
		4534	start_reg = ib[idx + 1] * 4;
		4535	if (idx_value & 0x10000) {
		4536	if (!si_vm_reg_valid(start_reg))
		4537	return -EINVAL;
		4538	} else {
		4539	for (i = 0; i < (pkt->count - 2); i++) {
		4540	reg = start_reg + (4 * i);
		4541	if (!si_vm_reg_valid(reg))
		4542	return -EINVAL;
		4543	}
		4544	}
		4545	}
		4546	break;
		4547	case PACKET3_COND_WRITE:
		4548	if (idx_value & 0x100) {
		4549	reg = ib[idx + 5] * 4;
		4550	if (!si_vm_reg_valid(reg))
		4551	return -EINVAL;
		4552	}
		4553	break;
		4554	case PACKET3_COPY_DW:
		4555	if (idx_value & 0x2) {
		4556	reg = ib[idx + 3] * 4;
		4557	if (!si_vm_reg_valid(reg))
		4558	return -EINVAL;
		4559	}
		4560	break;
		4561	case PACKET3_SET_CONFIG_REG:
		4562	start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
		4563	end_reg = 4 * pkt->count + start_reg - 4;
		4564	if ((start_reg < PACKET3_SET_CONFIG_REG_START) \|\|
		4565	(start_reg >= PACKET3_SET_CONFIG_REG_END) \|\|
		4566	(end_reg >= PACKET3_SET_CONFIG_REG_END)) {
		4567	DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
		4568	return -EINVAL;
		4569	}
		4570	for (i = 0; i < pkt->count; i++) {
		4571	reg = start_reg + (4 * i);
		4572	if (!si_vm_reg_valid(reg))
		4573	return -EINVAL;
		4574	}
		4575	break;
3192	Serge	4576	case PACKET3_CP_DMA:
5078	serge	4577	r = si_vm_packet3_cp_dma_check(ib, idx);
		4578	if (r)
		4579	return r;
3192	Serge	4580	break;
2997	Serge	4581	default:
		4582	DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
		4583	return -EINVAL;
		4584	}
		4585	return 0;
		4586	}
		4587
		4588	static int si_vm_packet3_compute_check(struct radeon_device *rdev,
		4589	u32 ib, struct radeon_cs_packet pkt)
		4590	{
5078	serge	4591	int r;
2997	Serge	4592	u32 idx = pkt->idx + 1;
		4593	u32 idx_value = ib[idx];
		4594	u32 start_reg, reg, i;
		4595
		4596	switch (pkt->opcode) {
		4597	case PACKET3_NOP:
		4598	case PACKET3_SET_BASE:
		4599	case PACKET3_CLEAR_STATE:
		4600	case PACKET3_DISPATCH_DIRECT:
		4601	case PACKET3_DISPATCH_INDIRECT:
		4602	case PACKET3_ALLOC_GDS:
		4603	case PACKET3_WRITE_GDS_RAM:
		4604	case PACKET3_ATOMIC_GDS:
		4605	case PACKET3_ATOMIC:
		4606	case PACKET3_OCCLUSION_QUERY:
		4607	case PACKET3_SET_PREDICATION:
		4608	case PACKET3_COND_EXEC:
		4609	case PACKET3_PRED_EXEC:
		4610	case PACKET3_CONTEXT_CONTROL:
		4611	case PACKET3_STRMOUT_BUFFER_UPDATE:
		4612	case PACKET3_WAIT_REG_MEM:
		4613	case PACKET3_MEM_WRITE:
		4614	case PACKET3_PFP_SYNC_ME:
		4615	case PACKET3_SURFACE_SYNC:
		4616	case PACKET3_EVENT_WRITE:
		4617	case PACKET3_EVENT_WRITE_EOP:
		4618	case PACKET3_EVENT_WRITE_EOS:
		4619	case PACKET3_SET_CONTEXT_REG:
		4620	case PACKET3_SET_CONTEXT_REG_INDIRECT:
		4621	case PACKET3_SET_SH_REG:
		4622	case PACKET3_SET_SH_REG_OFFSET:
		4623	case PACKET3_INCREMENT_DE_COUNTER:
		4624	case PACKET3_WAIT_ON_CE_COUNTER:
		4625	case PACKET3_WAIT_ON_AVAIL_BUFFER:
		4626	case PACKET3_ME_WRITE:
		4627	break;
		4628	case PACKET3_COPY_DATA:
		4629	if ((idx_value & 0xf00) == 0) {
		4630	reg = ib[idx + 3] * 4;
		4631	if (!si_vm_reg_valid(reg))
		4632	return -EINVAL;
		4633	}
		4634	break;
		4635	case PACKET3_WRITE_DATA:
		4636	if ((idx_value & 0xf00) == 0) {
		4637	start_reg = ib[idx + 1] * 4;
		4638	if (idx_value & 0x10000) {
		4639	if (!si_vm_reg_valid(start_reg))
		4640	return -EINVAL;
		4641	} else {
		4642	for (i = 0; i < (pkt->count - 2); i++) {
		4643	reg = start_reg + (4 * i);
		4644	if (!si_vm_reg_valid(reg))
		4645	return -EINVAL;
		4646	}
		4647	}
		4648	}
		4649	break;
		4650	case PACKET3_COND_WRITE:
		4651	if (idx_value & 0x100) {
		4652	reg = ib[idx + 5] * 4;
		4653	if (!si_vm_reg_valid(reg))
		4654	return -EINVAL;
		4655	}
		4656	break;
		4657	case PACKET3_COPY_DW:
		4658	if (idx_value & 0x2) {
		4659	reg = ib[idx + 3] * 4;
		4660	if (!si_vm_reg_valid(reg))
		4661	return -EINVAL;
		4662	}
		4663	break;
5078	serge	4664	case PACKET3_CP_DMA:
		4665	r = si_vm_packet3_cp_dma_check(ib, idx);
		4666	if (r)
		4667	return r;
		4668	break;
2997	Serge	4669	default:
		4670	DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
		4671	return -EINVAL;
		4672	}
		4673	return 0;
		4674	}
		4675
		4676	int si_ib_parse(struct radeon_device rdev, struct radeon_ib ib)
		4677	{
		4678	int ret = 0;
5271	serge	4679	u32 idx = 0, i;
2997	Serge	4680	struct radeon_cs_packet pkt;
		4681
		4682	do {
		4683	pkt.idx = idx;
3764	Serge	4684	pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
		4685	pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
2997	Serge	4686	pkt.one_reg_wr = 0;
		4687	switch (pkt.type) {
3764	Serge	4688	case RADEON_PACKET_TYPE0:
2997	Serge	4689	dev_err(rdev->dev, "Packet0 not allowed!\n");
		4690	ret = -EINVAL;
		4691	break;
3764	Serge	4692	case RADEON_PACKET_TYPE2:
2997	Serge	4693	idx += 1;
		4694	break;
3764	Serge	4695	case RADEON_PACKET_TYPE3:
		4696	pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
2997	Serge	4697	if (ib->is_const_ib)
		4698	ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
		4699	else {
		4700	switch (ib->ring) {
		4701	case RADEON_RING_TYPE_GFX_INDEX:
		4702	ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
		4703	break;
		4704	case CAYMAN_RING_TYPE_CP1_INDEX:
		4705	case CAYMAN_RING_TYPE_CP2_INDEX:
		4706	ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
		4707	break;
		4708	default:
		4709	dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
		4710	ret = -EINVAL;
		4711	break;
		4712	}
		4713	}
		4714	idx += pkt.count + 2;
		4715	break;
		4716	default:
		4717	dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
		4718	ret = -EINVAL;
		4719	break;
		4720	}
6104	serge	4721	if (ret) {
		4722	for (i = 0; i < ib->length_dw; i++) {
		4723	if (i == idx)
		4724	printk("\t0x%08x <---\n", ib->ptr[i]);
		4725	else
		4726	printk("\t0x%08x\n", ib->ptr[i]);
		4727	}
2997	Serge	4728	break;
6104	serge	4729	}
2997	Serge	4730	} while (idx < ib->length_dw);
		4731
		4732	return ret;
		4733	}
		4734
		4735	/*
		4736	* vm
		4737	*/
		4738	int si_vm_init(struct radeon_device *rdev)
		4739	{
		4740	/* number of VMs */
		4741	rdev->vm_manager.nvm = 16;
		4742	/* base offset of vram pages */
		4743	rdev->vm_manager.vram_base_offset = 0;
		4744
		4745	return 0;
		4746	}
		4747
		4748	void si_vm_fini(struct radeon_device *rdev)
		4749	{
		4750	}
		4751
		4752	/**
5078	serge	4753	* si_vm_decode_fault - print human readable fault info
2997	Serge	4754	*
		4755	* @rdev: radeon_device pointer
5078	serge	4756	* @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
		4757	* @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2997	Serge	4758	*
5078	serge	4759	* Print human readable fault information (SI).
2997	Serge	4760	*/
5078	serge	4761	static void si_vm_decode_fault(struct radeon_device *rdev,
		4762	u32 status, u32 addr)
2997	Serge	4763	{
5078	serge	4764	u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
		4765	u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
		4766	u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
		4767	char *block;
2997	Serge	4768
5078	serge	4769	if (rdev->family == CHIP_TAHITI) {
		4770	switch (mc_id) {
		4771	case 160:
		4772	case 144:
		4773	case 96:
		4774	case 80:
		4775	case 224:
		4776	case 208:
		4777	case 32:
		4778	case 16:
		4779	block = "CB";
		4780	break;
		4781	case 161:
		4782	case 145:
		4783	case 97:
		4784	case 81:
		4785	case 225:
		4786	case 209:
		4787	case 33:
		4788	case 17:
		4789	block = "CB_FMASK";
		4790	break;
		4791	case 162:
		4792	case 146:
		4793	case 98:
		4794	case 82:
		4795	case 226:
		4796	case 210:
		4797	case 34:
		4798	case 18:
		4799	block = "CB_CMASK";
		4800	break;
		4801	case 163:
		4802	case 147:
		4803	case 99:
		4804	case 83:
		4805	case 227:
		4806	case 211:
		4807	case 35:
		4808	case 19:
		4809	block = "CB_IMMED";
		4810	break;
		4811	case 164:
		4812	case 148:
		4813	case 100:
		4814	case 84:
		4815	case 228:
		4816	case 212:
		4817	case 36:
		4818	case 20:
		4819	block = "DB";
		4820	break;
		4821	case 165:
		4822	case 149:
		4823	case 101:
		4824	case 85:
		4825	case 229:
		4826	case 213:
		4827	case 37:
		4828	case 21:
		4829	block = "DB_HTILE";
		4830	break;
		4831	case 167:
		4832	case 151:
		4833	case 103:
		4834	case 87:
		4835	case 231:
		4836	case 215:
		4837	case 39:
		4838	case 23:
		4839	block = "DB_STEN";
		4840	break;
		4841	case 72:
		4842	case 68:
		4843	case 64:
		4844	case 8:
		4845	case 4:
		4846	case 0:
		4847	case 136:
		4848	case 132:
		4849	case 128:
		4850	case 200:
		4851	case 196:
		4852	case 192:
		4853	block = "TC";
		4854	break;
		4855	case 112:
		4856	case 48:
		4857	block = "CP";
		4858	break;
		4859	case 49:
		4860	case 177:
		4861	case 50:
		4862	case 178:
		4863	block = "SH";
		4864	break;
		4865	case 53:
		4866	case 190:
		4867	block = "VGT";
		4868	break;
		4869	case 117:
		4870	block = "IH";
		4871	break;
		4872	case 51:
		4873	case 115:
		4874	block = "RLC";
		4875	break;
		4876	case 119:
		4877	case 183:
		4878	block = "DMA0";
		4879	break;
		4880	case 61:
		4881	block = "DMA1";
		4882	break;
		4883	case 248:
		4884	case 120:
		4885	block = "HDP";
		4886	break;
		4887	default:
		4888	block = "unknown";
		4889	break;
2997	Serge	4890	}
3192	Serge	4891	} else {
5078	serge	4892	switch (mc_id) {
		4893	case 32:
		4894	case 16:
		4895	case 96:
		4896	case 80:
		4897	case 160:
		4898	case 144:
		4899	case 224:
		4900	case 208:
		4901	block = "CB";
		4902	break;
		4903	case 33:
		4904	case 17:
		4905	case 97:
		4906	case 81:
		4907	case 161:
		4908	case 145:
		4909	case 225:
		4910	case 209:
		4911	block = "CB_FMASK";
		4912	break;
		4913	case 34:
		4914	case 18:
		4915	case 98:
		4916	case 82:
		4917	case 162:
		4918	case 146:
		4919	case 226:
		4920	case 210:
		4921	block = "CB_CMASK";
		4922	break;
		4923	case 35:
		4924	case 19:
		4925	case 99:
		4926	case 83:
		4927	case 163:
		4928	case 147:
		4929	case 227:
		4930	case 211:
		4931	block = "CB_IMMED";
		4932	break;
		4933	case 36:
		4934	case 20:
		4935	case 100:
		4936	case 84:
		4937	case 164:
		4938	case 148:
		4939	case 228:
		4940	case 212:
		4941	block = "DB";
		4942	break;
		4943	case 37:
		4944	case 21:
		4945	case 101:
		4946	case 85:
		4947	case 165:
		4948	case 149:
		4949	case 229:
		4950	case 213:
		4951	block = "DB_HTILE";
		4952	break;
		4953	case 39:
		4954	case 23:
		4955	case 103:
		4956	case 87:
		4957	case 167:
		4958	case 151:
		4959	case 231:
		4960	case 215:
		4961	block = "DB_STEN";
		4962	break;
		4963	case 72:
		4964	case 68:
		4965	case 8:
		4966	case 4:
		4967	case 136:
		4968	case 132:
		4969	case 200:
		4970	case 196:
		4971	block = "TC";
		4972	break;
		4973	case 112:
		4974	case 48:
		4975	block = "CP";
		4976	break;
		4977	case 49:
		4978	case 177:
		4979	case 50:
		4980	case 178:
		4981	block = "SH";
		4982	break;
		4983	case 53:
		4984	block = "VGT";
		4985	break;
		4986	case 117:
		4987	block = "IH";
		4988	break;
		4989	case 51:
		4990	case 115:
		4991	block = "RLC";
		4992	break;
		4993	case 119:
		4994	case 183:
		4995	block = "DMA0";
		4996	break;
		4997	case 61:
		4998	block = "DMA1";
		4999	break;
		5000	case 248:
		5001	case 120:
		5002	block = "HDP";
		5003	break;
		5004	default:
		5005	block = "unknown";
		5006	break;
3192	Serge	5007	}
		5008	}
		5009
5078	serge	5010	printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
		5011	protections, vmid, addr,
		5012	(status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
		5013	block, mc_id);
2997	Serge	5014	}
		5015
5271	serge	5016	void si_vm_flush(struct radeon_device rdev, struct radeon_ring ring,
		5017	unsigned vm_id, uint64_t pd_addr)
2997	Serge	5018	{
		5019	/* write new base address */
		5020	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5078	serge	5021	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) \|
2997	Serge	5022	WRITE_DATA_DST_SEL(0)));
		5023
5271	serge	5024	if (vm_id < 8) {
2997	Serge	5025	radeon_ring_write(ring,
5271	serge	5026	(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
2997	Serge	5027	} else {
		5028	radeon_ring_write(ring,
5271	serge	5029	(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
2997	Serge	5030	}
		5031	radeon_ring_write(ring, 0);
5271	serge	5032	radeon_ring_write(ring, pd_addr >> 12);
2997	Serge	5033
		5034	/* flush hdp cache */
		5035	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5078	serge	5036	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) \|
2997	Serge	5037	WRITE_DATA_DST_SEL(0)));
		5038	radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
		5039	radeon_ring_write(ring, 0);
		5040	radeon_ring_write(ring, 0x1);
		5041
		5042	/* bits 0-15 are the VM contexts0-15 */
		5043	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5078	serge	5044	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) \|
2997	Serge	5045	WRITE_DATA_DST_SEL(0)));
		5046	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
		5047	radeon_ring_write(ring, 0);
5271	serge	5048	radeon_ring_write(ring, 1 << vm_id);
2997	Serge	5049
6104	serge	5050	/* wait for the invalidate to complete */
		5051	radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
		5052	radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) \| /* always */
		5053	WAIT_REG_MEM_ENGINE(0))); /* me */
		5054	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
		5055	radeon_ring_write(ring, 0);
		5056	radeon_ring_write(ring, 0); /* ref */
		5057	radeon_ring_write(ring, 0); /* mask */
		5058	radeon_ring_write(ring, 0x20); /* poll interval */
		5059
2997	Serge	5060	/* sync PFP to ME, otherwise we might get invalid PFP reads */
		5061	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
		5062	radeon_ring_write(ring, 0x0);
		5063	}
		5064
5078	serge	5065	/*
		5066	* Power and clock gating
		5067	*/
		5068	static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
3192	Serge	5069	{
5078	serge	5070	int i;
3192	Serge	5071
5078	serge	5072	for (i = 0; i < rdev->usec_timeout; i++) {
		5073	if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
		5074	break;
		5075	udelay(1);
		5076	}
3192	Serge	5077
5078	serge	5078	for (i = 0; i < rdev->usec_timeout; i++) {
		5079	if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
		5080	break;
		5081	udelay(1);
		5082	}
		5083	}
		5084
		5085	static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
		5086	bool enable)
		5087	{
		5088	u32 tmp = RREG32(CP_INT_CNTL_RING0);
		5089	u32 mask;
		5090	int i;
		5091
		5092	if (enable)
		5093	tmp \|= (CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		5094	else
		5095	tmp &= ~(CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		5096	WREG32(CP_INT_CNTL_RING0, tmp);
		5097
		5098	if (!enable) {
		5099	/* read a gfx register */
		5100	tmp = RREG32(DB_DEPTH_INFO);
		5101
		5102	mask = RLC_BUSY_STATUS \| GFX_POWER_STATUS \| GFX_CLOCK_STATUS \| GFX_LS_STATUS;
		5103	for (i = 0; i < rdev->usec_timeout; i++) {
		5104	if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS \| GFX_POWER_STATUS))
		5105	break;
		5106	udelay(1);
		5107	}
		5108	}
		5109	}
		5110
		5111	static void si_set_uvd_dcm(struct radeon_device *rdev,
		5112	bool sw_mode)
		5113	{
		5114	u32 tmp, tmp2;
		5115
		5116	tmp = RREG32(UVD_CGC_CTRL);
		5117	tmp &= ~(CLK_OD_MASK \| CG_DT_MASK);
		5118	tmp \|= DCM \| CG_DT(1) \| CLK_OD(4);
		5119
		5120	if (sw_mode) {
		5121	tmp &= ~0x7ffff800;
		5122	tmp2 = DYN_OR_EN \| DYN_RR_EN \| G_DIV_ID(7);
3192	Serge	5123	} else {
5078	serge	5124	tmp \|= 0x7ffff800;
		5125	tmp2 = 0;
3192	Serge	5126	}
		5127
5078	serge	5128	WREG32(UVD_CGC_CTRL, tmp);
		5129	WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
		5130	}
3192	Serge	5131
5078	serge	5132	void si_init_uvd_internal_cg(struct radeon_device *rdev)
		5133	{
		5134	bool hw_mode = true;
		5135
		5136	if (hw_mode) {
		5137	si_set_uvd_dcm(rdev, false);
		5138	} else {
		5139	u32 tmp = RREG32(UVD_CGC_CTRL);
		5140	tmp &= ~DCM;
		5141	WREG32(UVD_CGC_CTRL, tmp);
		5142	}
3192	Serge	5143	}
		5144
5078	serge	5145	static u32 si_halt_rlc(struct radeon_device *rdev)
2997	Serge	5146	{
5078	serge	5147	u32 data, orig;
2997	Serge	5148
5078	serge	5149	orig = data = RREG32(RLC_CNTL);
2997	Serge	5150
5078	serge	5151	if (data & RLC_ENABLE) {
		5152	data &= ~RLC_ENABLE;
		5153	WREG32(RLC_CNTL, data);
		5154
		5155	si_wait_for_rlc_serdes(rdev);
2997	Serge	5156	}
		5157
5078	serge	5158	return orig;
		5159	}
2997	Serge	5160
5078	serge	5161	static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
		5162	{
		5163	u32 tmp;
		5164
		5165	tmp = RREG32(RLC_CNTL);
		5166	if (tmp != rlc)
		5167	WREG32(RLC_CNTL, rlc);
		5168	}
		5169
		5170	static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
		5171	{
		5172	u32 data, orig;
		5173
		5174	orig = data = RREG32(DMA_PG);
		5175	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
		5176	data \|= PG_CNTL_ENABLE;
		5177	else
		5178	data &= ~PG_CNTL_ENABLE;
		5179	if (orig != data)
		5180	WREG32(DMA_PG, data);
		5181	}
		5182
		5183	static void si_init_dma_pg(struct radeon_device *rdev)
		5184	{
		5185	u32 tmp;
		5186
		5187	WREG32(DMA_PGFSM_WRITE, 0x00002000);
		5188	WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
		5189
		5190	for (tmp = 0; tmp < 5; tmp++)
		5191	WREG32(DMA_PGFSM_WRITE, 0);
		5192	}
		5193
		5194	static void si_enable_gfx_cgpg(struct radeon_device *rdev,
		5195	bool enable)
		5196	{
		5197	u32 tmp;
		5198
		5199	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
		5200	tmp = RLC_PUD(0x10) \| RLC_PDD(0x10) \| RLC_TTPD(0x10) \| RLC_MSD(0x10);
		5201	WREG32(RLC_TTOP_D, tmp);
		5202
		5203	tmp = RREG32(RLC_PG_CNTL);
		5204	tmp \|= GFX_PG_ENABLE;
		5205	WREG32(RLC_PG_CNTL, tmp);
		5206
		5207	tmp = RREG32(RLC_AUTO_PG_CTRL);
		5208	tmp \|= AUTO_PG_EN;
		5209	WREG32(RLC_AUTO_PG_CTRL, tmp);
		5210	} else {
		5211	tmp = RREG32(RLC_AUTO_PG_CTRL);
		5212	tmp &= ~AUTO_PG_EN;
		5213	WREG32(RLC_AUTO_PG_CTRL, tmp);
		5214
		5215	tmp = RREG32(DB_RENDER_CONTROL);
2997	Serge	5216	}
		5217	}
		5218
5078	serge	5219	static void si_init_gfx_cgpg(struct radeon_device *rdev)
2997	Serge	5220	{
5078	serge	5221	u32 tmp;
2997	Serge	5222
5078	serge	5223	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		5224
		5225	tmp = RREG32(RLC_PG_CNTL);
		5226	tmp \|= GFX_PG_SRC;
		5227	WREG32(RLC_PG_CNTL, tmp);
		5228
		5229	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		5230
		5231	tmp = RREG32(RLC_AUTO_PG_CTRL);
		5232
		5233	tmp &= ~GRBM_REG_SGIT_MASK;
		5234	tmp \|= GRBM_REG_SGIT(0x700);
		5235	tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
		5236	WREG32(RLC_AUTO_PG_CTRL, tmp);
		5237	}
		5238
		5239	static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
		5240	{
		5241	u32 mask = 0, tmp, tmp1;
		5242	int i;
		5243
		5244	si_select_se_sh(rdev, se, sh);
		5245	tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
		5246	tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
		5247	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		5248
		5249	tmp &= 0xffff0000;
		5250
		5251	tmp \|= tmp1;
		5252	tmp >>= 16;
		5253
		5254	for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
		5255	mask <<= 1;
		5256	mask \|= 1;
		5257	}
		5258
		5259	return (~tmp) & mask;
		5260	}
		5261
		5262	static void si_init_ao_cu_mask(struct radeon_device *rdev)
		5263	{
		5264	u32 i, j, k, active_cu_number = 0;
		5265	u32 mask, counter, cu_bitmap;
		5266	u32 tmp = 0;
		5267
		5268	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
		5269	for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
		5270	mask = 1;
		5271	cu_bitmap = 0;
		5272	counter = 0;
		5273	for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
		5274	if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
		5275	if (counter < 2)
		5276	cu_bitmap \|= mask;
		5277	counter++;
		5278	}
		5279	mask <<= 1;
		5280	}
		5281
		5282	active_cu_number += counter;
		5283	tmp \|= (cu_bitmap << (i * 16 + j * 8));
2997	Serge	5284	}
		5285	}
		5286
5078	serge	5287	WREG32(RLC_PG_AO_CU_MASK, tmp);
		5288
		5289	tmp = RREG32(RLC_MAX_PG_CU);
		5290	tmp &= ~MAX_PU_CU_MASK;
		5291	tmp \|= MAX_PU_CU(active_cu_number);
		5292	WREG32(RLC_MAX_PG_CU, tmp);
		5293	}
		5294
		5295	static void si_enable_cgcg(struct radeon_device *rdev,
		5296	bool enable)
		5297	{
		5298	u32 data, orig, tmp;
		5299
		5300	orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
		5301
		5302	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
		5303	si_enable_gui_idle_interrupt(rdev, true);
		5304
		5305	WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
		5306
		5307	tmp = si_halt_rlc(rdev);
		5308
		5309	WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
		5310	WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
		5311	WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
		5312
		5313	si_wait_for_rlc_serdes(rdev);
		5314
		5315	si_update_rlc(rdev, tmp);
		5316
		5317	WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
		5318
		5319	data \|= CGCG_EN \| CGLS_EN;
		5320	} else {
		5321	si_enable_gui_idle_interrupt(rdev, false);
		5322
		5323	RREG32(CB_CGTT_SCLK_CTRL);
		5324	RREG32(CB_CGTT_SCLK_CTRL);
		5325	RREG32(CB_CGTT_SCLK_CTRL);
		5326	RREG32(CB_CGTT_SCLK_CTRL);
		5327
		5328	data &= ~(CGCG_EN \| CGLS_EN);
2997	Serge	5329	}
5078	serge	5330
		5331	if (orig != data)
		5332	WREG32(RLC_CGCG_CGLS_CTRL, data);
		5333	}
		5334
		5335	static void si_enable_mgcg(struct radeon_device *rdev,
		5336	bool enable)
		5337	{
		5338	u32 data, orig, tmp = 0;
		5339
		5340	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
		5341	orig = data = RREG32(CGTS_SM_CTRL_REG);
		5342	data = 0x96940200;
		5343	if (orig != data)
		5344	WREG32(CGTS_SM_CTRL_REG, data);
		5345
		5346	if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
		5347	orig = data = RREG32(CP_MEM_SLP_CNTL);
		5348	data \|= CP_MEM_LS_EN;
		5349	if (orig != data)
		5350	WREG32(CP_MEM_SLP_CNTL, data);
		5351	}
		5352
		5353	orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
		5354	data &= 0xffffffc0;
		5355	if (orig != data)
		5356	WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
		5357
		5358	tmp = si_halt_rlc(rdev);
		5359
		5360	WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
		5361	WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
		5362	WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
		5363
		5364	si_update_rlc(rdev, tmp);
		5365	} else {
		5366	orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
		5367	data \|= 0x00000003;
		5368	if (orig != data)
		5369	WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
		5370
		5371	data = RREG32(CP_MEM_SLP_CNTL);
		5372	if (data & CP_MEM_LS_EN) {
		5373	data &= ~CP_MEM_LS_EN;
		5374	WREG32(CP_MEM_SLP_CNTL, data);
		5375	}
		5376	orig = data = RREG32(CGTS_SM_CTRL_REG);
		5377	data \|= LS_OVERRIDE \| OVERRIDE;
		5378	if (orig != data)
		5379	WREG32(CGTS_SM_CTRL_REG, data);
		5380
		5381	tmp = si_halt_rlc(rdev);
		5382
		5383	WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
		5384	WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
		5385	WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
		5386
		5387	si_update_rlc(rdev, tmp);
2997	Serge	5388	}
5078	serge	5389	}
2997	Serge	5390
5078	serge	5391	static void si_enable_uvd_mgcg(struct radeon_device *rdev,
		5392	bool enable)
		5393	{
		5394	u32 orig, data, tmp;
		5395
		5396	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
		5397	tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
		5398	tmp \|= 0x3fff;
		5399	WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
		5400
		5401	orig = data = RREG32(UVD_CGC_CTRL);
		5402	data \|= DCM;
		5403	if (orig != data)
		5404	WREG32(UVD_CGC_CTRL, data);
		5405
		5406	WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
		5407	WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
		5408	} else {
		5409	tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
		5410	tmp &= ~0x3fff;
		5411	WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
		5412
		5413	orig = data = RREG32(UVD_CGC_CTRL);
		5414	data &= ~DCM;
		5415	if (orig != data)
		5416	WREG32(UVD_CGC_CTRL, data);
		5417
		5418	WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
		5419	WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
		5420	}
		5421	}
		5422
		5423	static const u32 mc_cg_registers[] =
		5424	{
		5425	MC_HUB_MISC_HUB_CG,
		5426	MC_HUB_MISC_SIP_CG,
		5427	MC_HUB_MISC_VM_CG,
		5428	MC_XPB_CLK_GAT,
		5429	ATC_MISC_CG,
		5430	MC_CITF_MISC_WR_CG,
		5431	MC_CITF_MISC_RD_CG,
		5432	MC_CITF_MISC_VM_CG,
		5433	VM_L2_CG,
		5434	};
		5435
		5436	static void si_enable_mc_ls(struct radeon_device *rdev,
		5437	bool enable)
		5438	{
		5439	int i;
		5440	u32 orig, data;
		5441
		5442	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
		5443	orig = data = RREG32(mc_cg_registers[i]);
		5444	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
		5445	data \|= MC_LS_ENABLE;
		5446	else
		5447	data &= ~MC_LS_ENABLE;
		5448	if (data != orig)
		5449	WREG32(mc_cg_registers[i], data);
		5450	}
		5451	}
		5452
		5453	static void si_enable_mc_mgcg(struct radeon_device *rdev,
		5454	bool enable)
		5455	{
		5456	int i;
		5457	u32 orig, data;
		5458
		5459	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
		5460	orig = data = RREG32(mc_cg_registers[i]);
		5461	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
		5462	data \|= MC_CG_ENABLE;
		5463	else
		5464	data &= ~MC_CG_ENABLE;
		5465	if (data != orig)
		5466	WREG32(mc_cg_registers[i], data);
		5467	}
		5468	}
		5469
		5470	static void si_enable_dma_mgcg(struct radeon_device *rdev,
		5471	bool enable)
		5472	{
		5473	u32 orig, data, offset;
		5474	int i;
		5475
		5476	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
		5477	for (i = 0; i < 2; i++) {
		5478	if (i == 0)
		5479	offset = DMA0_REGISTER_OFFSET;
		5480	else
		5481	offset = DMA1_REGISTER_OFFSET;
		5482	orig = data = RREG32(DMA_POWER_CNTL + offset);
		5483	data &= ~MEM_POWER_OVERRIDE;
		5484	if (data != orig)
		5485	WREG32(DMA_POWER_CNTL + offset, data);
		5486	WREG32(DMA_CLK_CTRL + offset, 0x00000100);
2997	Serge	5487	}
5078	serge	5488	} else {
		5489	for (i = 0; i < 2; i++) {
		5490	if (i == 0)
		5491	offset = DMA0_REGISTER_OFFSET;
		5492	else
		5493	offset = DMA1_REGISTER_OFFSET;
		5494	orig = data = RREG32(DMA_POWER_CNTL + offset);
		5495	data \|= MEM_POWER_OVERRIDE;
		5496	if (data != orig)
		5497	WREG32(DMA_POWER_CNTL + offset, data);
		5498
		5499	orig = data = RREG32(DMA_CLK_CTRL + offset);
		5500	data = 0xff000000;
		5501	if (data != orig)
		5502	WREG32(DMA_CLK_CTRL + offset, data);
		5503	}
2997	Serge	5504	}
5078	serge	5505	}
		5506
		5507	static void si_enable_bif_mgls(struct radeon_device *rdev,
		5508	bool enable)
		5509	{
		5510	u32 orig, data;
		5511
		5512	orig = data = RREG32_PCIE(PCIE_CNTL2);
		5513
		5514	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
		5515	data \|= SLV_MEM_LS_EN \| MST_MEM_LS_EN \|
		5516	REPLAY_MEM_LS_EN \| SLV_MEM_AGGRESSIVE_LS_EN;
		5517	else
		5518	data &= ~(SLV_MEM_LS_EN \| MST_MEM_LS_EN \|
		5519	REPLAY_MEM_LS_EN \| SLV_MEM_AGGRESSIVE_LS_EN);
		5520
		5521	if (orig != data)
		5522	WREG32_PCIE(PCIE_CNTL2, data);
		5523	}
		5524
		5525	static void si_enable_hdp_mgcg(struct radeon_device *rdev,
		5526	bool enable)
		5527	{
		5528	u32 orig, data;
		5529
		5530	orig = data = RREG32(HDP_HOST_PATH_CNTL);
		5531
		5532	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
		5533	data &= ~CLOCK_GATING_DIS;
		5534	else
		5535	data \|= CLOCK_GATING_DIS;
		5536
		5537	if (orig != data)
		5538	WREG32(HDP_HOST_PATH_CNTL, data);
		5539	}
		5540
		5541	static void si_enable_hdp_ls(struct radeon_device *rdev,
		5542	bool enable)
		5543	{
		5544	u32 orig, data;
		5545
		5546	orig = data = RREG32(HDP_MEM_POWER_LS);
		5547
		5548	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
		5549	data \|= HDP_LS_ENABLE;
		5550	else
		5551	data &= ~HDP_LS_ENABLE;
		5552
		5553	if (orig != data)
		5554	WREG32(HDP_MEM_POWER_LS, data);
		5555	}
		5556
		5557	static void si_update_cg(struct radeon_device *rdev,
6104	serge	5558	u32 block, bool enable)
5078	serge	5559	{
		5560	if (block & RADEON_CG_BLOCK_GFX) {
		5561	si_enable_gui_idle_interrupt(rdev, false);
		5562	/* order matters! */
		5563	if (enable) {
		5564	si_enable_mgcg(rdev, true);
		5565	si_enable_cgcg(rdev, true);
		5566	} else {
		5567	si_enable_cgcg(rdev, false);
		5568	si_enable_mgcg(rdev, false);
		5569	}
		5570	si_enable_gui_idle_interrupt(rdev, true);
2997	Serge	5571	}
5078	serge	5572
		5573	if (block & RADEON_CG_BLOCK_MC) {
		5574	si_enable_mc_mgcg(rdev, enable);
		5575	si_enable_mc_ls(rdev, enable);
2997	Serge	5576	}
		5577
5078	serge	5578	if (block & RADEON_CG_BLOCK_SDMA) {
		5579	si_enable_dma_mgcg(rdev, enable);
		5580	}
		5581
		5582	if (block & RADEON_CG_BLOCK_BIF) {
		5583	si_enable_bif_mgls(rdev, enable);
		5584	}
		5585
		5586	if (block & RADEON_CG_BLOCK_UVD) {
		5587	if (rdev->has_uvd) {
		5588	si_enable_uvd_mgcg(rdev, enable);
		5589	}
		5590	}
		5591
		5592	if (block & RADEON_CG_BLOCK_HDP) {
		5593	si_enable_hdp_mgcg(rdev, enable);
		5594	si_enable_hdp_ls(rdev, enable);
		5595	}
		5596	}
		5597
		5598	static void si_init_cg(struct radeon_device *rdev)
		5599	{
		5600	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX \|
		5601	RADEON_CG_BLOCK_MC \|
		5602	RADEON_CG_BLOCK_SDMA \|
		5603	RADEON_CG_BLOCK_BIF \|
		5604	RADEON_CG_BLOCK_HDP), true);
		5605	if (rdev->has_uvd) {
		5606	si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
		5607	si_init_uvd_internal_cg(rdev);
6104	serge	5608	}
5078	serge	5609	}
		5610
		5611	static void si_fini_cg(struct radeon_device *rdev)
		5612	{
		5613	if (rdev->has_uvd) {
		5614	si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
		5615	}
		5616	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX \|
		5617	RADEON_CG_BLOCK_MC \|
		5618	RADEON_CG_BLOCK_SDMA \|
		5619	RADEON_CG_BLOCK_BIF \|
		5620	RADEON_CG_BLOCK_HDP), false);
		5621	}
		5622
		5623	u32 si_get_csb_size(struct radeon_device *rdev)
		5624	{
		5625	u32 count = 0;
		5626	const struct cs_section_def *sect = NULL;
		5627	const struct cs_extent_def *ext = NULL;
		5628
		5629	if (rdev->rlc.cs_data == NULL)
6104	serge	5630	return 0;
5078	serge	5631
		5632	/* begin clear state */
		5633	count += 2;
		5634	/* context control state */
		5635	count += 3;
		5636
		5637	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
		5638	for (ext = sect->section; ext->extent != NULL; ++ext) {
		5639	if (sect->id == SECT_CONTEXT)
		5640	count += 2 + ext->reg_count;
		5641	else
		5642	return 0;
		5643	}
		5644	}
		5645	/* pa_sc_raster_config */
		5646	count += 3;
		5647	/* end clear state */
		5648	count += 2;
		5649	/* clear state */
		5650	count += 2;
		5651
		5652	return count;
2997	Serge	5653	}
		5654
5078	serge	5655	void si_get_csb_buffer(struct radeon_device rdev, volatile u32 buffer)
		5656	{
		5657	u32 count = 0, i;
		5658	const struct cs_section_def *sect = NULL;
		5659	const struct cs_extent_def *ext = NULL;
		5660
		5661	if (rdev->rlc.cs_data == NULL)
		5662	return;
		5663	if (buffer == NULL)
		5664	return;
		5665
		5666	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		5667	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
		5668
		5669	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
		5670	buffer[count++] = cpu_to_le32(0x80000000);
		5671	buffer[count++] = cpu_to_le32(0x80000000);
		5672
		5673	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
		5674	for (ext = sect->section; ext->extent != NULL; ++ext) {
		5675	if (sect->id == SECT_CONTEXT) {
		5676	buffer[count++] =
		5677	cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
		5678	buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
		5679	for (i = 0; i < ext->reg_count; i++)
		5680	buffer[count++] = cpu_to_le32(ext->extent[i]);
		5681	} else {
		5682	return;
		5683	}
		5684	}
		5685	}
		5686
		5687	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
		5688	buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
		5689	switch (rdev->family) {
		5690	case CHIP_TAHITI:
		5691	case CHIP_PITCAIRN:
		5692	buffer[count++] = cpu_to_le32(0x2a00126a);
		5693	break;
		5694	case CHIP_VERDE:
		5695	buffer[count++] = cpu_to_le32(0x0000124a);
		5696	break;
		5697	case CHIP_OLAND:
		5698	buffer[count++] = cpu_to_le32(0x00000082);
		5699	break;
		5700	case CHIP_HAINAN:
		5701	buffer[count++] = cpu_to_le32(0x00000000);
		5702	break;
		5703	default:
		5704	buffer[count++] = cpu_to_le32(0x00000000);
		5705	break;
		5706	}
		5707
		5708	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		5709	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
		5710
		5711	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
		5712	buffer[count++] = cpu_to_le32(0);
		5713	}
		5714
		5715	static void si_init_pg(struct radeon_device *rdev)
		5716	{
		5717	if (rdev->pg_flags) {
		5718	if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
		5719	si_init_dma_pg(rdev);
		5720	}
		5721	si_init_ao_cu_mask(rdev);
		5722	if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
		5723	si_init_gfx_cgpg(rdev);
		5724	} else {
		5725	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		5726	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		5727	}
		5728	si_enable_dma_pg(rdev, true);
		5729	si_enable_gfx_cgpg(rdev, true);
		5730	} else {
		5731	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		5732	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		5733	}
		5734	}
		5735
		5736	static void si_fini_pg(struct radeon_device *rdev)
		5737	{
		5738	if (rdev->pg_flags) {
		5739	si_enable_dma_pg(rdev, false);
		5740	si_enable_gfx_cgpg(rdev, false);
		5741	}
		5742	}
		5743
		5744	/*
		5745	* RLC
		5746	*/
		5747	void si_rlc_reset(struct radeon_device *rdev)
		5748	{
		5749	u32 tmp = RREG32(GRBM_SOFT_RESET);
		5750
		5751	tmp \|= SOFT_RESET_RLC;
		5752	WREG32(GRBM_SOFT_RESET, tmp);
		5753	udelay(50);
		5754	tmp &= ~SOFT_RESET_RLC;
		5755	WREG32(GRBM_SOFT_RESET, tmp);
		5756	udelay(50);
		5757	}
		5758
2997	Serge	5759	static void si_rlc_stop(struct radeon_device *rdev)
		5760	{
		5761	WREG32(RLC_CNTL, 0);
5078	serge	5762
		5763	si_enable_gui_idle_interrupt(rdev, false);
		5764
		5765	si_wait_for_rlc_serdes(rdev);
2997	Serge	5766	}
		5767
		5768	static void si_rlc_start(struct radeon_device *rdev)
		5769	{
		5770	WREG32(RLC_CNTL, RLC_ENABLE);
5078	serge	5771
		5772	si_enable_gui_idle_interrupt(rdev, true);
		5773
		5774	udelay(50);
2997	Serge	5775	}
		5776
5078	serge	5777	static bool si_lbpw_supported(struct radeon_device *rdev)
		5778	{
		5779	u32 tmp;
		5780
		5781	/* Enable LBPW only for DDR3 */
		5782	tmp = RREG32(MC_SEQ_MISC0);
		5783	if ((tmp & 0xF0000000) == 0xB0000000)
		5784	return true;
		5785	return false;
		5786	}
		5787
		5788	static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
		5789	{
		5790	u32 tmp;
		5791
		5792	tmp = RREG32(RLC_LB_CNTL);
		5793	if (enable)
		5794	tmp \|= LOAD_BALANCE_ENABLE;
		5795	else
		5796	tmp &= ~LOAD_BALANCE_ENABLE;
		5797	WREG32(RLC_LB_CNTL, tmp);
		5798
		5799	if (!enable) {
		5800	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		5801	WREG32(SPI_LB_CU_MASK, 0x00ff);
		5802	}
		5803	}
		5804
2997	Serge	5805	static int si_rlc_resume(struct radeon_device *rdev)
		5806	{
		5807	u32 i;
		5808
		5809	if (!rdev->rlc_fw)
		5810	return -EINVAL;
		5811
		5812	si_rlc_stop(rdev);
		5813
5078	serge	5814	si_rlc_reset(rdev);
		5815
		5816	si_init_pg(rdev);
		5817
		5818	si_init_cg(rdev);
		5819
2997	Serge	5820	WREG32(RLC_RL_BASE, 0);
		5821	WREG32(RLC_RL_SIZE, 0);
		5822	WREG32(RLC_LB_CNTL, 0);
		5823	WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
		5824	WREG32(RLC_LB_CNTR_INIT, 0);
5078	serge	5825	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
2997	Serge	5826
		5827	WREG32(RLC_MC_CNTL, 0);
		5828	WREG32(RLC_UCODE_CNTL, 0);
		5829
5078	serge	5830	if (rdev->new_fw) {
		5831	const struct rlc_firmware_header_v1_0 *hdr =
		5832	(const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
		5833	u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
		5834	const __le32 fw_data = (const __le32 )
		5835	(rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
		5836
		5837	radeon_ucode_print_rlc_hdr(&hdr->header);
		5838
		5839	for (i = 0; i < fw_size; i++) {
		5840	WREG32(RLC_UCODE_ADDR, i);
		5841	WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
		5842	}
		5843	} else {
		5844	const __be32 *fw_data =
		5845	(const __be32 *)rdev->rlc_fw->data;
6104	serge	5846	for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
		5847	WREG32(RLC_UCODE_ADDR, i);
		5848	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		5849	}
2997	Serge	5850	}
		5851	WREG32(RLC_UCODE_ADDR, 0);
		5852
5078	serge	5853	si_enable_lbpw(rdev, si_lbpw_supported(rdev));
		5854
2997	Serge	5855	si_rlc_start(rdev);
		5856
		5857	return 0;
		5858	}
		5859
		5860	static void si_enable_interrupts(struct radeon_device *rdev)
		5861	{
		5862	u32 ih_cntl = RREG32(IH_CNTL);
		5863	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		5864
		5865	ih_cntl \|= ENABLE_INTR;
		5866	ih_rb_cntl \|= IH_RB_ENABLE;
		5867	WREG32(IH_CNTL, ih_cntl);
		5868	WREG32(IH_RB_CNTL, ih_rb_cntl);
		5869	rdev->ih.enabled = true;
		5870	}
		5871
		5872	static void si_disable_interrupts(struct radeon_device *rdev)
		5873	{
		5874	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		5875	u32 ih_cntl = RREG32(IH_CNTL);
		5876
		5877	ih_rb_cntl &= ~IH_RB_ENABLE;
		5878	ih_cntl &= ~ENABLE_INTR;
		5879	WREG32(IH_RB_CNTL, ih_rb_cntl);
		5880	WREG32(IH_CNTL, ih_cntl);
		5881	/* set rptr, wptr to 0 */
		5882	WREG32(IH_RB_RPTR, 0);
		5883	WREG32(IH_RB_WPTR, 0);
		5884	rdev->ih.enabled = false;
		5885	rdev->ih.rptr = 0;
		5886	}
		5887
		5888	static void si_disable_interrupt_state(struct radeon_device *rdev)
		5889	{
		5890	u32 tmp;
		5891
5078	serge	5892	tmp = RREG32(CP_INT_CNTL_RING0) &
		5893	(CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		5894	WREG32(CP_INT_CNTL_RING0, tmp);
2997	Serge	5895	WREG32(CP_INT_CNTL_RING1, 0);
		5896	WREG32(CP_INT_CNTL_RING2, 0);
3192	Serge	5897	tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
		5898	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
		5899	tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
		5900	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
2997	Serge	5901	WREG32(GRBM_INT_CNTL, 0);
6104	serge	5902	WREG32(SRBM_INT_CNTL, 0);
3764	Serge	5903	if (rdev->num_crtc >= 2) {
6104	serge	5904	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		5905	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
3764	Serge	5906	}
2997	Serge	5907	if (rdev->num_crtc >= 4) {
		5908	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		5909	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		5910	}
		5911	if (rdev->num_crtc >= 6) {
		5912	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		5913	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		5914	}
		5915
3764	Serge	5916	if (rdev->num_crtc >= 2) {
6104	serge	5917	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		5918	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
3764	Serge	5919	}
2997	Serge	5920	if (rdev->num_crtc >= 4) {
		5921	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		5922	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		5923	}
		5924	if (rdev->num_crtc >= 6) {
		5925	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		5926	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		5927	}
		5928
3764	Serge	5929	if (!ASIC_IS_NODCE(rdev)) {
5078	serge	5930	WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
2997	Serge	5931
6104	serge	5932	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5933	WREG32(DC_HPD1_INT_CONTROL, tmp);
		5934	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5935	WREG32(DC_HPD2_INT_CONTROL, tmp);
		5936	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5937	WREG32(DC_HPD3_INT_CONTROL, tmp);
		5938	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5939	WREG32(DC_HPD4_INT_CONTROL, tmp);
		5940	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5941	WREG32(DC_HPD5_INT_CONTROL, tmp);
		5942	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		5943	WREG32(DC_HPD6_INT_CONTROL, tmp);
3764	Serge	5944	}
2997	Serge	5945	}
		5946
		5947	static int si_irq_init(struct radeon_device *rdev)
		5948	{
		5949	int ret = 0;
		5950	int rb_bufsz;
		5951	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
		5952
		5953	/* allocate ring */
		5954	ret = r600_ih_ring_alloc(rdev);
		5955	if (ret)
		5956	return ret;
		5957
		5958	/* disable irqs */
		5959	si_disable_interrupts(rdev);
		5960
		5961	/* init rlc */
		5962	ret = si_rlc_resume(rdev);
		5963	if (ret) {
		5964	r600_ih_ring_fini(rdev);
		5965	return ret;
		5966	}
		5967
		5968	/* setup interrupt control */
		5969	/* set dummy read address to ring address */
		5970	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
		5971	interrupt_cntl = RREG32(INTERRUPT_CNTL);
		5972	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
		5973	* IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
		5974	*/
		5975	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
		5976	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
		5977	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
		5978	WREG32(INTERRUPT_CNTL, interrupt_cntl);
		5979
		5980	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5078	serge	5981	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
2997	Serge	5982
		5983	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE \|
		5984	IH_WPTR_OVERFLOW_CLEAR \|
		5985	(rb_bufsz << 1));
		5986
		5987	if (rdev->wb.enabled)
		5988	ih_rb_cntl \|= IH_WPTR_WRITEBACK_ENABLE;
		5989
		5990	/* set the writeback address whether it's enabled or not */
		5991	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
		5992	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
		5993
		5994	WREG32(IH_RB_CNTL, ih_rb_cntl);
		5995
		5996	/* set rptr, wptr to 0 */
		5997	WREG32(IH_RB_RPTR, 0);
		5998	WREG32(IH_RB_WPTR, 0);
		5999
		6000	/* Default settings for IH_CNTL (disabled at first) */
		6001	ih_cntl = MC_WRREQ_CREDIT(0x10) \| MC_WR_CLEAN_CNT(0x10) \| MC_VMID(0);
		6002	/* RPTR_REARM only works if msi's are enabled */
		6003	if (rdev->msi_enabled)
		6004	ih_cntl \|= RPTR_REARM;
		6005	WREG32(IH_CNTL, ih_cntl);
		6006
		6007	/* force the active interrupt state to all disabled */
		6008	si_disable_interrupt_state(rdev);
		6009
		6010	pci_set_master(rdev->pdev);
		6011
		6012	/* enable irqs */
		6013	si_enable_interrupts(rdev);
		6014
		6015	return ret;
		6016	}
		6017
		6018	int si_irq_set(struct radeon_device *rdev)
		6019	{
5078	serge	6020	u32 cp_int_cntl;
2997	Serge	6021	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
		6022	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
3764	Serge	6023	u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2997	Serge	6024	u32 grbm_int_cntl = 0;
3192	Serge	6025	u32 dma_cntl, dma_cntl1;
5078	serge	6026	u32 thermal_int = 0;
2997	Serge	6027
		6028	if (!rdev->irq.installed) {
		6029	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		6030	return -EINVAL;
		6031	}
		6032	/* don't enable anything if the ih is disabled */
		6033	if (!rdev->ih.enabled) {
		6034	si_disable_interrupts(rdev);
		6035	/* force the active interrupt state to all disabled */
		6036	si_disable_interrupt_state(rdev);
		6037	return 0;
		6038	}
		6039
5078	serge	6040	cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
		6041	(CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		6042
3764	Serge	6043	if (!ASIC_IS_NODCE(rdev)) {
6104	serge	6044	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
		6045	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
		6046	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
		6047	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
		6048	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
		6049	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN);
3764	Serge	6050	}
2997	Serge	6051
3192	Serge	6052	dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
		6053	dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
		6054
5078	serge	6055	thermal_int = RREG32(CG_THERMAL_INT) &
		6056	~(THERM_INT_MASK_HIGH \| THERM_INT_MASK_LOW);
		6057
2997	Serge	6058	/* enable CP interrupts on all rings */
		6059	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		6060	DRM_DEBUG("si_irq_set: sw int gfx\n");
		6061	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		6062	}
		6063	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
		6064	DRM_DEBUG("si_irq_set: sw int cp1\n");
		6065	cp_int_cntl1 \|= TIME_STAMP_INT_ENABLE;
		6066	}
		6067	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
		6068	DRM_DEBUG("si_irq_set: sw int cp2\n");
		6069	cp_int_cntl2 \|= TIME_STAMP_INT_ENABLE;
		6070	}
3192	Serge	6071	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
		6072	DRM_DEBUG("si_irq_set: sw int dma\n");
		6073	dma_cntl \|= TRAP_ENABLE;
		6074	}
		6075
		6076	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
		6077	DRM_DEBUG("si_irq_set: sw int dma1\n");
		6078	dma_cntl1 \|= TRAP_ENABLE;
		6079	}
2997	Serge	6080	if (rdev->irq.crtc_vblank_int[0] \|\|
		6081	atomic_read(&rdev->irq.pflip[0])) {
		6082	DRM_DEBUG("si_irq_set: vblank 0\n");
		6083	crtc1 \|= VBLANK_INT_MASK;
		6084	}
		6085	if (rdev->irq.crtc_vblank_int[1] \|\|
		6086	atomic_read(&rdev->irq.pflip[1])) {
		6087	DRM_DEBUG("si_irq_set: vblank 1\n");
		6088	crtc2 \|= VBLANK_INT_MASK;
		6089	}
		6090	if (rdev->irq.crtc_vblank_int[2] \|\|
		6091	atomic_read(&rdev->irq.pflip[2])) {
		6092	DRM_DEBUG("si_irq_set: vblank 2\n");
		6093	crtc3 \|= VBLANK_INT_MASK;
		6094	}
		6095	if (rdev->irq.crtc_vblank_int[3] \|\|
		6096	atomic_read(&rdev->irq.pflip[3])) {
		6097	DRM_DEBUG("si_irq_set: vblank 3\n");
		6098	crtc4 \|= VBLANK_INT_MASK;
		6099	}
		6100	if (rdev->irq.crtc_vblank_int[4] \|\|
		6101	atomic_read(&rdev->irq.pflip[4])) {
		6102	DRM_DEBUG("si_irq_set: vblank 4\n");
		6103	crtc5 \|= VBLANK_INT_MASK;
		6104	}
		6105	if (rdev->irq.crtc_vblank_int[5] \|\|
		6106	atomic_read(&rdev->irq.pflip[5])) {
		6107	DRM_DEBUG("si_irq_set: vblank 5\n");
		6108	crtc6 \|= VBLANK_INT_MASK;
		6109	}
		6110	if (rdev->irq.hpd[0]) {
		6111	DRM_DEBUG("si_irq_set: hpd 1\n");
6104	serge	6112	hpd1 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6113	}
		6114	if (rdev->irq.hpd[1]) {
		6115	DRM_DEBUG("si_irq_set: hpd 2\n");
6104	serge	6116	hpd2 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6117	}
		6118	if (rdev->irq.hpd[2]) {
		6119	DRM_DEBUG("si_irq_set: hpd 3\n");
6104	serge	6120	hpd3 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6121	}
		6122	if (rdev->irq.hpd[3]) {
		6123	DRM_DEBUG("si_irq_set: hpd 4\n");
6104	serge	6124	hpd4 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6125	}
		6126	if (rdev->irq.hpd[4]) {
		6127	DRM_DEBUG("si_irq_set: hpd 5\n");
6104	serge	6128	hpd5 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6129	}
		6130	if (rdev->irq.hpd[5]) {
		6131	DRM_DEBUG("si_irq_set: hpd 6\n");
6104	serge	6132	hpd6 \|= DC_HPDx_INT_EN \| DC_HPDx_RX_INT_EN;
2997	Serge	6133	}
		6134
		6135	WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
		6136	WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
		6137	WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
		6138
3192	Serge	6139	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
		6140	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
		6141
2997	Serge	6142	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		6143
5078	serge	6144	if (rdev->irq.dpm_thermal) {
		6145	DRM_DEBUG("dpm thermal\n");
		6146	thermal_int \|= THERM_INT_MASK_HIGH \| THERM_INT_MASK_LOW;
		6147	}
		6148
3764	Serge	6149	if (rdev->num_crtc >= 2) {
6104	serge	6150	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
		6151	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
3764	Serge	6152	}
2997	Serge	6153	if (rdev->num_crtc >= 4) {
		6154	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
		6155	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
		6156	}
		6157	if (rdev->num_crtc >= 6) {
		6158	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
		6159	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
		6160	}
		6161
3764	Serge	6162	if (rdev->num_crtc >= 2) {
5078	serge	6163	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
		6164	GRPH_PFLIP_INT_MASK);
		6165	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
		6166	GRPH_PFLIP_INT_MASK);
3764	Serge	6167	}
2997	Serge	6168	if (rdev->num_crtc >= 4) {
5078	serge	6169	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
		6170	GRPH_PFLIP_INT_MASK);
		6171	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
		6172	GRPH_PFLIP_INT_MASK);
2997	Serge	6173	}
		6174	if (rdev->num_crtc >= 6) {
5078	serge	6175	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
		6176	GRPH_PFLIP_INT_MASK);
		6177	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
		6178	GRPH_PFLIP_INT_MASK);
2997	Serge	6179	}
		6180
3764	Serge	6181	if (!ASIC_IS_NODCE(rdev)) {
6104	serge	6182	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		6183	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		6184	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		6185	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		6186	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		6187	WREG32(DC_HPD6_INT_CONTROL, hpd6);
3764	Serge	6188	}
2997	Serge	6189
5078	serge	6190	WREG32(CG_THERMAL_INT, thermal_int);
		6191
6104	serge	6192	/* posting read */
		6193	RREG32(SRBM_STATUS);
		6194
2997	Serge	6195	return 0;
		6196	}
		6197
		6198	static inline void si_irq_ack(struct radeon_device *rdev)
		6199	{
		6200	u32 tmp;
		6201
3764	Serge	6202	if (ASIC_IS_NODCE(rdev))
		6203	return;
		6204
2997	Serge	6205	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		6206	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		6207	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
		6208	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
		6209	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
		6210	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
		6211	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		6212	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		6213	if (rdev->num_crtc >= 4) {
		6214	rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		6215	rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		6216	}
		6217	if (rdev->num_crtc >= 6) {
		6218	rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		6219	rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		6220	}
		6221
		6222	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
		6223	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6224	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
		6225	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6226	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
		6227	WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
		6228	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
		6229	WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
		6230	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
		6231	WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
		6232	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
		6233	WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
		6234
		6235	if (rdev->num_crtc >= 4) {
		6236	if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
		6237	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6238	if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
		6239	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6240	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
		6241	WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
		6242	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
		6243	WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
		6244	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
		6245	WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
		6246	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
		6247	WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
		6248	}
		6249
		6250	if (rdev->num_crtc >= 6) {
		6251	if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
		6252	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6253	if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
		6254	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		6255	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
		6256	WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
		6257	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
		6258	WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
		6259	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
		6260	WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
		6261	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
		6262	WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
		6263	}
		6264
		6265	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		6266	tmp = RREG32(DC_HPD1_INT_CONTROL);
		6267	tmp \|= DC_HPDx_INT_ACK;
		6268	WREG32(DC_HPD1_INT_CONTROL, tmp);
		6269	}
		6270	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		6271	tmp = RREG32(DC_HPD2_INT_CONTROL);
		6272	tmp \|= DC_HPDx_INT_ACK;
		6273	WREG32(DC_HPD2_INT_CONTROL, tmp);
		6274	}
		6275	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		6276	tmp = RREG32(DC_HPD3_INT_CONTROL);
		6277	tmp \|= DC_HPDx_INT_ACK;
		6278	WREG32(DC_HPD3_INT_CONTROL, tmp);
		6279	}
		6280	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		6281	tmp = RREG32(DC_HPD4_INT_CONTROL);
		6282	tmp \|= DC_HPDx_INT_ACK;
		6283	WREG32(DC_HPD4_INT_CONTROL, tmp);
		6284	}
		6285	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		6286	tmp = RREG32(DC_HPD5_INT_CONTROL);
		6287	tmp \|= DC_HPDx_INT_ACK;
		6288	WREG32(DC_HPD5_INT_CONTROL, tmp);
		6289	}
		6290	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		6291	tmp = RREG32(DC_HPD5_INT_CONTROL);
		6292	tmp \|= DC_HPDx_INT_ACK;
		6293	WREG32(DC_HPD6_INT_CONTROL, tmp);
		6294	}
6104	serge	6295
		6296	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
		6297	tmp = RREG32(DC_HPD1_INT_CONTROL);
		6298	tmp \|= DC_HPDx_RX_INT_ACK;
		6299	WREG32(DC_HPD1_INT_CONTROL, tmp);
		6300	}
		6301	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
		6302	tmp = RREG32(DC_HPD2_INT_CONTROL);
		6303	tmp \|= DC_HPDx_RX_INT_ACK;
		6304	WREG32(DC_HPD2_INT_CONTROL, tmp);
		6305	}
		6306	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
		6307	tmp = RREG32(DC_HPD3_INT_CONTROL);
		6308	tmp \|= DC_HPDx_RX_INT_ACK;
		6309	WREG32(DC_HPD3_INT_CONTROL, tmp);
		6310	}
		6311	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
		6312	tmp = RREG32(DC_HPD4_INT_CONTROL);
		6313	tmp \|= DC_HPDx_RX_INT_ACK;
		6314	WREG32(DC_HPD4_INT_CONTROL, tmp);
		6315	}
		6316	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
		6317	tmp = RREG32(DC_HPD5_INT_CONTROL);
		6318	tmp \|= DC_HPDx_RX_INT_ACK;
		6319	WREG32(DC_HPD5_INT_CONTROL, tmp);
		6320	}
		6321	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
		6322	tmp = RREG32(DC_HPD5_INT_CONTROL);
		6323	tmp \|= DC_HPDx_RX_INT_ACK;
		6324	WREG32(DC_HPD6_INT_CONTROL, tmp);
		6325	}
2997	Serge	6326	}
		6327
		6328	static void si_irq_disable(struct radeon_device *rdev)
		6329	{
		6330	si_disable_interrupts(rdev);
		6331	/* Wait and acknowledge irq */
		6332	mdelay(1);
		6333	si_irq_ack(rdev);
		6334	si_disable_interrupt_state(rdev);
		6335	}
		6336
		6337	static void si_irq_suspend(struct radeon_device *rdev)
		6338	{
		6339	si_irq_disable(rdev);
		6340	si_rlc_stop(rdev);
		6341	}
		6342
		6343	static void si_irq_fini(struct radeon_device *rdev)
		6344	{
		6345	si_irq_suspend(rdev);
		6346	r600_ih_ring_fini(rdev);
		6347	}
		6348
		6349	static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
		6350	{
		6351	u32 wptr, tmp;
		6352
		6353	if (rdev->wb.enabled)
		6354	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		6355	else
		6356	wptr = RREG32(IH_RB_WPTR);
		6357
		6358	if (wptr & RB_OVERFLOW) {
5179	serge	6359	wptr &= ~RB_OVERFLOW;
2997	Serge	6360	/* When a ring buffer overflow happen start parsing interrupt
		6361	* from the last not overwritten vector (wptr + 16). Hopefully
		6362	* this should allow us to catchup.
		6363	*/
5179	serge	6364	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
		6365	wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
2997	Serge	6366	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		6367	tmp = RREG32(IH_RB_CNTL);
		6368	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		6369	WREG32(IH_RB_CNTL, tmp);
		6370	}
		6371	return (wptr & rdev->ih.ptr_mask);
		6372	}
		6373
		6374	/* SI IV Ring
		6375	* Each IV ring entry is 128 bits:
		6376	* [7:0] - interrupt source id
		6377	* [31:8] - reserved
		6378	* [59:32] - interrupt source data
		6379	* [63:60] - reserved
		6380	* [71:64] - RINGID
		6381	* [79:72] - VMID
		6382	* [127:80] - reserved
		6383	*/
		6384	int si_irq_process(struct radeon_device *rdev)
		6385	{
		6386	u32 wptr;
		6387	u32 rptr;
		6388	u32 src_id, src_data, ring_id;
		6389	u32 ring_index;
		6390	bool queue_hotplug = false;
6104	serge	6391	bool queue_dp = false;
5078	serge	6392	bool queue_thermal = false;
		6393	u32 status, addr;
2997	Serge	6394
		6395	if (!rdev->ih.enabled \|\| rdev->shutdown)
		6396	return IRQ_NONE;
		6397
		6398	wptr = si_get_ih_wptr(rdev);
		6399
		6400	restart_ih:
		6401	/* is somebody else already processing irqs? */
		6402	if (atomic_xchg(&rdev->ih.lock, 1))
		6403	return IRQ_NONE;
		6404
		6405	rptr = rdev->ih.rptr;
		6406	DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		6407
		6408	/* Order reading of wptr vs. reading of IH ring data */
		6409	rmb();
		6410
		6411	/* display interrupts */
		6412	si_irq_ack(rdev);
		6413
		6414	while (rptr != wptr) {
		6415	/* wptr/rptr are in bytes! */
		6416	ring_index = rptr / 4;
		6417	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		6418	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		6419	ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
		6420
		6421	switch (src_id) {
		6422	case 1: /* D1 vblank/vline */
		6423	switch (src_data) {
		6424	case 0: /* D1 vblank */
6104	serge	6425	if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
		6426	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6427
		6428	if (rdev->irq.crtc_vblank_int[0]) {
		6429	drm_handle_vblank(rdev->ddev, 0);
		6430	rdev->pm.vblank_sync = true;
		6431	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6432	}
6104	serge	6433	if (atomic_read(&rdev->irq.pflip[0]))
		6434	radeon_crtc_handle_vblank(rdev, 0);
		6435	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		6436	DRM_DEBUG("IH: D1 vblank\n");
		6437
2997	Serge	6438	break;
		6439	case 1: /* D1 vline */
6104	serge	6440	if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
		6441	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6442
		6443	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		6444	DRM_DEBUG("IH: D1 vline\n");
		6445
2997	Serge	6446	break;
		6447	default:
		6448	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6449	break;
		6450	}
		6451	break;
		6452	case 2: /* D2 vblank/vline */
		6453	switch (src_data) {
		6454	case 0: /* D2 vblank */
6104	serge	6455	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
		6456	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6457
		6458	if (rdev->irq.crtc_vblank_int[1]) {
		6459	drm_handle_vblank(rdev->ddev, 1);
		6460	rdev->pm.vblank_sync = true;
		6461	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6462	}
6104	serge	6463	if (atomic_read(&rdev->irq.pflip[1]))
		6464	radeon_crtc_handle_vblank(rdev, 1);
		6465	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
		6466	DRM_DEBUG("IH: D2 vblank\n");
		6467
2997	Serge	6468	break;
		6469	case 1: /* D2 vline */
6104	serge	6470	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
		6471	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6472
		6473	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
		6474	DRM_DEBUG("IH: D2 vline\n");
		6475
2997	Serge	6476	break;
		6477	default:
		6478	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6479	break;
		6480	}
		6481	break;
		6482	case 3: /* D3 vblank/vline */
		6483	switch (src_data) {
		6484	case 0: /* D3 vblank */
6104	serge	6485	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
		6486	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6487
		6488	if (rdev->irq.crtc_vblank_int[2]) {
		6489	drm_handle_vblank(rdev->ddev, 2);
		6490	rdev->pm.vblank_sync = true;
		6491	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6492	}
6104	serge	6493	if (atomic_read(&rdev->irq.pflip[2]))
		6494	radeon_crtc_handle_vblank(rdev, 2);
		6495	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
		6496	DRM_DEBUG("IH: D3 vblank\n");
		6497
2997	Serge	6498	break;
		6499	case 1: /* D3 vline */
6104	serge	6500	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
		6501	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6502
		6503	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
		6504	DRM_DEBUG("IH: D3 vline\n");
		6505
2997	Serge	6506	break;
		6507	default:
		6508	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6509	break;
		6510	}
		6511	break;
		6512	case 4: /* D4 vblank/vline */
		6513	switch (src_data) {
		6514	case 0: /* D4 vblank */
6104	serge	6515	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
		6516	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6517
		6518	if (rdev->irq.crtc_vblank_int[3]) {
		6519	drm_handle_vblank(rdev->ddev, 3);
		6520	rdev->pm.vblank_sync = true;
		6521	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6522	}
6104	serge	6523	if (atomic_read(&rdev->irq.pflip[3]))
		6524	radeon_crtc_handle_vblank(rdev, 3);
		6525	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
		6526	DRM_DEBUG("IH: D4 vblank\n");
		6527
2997	Serge	6528	break;
		6529	case 1: /* D4 vline */
6104	serge	6530	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
		6531	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6532
		6533	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
		6534	DRM_DEBUG("IH: D4 vline\n");
		6535
2997	Serge	6536	break;
		6537	default:
		6538	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6539	break;
		6540	}
		6541	break;
		6542	case 5: /* D5 vblank/vline */
		6543	switch (src_data) {
		6544	case 0: /* D5 vblank */
6104	serge	6545	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
		6546	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6547
		6548	if (rdev->irq.crtc_vblank_int[4]) {
		6549	drm_handle_vblank(rdev->ddev, 4);
		6550	rdev->pm.vblank_sync = true;
		6551	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6552	}
6104	serge	6553	if (atomic_read(&rdev->irq.pflip[4]))
		6554	radeon_crtc_handle_vblank(rdev, 4);
		6555	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
		6556	DRM_DEBUG("IH: D5 vblank\n");
		6557
2997	Serge	6558	break;
		6559	case 1: /* D5 vline */
6104	serge	6560	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
		6561	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6562
		6563	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
		6564	DRM_DEBUG("IH: D5 vline\n");
		6565
2997	Serge	6566	break;
		6567	default:
		6568	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6569	break;
		6570	}
		6571	break;
		6572	case 6: /* D6 vblank/vline */
		6573	switch (src_data) {
		6574	case 0: /* D6 vblank */
6104	serge	6575	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
		6576	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6577
		6578	if (rdev->irq.crtc_vblank_int[5]) {
		6579	drm_handle_vblank(rdev->ddev, 5);
		6580	rdev->pm.vblank_sync = true;
		6581	wake_up(&rdev->irq.vblank_queue);
2997	Serge	6582	}
6104	serge	6583	if (atomic_read(&rdev->irq.pflip[5]))
		6584	radeon_crtc_handle_vblank(rdev, 5);
		6585	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
		6586	DRM_DEBUG("IH: D6 vblank\n");
		6587
2997	Serge	6588	break;
		6589	case 1: /* D6 vline */
6104	serge	6590	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
		6591	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6592
		6593	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
		6594	DRM_DEBUG("IH: D6 vline\n");
		6595
2997	Serge	6596	break;
		6597	default:
		6598	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6599	break;
		6600	}
		6601	break;
5078	serge	6602	case 8: /* D1 page flip */
		6603	case 10: /* D2 page flip */
		6604	case 12: /* D3 page flip */
		6605	case 14: /* D4 page flip */
		6606	case 16: /* D5 page flip */
		6607	case 18: /* D6 page flip */
		6608	DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
		6609	break;
2997	Serge	6610	case 42: /* HPD hotplug */
		6611	switch (src_data) {
		6612	case 0:
6104	serge	6613	if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
		6614	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6615
		6616	rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
		6617	queue_hotplug = true;
		6618	DRM_DEBUG("IH: HPD1\n");
		6619
2997	Serge	6620	break;
		6621	case 1:
6104	serge	6622	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
		6623	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6624
		6625	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
		6626	queue_hotplug = true;
		6627	DRM_DEBUG("IH: HPD2\n");
		6628
2997	Serge	6629	break;
		6630	case 2:
6104	serge	6631	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
		6632	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6633
		6634	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
		6635	queue_hotplug = true;
		6636	DRM_DEBUG("IH: HPD3\n");
		6637
2997	Serge	6638	break;
		6639	case 3:
6104	serge	6640	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
		6641	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6642
		6643	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
		6644	queue_hotplug = true;
		6645	DRM_DEBUG("IH: HPD4\n");
		6646
2997	Serge	6647	break;
		6648	case 4:
6104	serge	6649	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
		6650	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6651
		6652	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
		6653	queue_hotplug = true;
		6654	DRM_DEBUG("IH: HPD5\n");
		6655
2997	Serge	6656	break;
		6657	case 5:
6104	serge	6658	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
		6659	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6660
		6661	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
		6662	queue_hotplug = true;
		6663	DRM_DEBUG("IH: HPD6\n");
		6664
2997	Serge	6665	break;
6104	serge	6666	case 6:
		6667	if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
		6668	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6669
		6670	rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
		6671	queue_dp = true;
		6672	DRM_DEBUG("IH: HPD_RX 1\n");
		6673
		6674	break;
		6675	case 7:
		6676	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
		6677	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6678
		6679	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
		6680	queue_dp = true;
		6681	DRM_DEBUG("IH: HPD_RX 2\n");
		6682
		6683	break;
		6684	case 8:
		6685	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
		6686	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6687
		6688	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
		6689	queue_dp = true;
		6690	DRM_DEBUG("IH: HPD_RX 3\n");
		6691
		6692	break;
		6693	case 9:
		6694	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
		6695	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6696
		6697	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
		6698	queue_dp = true;
		6699	DRM_DEBUG("IH: HPD_RX 4\n");
		6700
		6701	break;
		6702	case 10:
		6703	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
		6704	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6705
		6706	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
		6707	queue_dp = true;
		6708	DRM_DEBUG("IH: HPD_RX 5\n");
		6709
		6710	break;
		6711	case 11:
		6712	if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
		6713	DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
		6714
		6715	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
		6716	queue_dp = true;
		6717	DRM_DEBUG("IH: HPD_RX 6\n");
		6718
		6719	break;
2997	Serge	6720	default:
		6721	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6722	break;
		6723	}
		6724	break;
6104	serge	6725	case 96:
		6726	DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
		6727	WREG32(SRBM_INT_ACK, 0x1);
		6728	break;
5078	serge	6729	case 124: /* UVD */
		6730	DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
		6731	radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
		6732	break;
3192	Serge	6733	case 146:
		6734	case 147:
5078	serge	6735	addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
		6736	status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
		6737	/* reset addr and status */
		6738	WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
		6739	if (addr == 0x0 && status == 0x0)
		6740	break;
3192	Serge	6741	dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
		6742	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5078	serge	6743	addr);
3192	Serge	6744	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5078	serge	6745	status);
		6746	si_vm_decode_fault(rdev, status, addr);
3192	Serge	6747	break;
2997	Serge	6748	case 176: /* RINGID0 CP_INT */
		6749	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
		6750	break;
		6751	case 177: /* RINGID1 CP_INT */
		6752	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		6753	break;
		6754	case 178: /* RINGID2 CP_INT */
		6755	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		6756	break;
		6757	case 181: /* CP EOP event */
		6758	DRM_DEBUG("IH: CP EOP\n");
		6759	switch (ring_id) {
		6760	case 0:
		6761	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
		6762	break;
		6763	case 1:
		6764	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		6765	break;
		6766	case 2:
		6767	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		6768	break;
		6769	}
		6770	break;
3192	Serge	6771	case 224: /* DMA trap event */
		6772	DRM_DEBUG("IH: DMA trap\n");
		6773	radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
		6774	break;
5078	serge	6775	case 230: /* thermal low to high */
		6776	DRM_DEBUG("IH: thermal low to high\n");
		6777	rdev->pm.dpm.thermal.high_to_low = false;
		6778	queue_thermal = true;
		6779	break;
		6780	case 231: /* thermal high to low */
		6781	DRM_DEBUG("IH: thermal high to low\n");
		6782	rdev->pm.dpm.thermal.high_to_low = true;
		6783	queue_thermal = true;
		6784	break;
2997	Serge	6785	case 233: /* GUI IDLE */
		6786	DRM_DEBUG("IH: GUI idle\n");
		6787	break;
3192	Serge	6788	case 244: /* DMA trap event */
		6789	DRM_DEBUG("IH: DMA1 trap\n");
		6790	radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
		6791	break;
2997	Serge	6792	default:
		6793	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		6794	break;
		6795	}
		6796
		6797	/* wptr/rptr are in bytes! */
		6798	rptr += 16;
		6799	rptr &= rdev->ih.ptr_mask;
5179	serge	6800	WREG32(IH_RB_RPTR, rptr);
2997	Serge	6801	}
		6802	// if (queue_hotplug)
		6803	// schedule_work(&rdev->hotplug_work);
		6804	rdev->ih.rptr = rptr;
		6805	atomic_set(&rdev->ih.lock, 0);
		6806
		6807	/* make sure wptr hasn't changed while processing */
		6808	wptr = si_get_ih_wptr(rdev);
		6809	if (wptr != rptr)
		6810	goto restart_ih;
		6811
		6812	return IRQ_HANDLED;
		6813	}
		6814
		6815	/*
		6816	* startup/shutdown callbacks
		6817	*/
		6818	static int si_startup(struct radeon_device *rdev)
		6819	{
		6820	struct radeon_ring *ring;
		6821	int r;
		6822
5078	serge	6823	/* enable pcie gen2/3 link */
		6824	si_pcie_gen3_enable(rdev);
		6825	/* enable aspm */
		6826	si_program_aspm(rdev);
2997	Serge	6827
5078	serge	6828	/* scratch needs to be initialized before MC */
2997	Serge	6829	r = r600_vram_scratch_init(rdev);
		6830	if (r)
		6831	return r;
		6832
		6833	si_mc_program(rdev);
5078	serge	6834
		6835	if (!rdev->pm.dpm_enabled) {
		6836	r = si_mc_load_microcode(rdev);
		6837	if (r) {
		6838	DRM_ERROR("Failed to load MC firmware!\n");
		6839	return r;
		6840	}
		6841	}
		6842
2997	Serge	6843	r = si_pcie_gart_enable(rdev);
		6844	if (r)
		6845	return r;
		6846	si_gpu_init(rdev);
		6847
		6848	/* allocate rlc buffers */
5078	serge	6849	if (rdev->family == CHIP_VERDE) {
		6850	rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
		6851	rdev->rlc.reg_list_size =
		6852	(u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
		6853	}
		6854	rdev->rlc.cs_data = si_cs_data;
		6855	r = sumo_rlc_init(rdev);
2997	Serge	6856	if (r) {
		6857	DRM_ERROR("Failed to init rlc BOs!\n");
		6858	return r;
		6859	}
		6860
		6861	/* allocate wb buffer */
		6862	r = radeon_wb_init(rdev);
		6863	if (r)
		6864	return r;
		6865
		6866	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
		6867	if (r) {
		6868	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		6869	return r;
		6870	}
		6871
		6872	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		6873	if (r) {
		6874	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		6875	return r;
		6876	}
		6877
		6878	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		6879	if (r) {
		6880	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		6881	return r;
		6882	}
		6883
3192	Serge	6884	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
		6885	if (r) {
		6886	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		6887	return r;
		6888	}
		6889
		6890	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
		6891	if (r) {
		6892	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		6893	return r;
		6894	}
		6895
5078	serge	6896	if (rdev->has_uvd) {
		6897	r = uvd_v2_2_resume(rdev);
		6898	if (!r) {
		6899	r = radeon_fence_driver_start_ring(rdev,
		6900	R600_RING_TYPE_UVD_INDEX);
		6901	if (r)
		6902	dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
		6903	}
		6904	if (r)
		6905	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
		6906	}
3764	Serge	6907
6104	serge	6908	r = radeon_vce_resume(rdev);
		6909	if (!r) {
		6910	r = vce_v1_0_resume(rdev);
		6911	if (!r)
		6912	r = radeon_fence_driver_start_ring(rdev,
		6913	TN_RING_TYPE_VCE1_INDEX);
		6914	if (!r)
		6915	r = radeon_fence_driver_start_ring(rdev,
		6916	TN_RING_TYPE_VCE2_INDEX);
		6917	}
		6918	if (r) {
		6919	dev_err(rdev->dev, "VCE init error (%d).\n", r);
		6920	rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
		6921	rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
		6922	}
		6923
2997	Serge	6924	/* Enable IRQ */
3764	Serge	6925	if (!rdev->irq.installed) {
		6926	r = radeon_irq_kms_init(rdev);
		6927	if (r)
		6928	return r;
		6929	}
		6930
2997	Serge	6931	r = si_irq_init(rdev);
		6932	if (r) {
		6933	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		6934	radeon_irq_kms_fini(rdev);
		6935	return r;
		6936	}
		6937	si_irq_set(rdev);
		6938
		6939	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		6940	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5078	serge	6941	RADEON_CP_PACKET2);
2997	Serge	6942	if (r)
		6943	return r;
		6944
		6945	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
		6946	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
5078	serge	6947	RADEON_CP_PACKET2);
2997	Serge	6948	if (r)
		6949	return r;
		6950
		6951	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
		6952	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
5078	serge	6953	RADEON_CP_PACKET2);
2997	Serge	6954	if (r)
		6955	return r;
		6956
3192	Serge	6957	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		6958	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5078	serge	6959	DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
3192	Serge	6960	if (r)
		6961	return r;
		6962
		6963	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		6964	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5078	serge	6965	DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
3192	Serge	6966	if (r)
		6967	return r;
		6968
2997	Serge	6969	r = si_cp_load_microcode(rdev);
		6970	if (r)
		6971	return r;
		6972	r = si_cp_resume(rdev);
		6973	if (r)
		6974	return r;
		6975
3192	Serge	6976	r = cayman_dma_resume(rdev);
		6977	if (r)
		6978	return r;
		6979
5078	serge	6980	if (rdev->has_uvd) {
		6981	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
		6982	if (ring->ring_size) {
		6983	r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
		6984	RADEON_CP_PACKET2);
		6985	if (!r)
		6986	r = uvd_v1_0_init(rdev);
		6987	if (r)
		6988	DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
		6989	}
		6990	}
3764	Serge	6991
6104	serge	6992	r = -ENOENT;
		6993
		6994	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
		6995	if (ring->ring_size)
		6996	r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
		6997	VCE_CMD_NO_OP);
		6998
		6999	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
		7000	if (ring->ring_size)
		7001	r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
		7002	VCE_CMD_NO_OP);
		7003
		7004	if (!r)
		7005	r = vce_v1_0_init(rdev);
		7006	else if (r != -ENOENT)
		7007	DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
		7008
3120	serge	7009	r = radeon_ib_pool_init(rdev);
		7010	if (r) {
		7011	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		7012	return r;
		7013	}
2997	Serge	7014
3764	Serge	7015	r = radeon_vm_manager_init(rdev);
		7016	if (r) {
		7017	dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
		7018	return r;
		7019	}
2997	Serge	7020
6104	serge	7021	r = radeon_audio_init(rdev);
5078	serge	7022	if (r)
		7023	return r;
		7024
2997	Serge	7025	return 0;
		7026	}
		7027
6104	serge	7028	int si_resume(struct radeon_device *rdev)
		7029	{
		7030	int r;
2997	Serge	7031
6104	serge	7032	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
		7033	* posting will perform necessary task to bring back GPU into good
		7034	* shape.
		7035	*/
		7036	/* post card */
		7037	atom_asic_init(rdev->mode_info.atom_context);
3120	serge	7038
6104	serge	7039	/* init golden registers */
		7040	si_init_golden_registers(rdev);
3120	serge	7041
6104	serge	7042	if (rdev->pm.pm_method == PM_METHOD_DPM)
		7043	radeon_pm_resume(rdev);
		7044
		7045	rdev->accel_working = true;
		7046	r = si_startup(rdev);
		7047	if (r) {
		7048	DRM_ERROR("si startup failed on resume\n");
		7049	rdev->accel_working = false;
		7050	return r;
		7051	}
		7052
		7053	return r;
		7054
		7055	}
		7056
2997	Serge	7057	/* Plan is to move initialization in that function and use
		7058	* helper function so that radeon_device_init pretty much
		7059	* do nothing more than calling asic specific function. This
		7060	* should also allow to remove a bunch of callback function
		7061	* like vram_info.
		7062	*/
		7063	int si_init(struct radeon_device *rdev)
		7064	{
		7065	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		7066	int r;
		7067
		7068	/* Read BIOS */
		7069	if (!radeon_get_bios(rdev)) {
		7070	if (ASIC_IS_AVIVO(rdev))
		7071	return -EINVAL;
		7072	}
		7073	/* Must be an ATOMBIOS */
		7074	if (!rdev->is_atom_bios) {
		7075	dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
		7076	return -EINVAL;
		7077	}
		7078	r = radeon_atombios_init(rdev);
		7079	if (r)
		7080	return r;
		7081
		7082	/* Post card if necessary */
		7083	if (!radeon_card_posted(rdev)) {
		7084	if (!rdev->bios) {
		7085	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		7086	return -EINVAL;
		7087	}
		7088	DRM_INFO("GPU not posted. posting now...\n");
		7089	atom_asic_init(rdev->mode_info.atom_context);
		7090	}
3764	Serge	7091	/* init golden registers */
		7092	si_init_golden_registers(rdev);
2997	Serge	7093	/* Initialize scratch registers */
		7094	si_scratch_init(rdev);
		7095	/* Initialize surface registers */
		7096	radeon_surface_init(rdev);
		7097	/* Initialize clocks */
		7098	radeon_get_clock_info(rdev->ddev);
		7099
		7100	/* Fence driver */
		7101	r = radeon_fence_driver_init(rdev);
		7102	if (r)
		7103	return r;
		7104
		7105	/* initialize memory controller */
		7106	r = si_mc_init(rdev);
		7107	if (r)
		7108	return r;
		7109	/* Memory manager */
		7110	r = radeon_bo_init(rdev);
		7111	if (r)
		7112	return r;
		7113
5078	serge	7114	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->ce_fw \|\|
		7115	!rdev->rlc_fw \|\| !rdev->mc_fw) {
		7116	r = si_init_microcode(rdev);
		7117	if (r) {
		7118	DRM_ERROR("Failed to load firmware!\n");
		7119	return r;
		7120	}
		7121	}
		7122
		7123	/* Initialize power management */
		7124	radeon_pm_init(rdev);
		7125
2997	Serge	7126	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		7127	ring->ring_obj = NULL;
		7128	r600_ring_init(rdev, ring, 1024 * 1024);
		7129
		7130	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
		7131	ring->ring_obj = NULL;
		7132	r600_ring_init(rdev, ring, 1024 * 1024);
		7133
		7134	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
		7135	ring->ring_obj = NULL;
		7136	r600_ring_init(rdev, ring, 1024 * 1024);
		7137
3192	Serge	7138	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		7139	ring->ring_obj = NULL;
		7140	r600_ring_init(rdev, ring, 64 * 1024);
		7141
		7142	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		7143	ring->ring_obj = NULL;
		7144	r600_ring_init(rdev, ring, 64 * 1024);
		7145
5078	serge	7146	if (rdev->has_uvd) {
		7147	r = radeon_uvd_init(rdev);
		7148	if (!r) {
		7149	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
		7150	ring->ring_obj = NULL;
		7151	r600_ring_init(rdev, ring, 4096);
		7152	}
		7153	}
		7154
6104	serge	7155	r = radeon_vce_init(rdev);
		7156	if (!r) {
		7157	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
		7158	ring->ring_obj = NULL;
		7159	r600_ring_init(rdev, ring, 4096);
		7160
		7161	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
		7162	ring->ring_obj = NULL;
		7163	r600_ring_init(rdev, ring, 4096);
		7164	}
		7165
		7166	rdev->ih.ring_obj = NULL;
2997	Serge	7167	r600_ih_ring_init(rdev, 64 * 1024);
		7168
6104	serge	7169	r = r600_pcie_gart_init(rdev);
		7170	if (r)
		7171	return r;
2997	Serge	7172
		7173	rdev->accel_working = true;
6104	serge	7174	r = si_startup(rdev);
2997	Serge	7175	if (r) {
		7176	dev_err(rdev->dev, "disabling GPU acceleration\n");
5078	serge	7177	si_cp_fini(rdev);
6104	serge	7178	cayman_dma_fini(rdev);
		7179	si_irq_fini(rdev);
		7180	sumo_rlc_fini(rdev);
		7181	radeon_wb_fini(rdev);
		7182	radeon_ib_pool_fini(rdev);
		7183	radeon_vm_manager_fini(rdev);
		7184	radeon_irq_kms_fini(rdev);
		7185	si_pcie_gart_fini(rdev);
2997	Serge	7186	rdev->accel_working = false;
		7187	}
		7188
		7189	/* Don't start up if the MC ucode is missing.
		7190	* The default clocks and voltages before the MC ucode
		7191	* is loaded are not suffient for advanced operations.
		7192	*/
		7193	if (!rdev->mc_fw) {
		7194	DRM_ERROR("radeon: MC ucode required for NI+.\n");
		7195	return -EINVAL;
		7196	}
		7197
		7198	return 0;
		7199	}
		7200
6104	serge	7201	void si_fini(struct radeon_device *rdev)
		7202	{
		7203	radeon_pm_fini(rdev);
		7204	si_cp_fini(rdev);
		7205	cayman_dma_fini(rdev);
		7206	si_fini_pg(rdev);
		7207	si_fini_cg(rdev);
		7208	si_irq_fini(rdev);
		7209	sumo_rlc_fini(rdev);
		7210	radeon_wb_fini(rdev);
		7211	radeon_vm_manager_fini(rdev);
		7212	radeon_ib_pool_fini(rdev);
		7213	radeon_irq_kms_fini(rdev);
		7214	if (rdev->has_uvd) {
		7215	uvd_v1_0_fini(rdev);
		7216	radeon_uvd_fini(rdev);
		7217	radeon_vce_fini(rdev);
		7218	}
		7219	si_pcie_gart_fini(rdev);
		7220	r600_vram_scratch_fini(rdev);
		7221	radeon_gem_fini(rdev);
		7222	radeon_fence_driver_fini(rdev);
		7223	radeon_bo_fini(rdev);
		7224	radeon_atombios_fini(rdev);
		7225	kfree(rdev->bios);
		7226	rdev->bios = NULL;
		7227	}
		7228
2997	Serge	7229	/**
3764	Serge	7230	* si_get_gpu_clock_counter - return GPU clock counter snapshot
2997	Serge	7231	*
		7232	* @rdev: radeon_device pointer
		7233	*
		7234	* Fetches a GPU clock counter snapshot (SI).
		7235	* Returns the 64 bit clock counter snapshot.
		7236	*/
3764	Serge	7237	uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
2997	Serge	7238	{
		7239	uint64_t clock;
		7240
		7241	mutex_lock(&rdev->gpu_clock_mutex);
		7242	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
		7243	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) \|
7146	serge	7244	((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
2997	Serge	7245	mutex_unlock(&rdev->gpu_clock_mutex);
		7246	return clock;
		7247	}
3764	Serge	7248
		7249	int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
		7250	{
		7251	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
		7252	int r;
		7253
		7254	/* bypass vclk and dclk with bclk */
		7255	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		7256	VCLK_SRC_SEL(1) \| DCLK_SRC_SEL(1),
		7257	~(VCLK_SRC_SEL_MASK \| DCLK_SRC_SEL_MASK));
		7258
		7259	/* put PLL in bypass mode */
		7260	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
		7261
		7262	if (!vclk \|\| !dclk) {
6104	serge	7263	/* keep the Bypass mode */
3764	Serge	7264	return 0;
		7265	}
		7266
		7267	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
		7268	16384, 0x03FFFFFF, 0, 128, 5,
		7269	&fb_div, &vclk_div, &dclk_div);
		7270	if (r)
		7271	return r;
		7272
		7273	/* set RESET_ANTI_MUX to 0 */
		7274	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
		7275
		7276	/* set VCO_MODE to 1 */
		7277	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
		7278
6104	serge	7279	/* disable sleep mode */
3764	Serge	7280	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
		7281
		7282	/* deassert UPLL_RESET */
		7283	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
		7284
		7285	mdelay(1);
		7286
		7287	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
		7288	if (r)
		7289	return r;
		7290
		7291	/* assert UPLL_RESET again */
		7292	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
		7293
		7294	/* disable spread spectrum. */
		7295	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
		7296
		7297	/* set feedback divider */
		7298	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
		7299
		7300	/* set ref divider to 0 */
		7301	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
		7302
		7303	if (fb_div < 307200)
		7304	WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
		7305	else
		7306	WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
		7307
		7308	/* set PDIV_A and PDIV_B */
		7309	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		7310	UPLL_PDIV_A(vclk_div) \| UPLL_PDIV_B(dclk_div),
		7311	~(UPLL_PDIV_A_MASK \| UPLL_PDIV_B_MASK));
		7312
		7313	/* give the PLL some time to settle */
		7314	mdelay(15);
		7315
		7316	/* deassert PLL_RESET */
		7317	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
		7318
		7319	mdelay(15);
		7320
		7321	/* switch from bypass mode to normal mode */
		7322	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
		7323
		7324	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
		7325	if (r)
		7326	return r;
		7327
		7328	/* switch VCLK and DCLK selection */
		7329	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		7330	VCLK_SRC_SEL(2) \| DCLK_SRC_SEL(2),
		7331	~(VCLK_SRC_SEL_MASK \| DCLK_SRC_SEL_MASK));
		7332
		7333	mdelay(100);
		7334
		7335	return 0;
		7336	}
5078	serge	7337
		7338	static void si_pcie_gen3_enable(struct radeon_device *rdev)
		7339	{
		7340	struct pci_dev *root = rdev->pdev->bus->self;
		7341	int bridge_pos, gpu_pos;
		7342	u32 speed_cntl, mask, current_data_rate;
		7343	int ret, i;
		7344	u16 tmp16;
		7345
		7346	if (radeon_pcie_gen2 == 0)
		7347	return;
		7348
		7349	if (rdev->flags & RADEON_IS_IGP)
		7350	return;
		7351
		7352	if (!(rdev->flags & RADEON_IS_PCIE))
		7353	return;
		7354
		7355	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
		7356	if (ret != 0)
		7357	return;
		7358
		7359	if (!(mask & (DRM_PCIE_SPEED_50 \| DRM_PCIE_SPEED_80)))
		7360	return;
		7361
		7362	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
		7363	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
		7364	LC_CURRENT_DATA_RATE_SHIFT;
		7365	if (mask & DRM_PCIE_SPEED_80) {
		7366	if (current_data_rate == 2) {
		7367	DRM_INFO("PCIE gen 3 link speeds already enabled\n");
		7368	return;
		7369	}
		7370	DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
		7371	} else if (mask & DRM_PCIE_SPEED_50) {
		7372	if (current_data_rate == 1) {
		7373	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
		7374	return;
		7375	}
		7376	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
		7377	}
		7378
		7379	bridge_pos = pci_pcie_cap(root);
		7380	if (!bridge_pos)
		7381	return;
		7382
		7383	gpu_pos = pci_pcie_cap(rdev->pdev);
		7384	if (!gpu_pos)
		7385	return;
		7386
		7387	if (mask & DRM_PCIE_SPEED_80) {
		7388	/* re-try equalization if gen3 is not already enabled */
		7389	if (current_data_rate != 2) {
		7390	u16 bridge_cfg, gpu_cfg;
		7391	u16 bridge_cfg2, gpu_cfg2;
		7392	u32 max_lw, current_lw, tmp;
		7393
		7394	pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
		7395	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
		7396
		7397	tmp16 = bridge_cfg \| PCI_EXP_LNKCTL_HAWD;
		7398	pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
		7399
		7400	tmp16 = gpu_cfg \| PCI_EXP_LNKCTL_HAWD;
		7401	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
		7402
		7403	tmp = RREG32_PCIE(PCIE_LC_STATUS1);
		7404	max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
		7405	current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
		7406
		7407	if (current_lw < max_lw) {
		7408	tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
		7409	if (tmp & LC_RENEGOTIATION_SUPPORT) {
		7410	tmp &= ~(LC_LINK_WIDTH_MASK \| LC_UPCONFIGURE_DIS);
		7411	tmp \|= (max_lw << LC_LINK_WIDTH_SHIFT);
		7412	tmp \|= LC_UPCONFIGURE_SUPPORT \| LC_RENEGOTIATE_EN \| LC_RECONFIG_NOW;
		7413	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
		7414	}
		7415	}
		7416
		7417	for (i = 0; i < 10; i++) {
		7418	/* check status */
		7419	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
		7420	if (tmp16 & PCI_EXP_DEVSTA_TRPND)
		7421	break;
		7422
		7423	pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
		7424	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
		7425
		7426	pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
		7427	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
		7428
		7429	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
		7430	tmp \|= LC_SET_QUIESCE;
		7431	WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
		7432
		7433	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
		7434	tmp \|= LC_REDO_EQ;
		7435	WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
		7436
		7437	mdelay(100);
		7438
		7439	/* linkctl */
		7440	pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
		7441	tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
		7442	tmp16 \|= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
		7443	pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
		7444
		7445	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
		7446	tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
		7447	tmp16 \|= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
		7448	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
		7449
		7450	/* linkctl2 */
		7451	pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
		7452	tmp16 &= ~((1 << 4) \| (7 << 9));
		7453	tmp16 \|= (bridge_cfg2 & ((1 << 4) \| (7 << 9)));
		7454	pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
		7455
		7456	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
		7457	tmp16 &= ~((1 << 4) \| (7 << 9));
		7458	tmp16 \|= (gpu_cfg2 & ((1 << 4) \| (7 << 9)));
		7459	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
		7460
		7461	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
		7462	tmp &= ~LC_SET_QUIESCE;
		7463	WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
		7464	}
		7465	}
		7466	}
		7467
		7468	/* set the link speed */
		7469	speed_cntl \|= LC_FORCE_EN_SW_SPEED_CHANGE \| LC_FORCE_DIS_HW_SPEED_CHANGE;
		7470	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
		7471	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
		7472
		7473	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
		7474	tmp16 &= ~0xf;
		7475	if (mask & DRM_PCIE_SPEED_80)
		7476	tmp16 \|= 3; /* gen3 */
		7477	else if (mask & DRM_PCIE_SPEED_50)
		7478	tmp16 \|= 2; /* gen2 */
		7479	else
		7480	tmp16 \|= 1; /* gen1 */
		7481	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
		7482
		7483	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
		7484	speed_cntl \|= LC_INITIATE_LINK_SPEED_CHANGE;
		7485	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
		7486
		7487	for (i = 0; i < rdev->usec_timeout; i++) {
		7488	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
		7489	if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
		7490	break;
		7491	udelay(1);
		7492	}
		7493	}
		7494
		7495	static void si_program_aspm(struct radeon_device *rdev)
		7496	{
		7497	u32 data, orig;
		7498	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
		7499	bool disable_clkreq = false;
		7500
		7501	if (radeon_aspm == 0)
		7502	return;
		7503
		7504	if (!(rdev->flags & RADEON_IS_PCIE))
		7505	return;
		7506
		7507	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
		7508	data &= ~LC_XMIT_N_FTS_MASK;
		7509	data \|= LC_XMIT_N_FTS(0x24) \| LC_XMIT_N_FTS_OVERRIDE_EN;
		7510	if (orig != data)
		7511	WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
		7512
		7513	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
		7514	data \|= LC_GO_TO_RECOVERY;
		7515	if (orig != data)
		7516	WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
		7517
		7518	orig = data = RREG32_PCIE(PCIE_P_CNTL);
		7519	data \|= P_IGNORE_EDB_ERR;
		7520	if (orig != data)
		7521	WREG32_PCIE(PCIE_P_CNTL, data);
		7522
		7523	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
		7524	data &= ~(LC_L0S_INACTIVITY_MASK \| LC_L1_INACTIVITY_MASK);
		7525	data \|= LC_PMI_TO_L1_DIS;
		7526	if (!disable_l0s)
		7527	data \|= LC_L0S_INACTIVITY(7);
		7528
		7529	if (!disable_l1) {
		7530	data \|= LC_L1_INACTIVITY(7);
		7531	data &= ~LC_PMI_TO_L1_DIS;
		7532	if (orig != data)
		7533	WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
		7534
		7535	if (!disable_plloff_in_l1) {
		7536	bool clk_req_support;
		7537
		7538	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
		7539	data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK \| PLL_POWER_STATE_IN_TXS2_0_MASK);
		7540	data \|= PLL_POWER_STATE_IN_OFF_0(7) \| PLL_POWER_STATE_IN_TXS2_0(7);
		7541	if (orig != data)
		7542	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
		7543
		7544	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
		7545	data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK \| PLL_POWER_STATE_IN_TXS2_1_MASK);
		7546	data \|= PLL_POWER_STATE_IN_OFF_1(7) \| PLL_POWER_STATE_IN_TXS2_1(7);
		7547	if (orig != data)
		7548	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
		7549
		7550	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
		7551	data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK \| PLL_POWER_STATE_IN_TXS2_0_MASK);
		7552	data \|= PLL_POWER_STATE_IN_OFF_0(7) \| PLL_POWER_STATE_IN_TXS2_0(7);
		7553	if (orig != data)
		7554	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
		7555
		7556	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
		7557	data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK \| PLL_POWER_STATE_IN_TXS2_1_MASK);
		7558	data \|= PLL_POWER_STATE_IN_OFF_1(7) \| PLL_POWER_STATE_IN_TXS2_1(7);
		7559	if (orig != data)
		7560	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
		7561
		7562	if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
		7563	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
		7564	data &= ~PLL_RAMP_UP_TIME_0_MASK;
		7565	if (orig != data)
		7566	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
		7567
		7568	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
		7569	data &= ~PLL_RAMP_UP_TIME_1_MASK;
		7570	if (orig != data)
		7571	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
		7572
		7573	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
		7574	data &= ~PLL_RAMP_UP_TIME_2_MASK;
		7575	if (orig != data)
		7576	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
		7577
		7578	orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
		7579	data &= ~PLL_RAMP_UP_TIME_3_MASK;
		7580	if (orig != data)
		7581	WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
		7582
		7583	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
		7584	data &= ~PLL_RAMP_UP_TIME_0_MASK;
		7585	if (orig != data)
		7586	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
		7587
		7588	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
		7589	data &= ~PLL_RAMP_UP_TIME_1_MASK;
		7590	if (orig != data)
		7591	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
		7592
		7593	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
		7594	data &= ~PLL_RAMP_UP_TIME_2_MASK;
		7595	if (orig != data)
		7596	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
		7597
		7598	orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
		7599	data &= ~PLL_RAMP_UP_TIME_3_MASK;
		7600	if (orig != data)
		7601	WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
		7602	}
		7603	orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
		7604	data &= ~LC_DYN_LANES_PWR_STATE_MASK;
		7605	data \|= LC_DYN_LANES_PWR_STATE(3);
		7606	if (orig != data)
		7607	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
		7608
		7609	orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
		7610	data &= ~LS2_EXIT_TIME_MASK;
		7611	if ((rdev->family == CHIP_OLAND) \|\| (rdev->family == CHIP_HAINAN))
		7612	data \|= LS2_EXIT_TIME(5);
		7613	if (orig != data)
		7614	WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
		7615
		7616	orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
		7617	data &= ~LS2_EXIT_TIME_MASK;
		7618	if ((rdev->family == CHIP_OLAND) \|\| (rdev->family == CHIP_HAINAN))
		7619	data \|= LS2_EXIT_TIME(5);
		7620	if (orig != data)
		7621	WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
		7622
		7623	if (!disable_clkreq) {
		7624	struct pci_dev *root = rdev->pdev->bus->self;
		7625	u32 lnkcap;
		7626
		7627	clk_req_support = false;
		7628	pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
		7629	if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
		7630	clk_req_support = true;
		7631	} else {
		7632	clk_req_support = false;
		7633	}
		7634
		7635	if (clk_req_support) {
		7636	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
		7637	data \|= LC_ALLOW_PDWN_IN_L1 \| LC_ALLOW_PDWN_IN_L23;
		7638	if (orig != data)
		7639	WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
		7640
		7641	orig = data = RREG32(THM_CLK_CNTL);
		7642	data &= ~(CMON_CLK_SEL_MASK \| TMON_CLK_SEL_MASK);
		7643	data \|= CMON_CLK_SEL(1) \| TMON_CLK_SEL(1);
		7644	if (orig != data)
		7645	WREG32(THM_CLK_CNTL, data);
		7646
		7647	orig = data = RREG32(MISC_CLK_CNTL);
		7648	data &= ~(DEEP_SLEEP_CLK_SEL_MASK \| ZCLK_SEL_MASK);
		7649	data \|= DEEP_SLEEP_CLK_SEL(1) \| ZCLK_SEL(1);
		7650	if (orig != data)
		7651	WREG32(MISC_CLK_CNTL, data);
		7652
		7653	orig = data = RREG32(CG_CLKPIN_CNTL);
		7654	data &= ~BCLK_AS_XCLK;
		7655	if (orig != data)
		7656	WREG32(CG_CLKPIN_CNTL, data);
		7657
		7658	orig = data = RREG32(CG_CLKPIN_CNTL_2);
		7659	data &= ~FORCE_BIF_REFCLK_EN;
		7660	if (orig != data)
		7661	WREG32(CG_CLKPIN_CNTL_2, data);
		7662
		7663	orig = data = RREG32(MPLL_BYPASSCLK_SEL);
		7664	data &= ~MPLL_CLKOUT_SEL_MASK;
		7665	data \|= MPLL_CLKOUT_SEL(4);
		7666	if (orig != data)
		7667	WREG32(MPLL_BYPASSCLK_SEL, data);
		7668
		7669	orig = data = RREG32(SPLL_CNTL_MODE);
		7670	data &= ~SPLL_REFCLK_SEL_MASK;
		7671	if (orig != data)
		7672	WREG32(SPLL_CNTL_MODE, data);
		7673	}
		7674	}
		7675	} else {
		7676	if (orig != data)
		7677	WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
		7678	}
		7679
		7680	orig = data = RREG32_PCIE(PCIE_CNTL2);
		7681	data \|= SLV_MEM_LS_EN \| MST_MEM_LS_EN \| REPLAY_MEM_LS_EN;
		7682	if (orig != data)
		7683	WREG32_PCIE(PCIE_CNTL2, data);
		7684
		7685	if (!disable_l0s) {
		7686	data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
		7687	if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
		7688	data = RREG32_PCIE(PCIE_LC_STATUS1);
		7689	if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
		7690	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
		7691	data &= ~LC_L0S_INACTIVITY_MASK;
		7692	if (orig != data)
		7693	WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
		7694	}
		7695	}
		7696	}
		7697	}
6104	serge	7698
		7699	int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
		7700	{
7146	serge	7701	unsigned i;
6104	serge	7702
7146	serge	7703	/* make sure VCEPLL_CTLREQ is deasserted */
		7704	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
6104	serge	7705
7146	serge	7706	mdelay(10);
6104	serge	7707
7146	serge	7708	/* assert UPLL_CTLREQ */
		7709	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
6104	serge	7710
7146	serge	7711	/* wait for CTLACK and CTLACK2 to get asserted */
		7712	for (i = 0; i < 100; ++i) {
		7713	uint32_t mask = UPLL_CTLACK_MASK \| UPLL_CTLACK2_MASK;
		7714	if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
		7715	break;
		7716	mdelay(10);
		7717	}
6104	serge	7718
7146	serge	7719	/* deassert UPLL_CTLREQ */
		7720	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
6104	serge	7721
7146	serge	7722	if (i == 100) {
		7723	DRM_ERROR("Timeout setting UVD clocks!\n");
		7724	return -ETIMEDOUT;
		7725	}
6104	serge	7726
7146	serge	7727	return 0;
6104	serge	7728	}
		7729
		7730	int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
		7731	{
		7732	unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
		7733	int r;
		7734
		7735	/* bypass evclk and ecclk with bclk */
		7736	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
		7737	EVCLK_SRC_SEL(1) \| ECCLK_SRC_SEL(1),
		7738	~(EVCLK_SRC_SEL_MASK \| ECCLK_SRC_SEL_MASK));
		7739
		7740	/* put PLL in bypass mode */
		7741	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
		7742	~VCEPLL_BYPASS_EN_MASK);
		7743
		7744	if (!evclk \|\| !ecclk) {
		7745	/* keep the Bypass mode, put PLL to sleep */
		7746	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
		7747	~VCEPLL_SLEEP_MASK);
		7748	return 0;
		7749	}
		7750
		7751	r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
		7752	16384, 0x03FFFFFF, 0, 128, 5,
		7753	&fb_div, &evclk_div, &ecclk_div);
		7754	if (r)
		7755	return r;
		7756
		7757	/* set RESET_ANTI_MUX to 0 */
		7758	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
		7759
		7760	/* set VCO_MODE to 1 */
		7761	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
		7762	~VCEPLL_VCO_MODE_MASK);
		7763
		7764	/* toggle VCEPLL_SLEEP to 1 then back to 0 */
		7765	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
		7766	~VCEPLL_SLEEP_MASK);
		7767	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
		7768
		7769	/* deassert VCEPLL_RESET */
		7770	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
		7771
		7772	mdelay(1);
		7773
		7774	r = si_vce_send_vcepll_ctlreq(rdev);
		7775	if (r)
		7776	return r;
		7777
		7778	/* assert VCEPLL_RESET again */
		7779	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
		7780
		7781	/* disable spread spectrum. */
		7782	WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
		7783
		7784	/* set feedback divider */
		7785	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
		7786
		7787	/* set ref divider to 0 */
		7788	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
		7789
		7790	/* set PDIV_A and PDIV_B */
		7791	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
		7792	VCEPLL_PDIV_A(evclk_div) \| VCEPLL_PDIV_B(ecclk_div),
		7793	~(VCEPLL_PDIV_A_MASK \| VCEPLL_PDIV_B_MASK));
		7794
		7795	/* give the PLL some time to settle */
		7796	mdelay(15);
		7797
		7798	/* deassert PLL_RESET */
		7799	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
		7800
		7801	mdelay(15);
		7802
		7803	/* switch from bypass mode to normal mode */
		7804	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
		7805
		7806	r = si_vce_send_vcepll_ctlreq(rdev);
		7807	if (r)
		7808	return r;
		7809
		7810	/* switch VCLK and DCLK selection */
		7811	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
		7812	EVCLK_SRC_SEL(16) \| ECCLK_SRC_SEL(16),
		7813	~(EVCLK_SRC_SEL_MASK \| ECCLK_SRC_SEL_MASK));
		7814
		7815	mdelay(100);
		7816
		7817	return 0;
		7818	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/si.c – Rev 7146