WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/si.c

Rev	Author	Line No.	Line
2997	Serge	1	/*
		2	* Copyright 2011 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	//#include
		26	#include
		27	#include
		28	#include
		29	#include "radeon.h"
		30	#include "radeon_asic.h"
		31	#include
		32	#include "sid.h"
		33	#include "atom.h"
		34	#include "si_blit_shaders.h"
		35
		36	#define SI_PFP_UCODE_SIZE 2144
		37	#define SI_PM4_UCODE_SIZE 2144
		38	#define SI_CE_UCODE_SIZE 2144
		39	#define SI_RLC_UCODE_SIZE 2048
		40	#define SI_MC_UCODE_SIZE 7769
3764	Serge	41	#define OLAND_MC_UCODE_SIZE 7863
2997	Serge	42
		43	MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
		44	MODULE_FIRMWARE("radeon/TAHITI_me.bin");
		45	MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
		46	MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
		47	MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
		48	MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
		49	MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
		50	MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
		51	MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
		52	MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
		53	MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
		54	MODULE_FIRMWARE("radeon/VERDE_me.bin");
		55	MODULE_FIRMWARE("radeon/VERDE_ce.bin");
		56	MODULE_FIRMWARE("radeon/VERDE_mc.bin");
		57	MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
3764	Serge	58	MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
		59	MODULE_FIRMWARE("radeon/OLAND_me.bin");
		60	MODULE_FIRMWARE("radeon/OLAND_ce.bin");
		61	MODULE_FIRMWARE("radeon/OLAND_mc.bin");
		62	MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
		63	MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
		64	MODULE_FIRMWARE("radeon/HAINAN_me.bin");
		65	MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
		66	MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
		67	MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
2997	Serge	68
		69	extern int r600_ih_ring_alloc(struct radeon_device *rdev);
		70	extern void r600_ih_ring_fini(struct radeon_device *rdev);
		71	extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
		72	extern void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save);
		73	extern void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save);
		74	extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
3764	Serge	75	extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
		76	extern bool evergreen_is_display_hung(struct radeon_device *rdev);
2997	Serge	77
3764	Serge	78	static const u32 tahiti_golden_rlc_registers[] =
		79	{
		80	0xc424, 0xffffffff, 0x00601005,
		81	0xc47c, 0xffffffff, 0x10104040,
		82	0xc488, 0xffffffff, 0x0100000a,
		83	0xc314, 0xffffffff, 0x00000800,
		84	0xc30c, 0xffffffff, 0x800000f4,
		85	0xf4a8, 0xffffffff, 0x00000000
		86	};
		87
		88	static const u32 tahiti_golden_registers[] =
		89	{
		90	0x9a10, 0x00010000, 0x00018208,
		91	0x9830, 0xffffffff, 0x00000000,
		92	0x9834, 0xf00fffff, 0x00000400,
		93	0x9838, 0x0002021c, 0x00020200,
		94	0xc78, 0x00000080, 0x00000000,
		95	0xd030, 0x000300c0, 0x00800040,
		96	0xd830, 0x000300c0, 0x00800040,
		97	0x5bb0, 0x000000f0, 0x00000070,
		98	0x5bc0, 0x00200000, 0x50100000,
		99	0x7030, 0x31000311, 0x00000011,
		100	0x277c, 0x00000003, 0x000007ff,
		101	0x240c, 0x000007ff, 0x00000000,
		102	0x8a14, 0xf000001f, 0x00000007,
		103	0x8b24, 0xffffffff, 0x00ffffff,
		104	0x8b10, 0x0000ff0f, 0x00000000,
		105	0x28a4c, 0x07ffffff, 0x4e000000,
		106	0x28350, 0x3f3f3fff, 0x2a00126a,
		107	0x30, 0x000000ff, 0x0040,
		108	0x34, 0x00000040, 0x00004040,
		109	0x9100, 0x07ffffff, 0x03000000,
		110	0x8e88, 0x01ff1f3f, 0x00000000,
		111	0x8e84, 0x01ff1f3f, 0x00000000,
		112	0x9060, 0x0000007f, 0x00000020,
		113	0x9508, 0x00010000, 0x00010000,
		114	0xac14, 0x00000200, 0x000002fb,
		115	0xac10, 0xffffffff, 0x0000543b,
		116	0xac0c, 0xffffffff, 0xa9210876,
		117	0x88d0, 0xffffffff, 0x000fff40,
		118	0x88d4, 0x0000001f, 0x00000010,
		119	0x1410, 0x20000000, 0x20fffed8,
		120	0x15c0, 0x000c0fc0, 0x000c0400
		121	};
		122
		123	static const u32 tahiti_golden_registers2[] =
		124	{
		125	0xc64, 0x00000001, 0x00000001
		126	};
		127
		128	static const u32 pitcairn_golden_rlc_registers[] =
		129	{
		130	0xc424, 0xffffffff, 0x00601004,
		131	0xc47c, 0xffffffff, 0x10102020,
		132	0xc488, 0xffffffff, 0x01000020,
		133	0xc314, 0xffffffff, 0x00000800,
		134	0xc30c, 0xffffffff, 0x800000a4
		135	};
		136
		137	static const u32 pitcairn_golden_registers[] =
		138	{
		139	0x9a10, 0x00010000, 0x00018208,
		140	0x9830, 0xffffffff, 0x00000000,
		141	0x9834, 0xf00fffff, 0x00000400,
		142	0x9838, 0x0002021c, 0x00020200,
		143	0xc78, 0x00000080, 0x00000000,
		144	0xd030, 0x000300c0, 0x00800040,
		145	0xd830, 0x000300c0, 0x00800040,
		146	0x5bb0, 0x000000f0, 0x00000070,
		147	0x5bc0, 0x00200000, 0x50100000,
		148	0x7030, 0x31000311, 0x00000011,
		149	0x2ae4, 0x00073ffe, 0x000022a2,
		150	0x240c, 0x000007ff, 0x00000000,
		151	0x8a14, 0xf000001f, 0x00000007,
		152	0x8b24, 0xffffffff, 0x00ffffff,
		153	0x8b10, 0x0000ff0f, 0x00000000,
		154	0x28a4c, 0x07ffffff, 0x4e000000,
		155	0x28350, 0x3f3f3fff, 0x2a00126a,
		156	0x30, 0x000000ff, 0x0040,
		157	0x34, 0x00000040, 0x00004040,
		158	0x9100, 0x07ffffff, 0x03000000,
		159	0x9060, 0x0000007f, 0x00000020,
		160	0x9508, 0x00010000, 0x00010000,
		161	0xac14, 0x000003ff, 0x000000f7,
		162	0xac10, 0xffffffff, 0x00000000,
		163	0xac0c, 0xffffffff, 0x32761054,
		164	0x88d4, 0x0000001f, 0x00000010,
		165	0x15c0, 0x000c0fc0, 0x000c0400
		166	};
		167
		168	static const u32 verde_golden_rlc_registers[] =
		169	{
		170	0xc424, 0xffffffff, 0x033f1005,
		171	0xc47c, 0xffffffff, 0x10808020,
		172	0xc488, 0xffffffff, 0x00800008,
		173	0xc314, 0xffffffff, 0x00001000,
		174	0xc30c, 0xffffffff, 0x80010014
		175	};
		176
		177	static const u32 verde_golden_registers[] =
		178	{
		179	0x9a10, 0x00010000, 0x00018208,
		180	0x9830, 0xffffffff, 0x00000000,
		181	0x9834, 0xf00fffff, 0x00000400,
		182	0x9838, 0x0002021c, 0x00020200,
		183	0xc78, 0x00000080, 0x00000000,
		184	0xd030, 0x000300c0, 0x00800040,
		185	0xd030, 0x000300c0, 0x00800040,
		186	0xd830, 0x000300c0, 0x00800040,
		187	0xd830, 0x000300c0, 0x00800040,
		188	0x5bb0, 0x000000f0, 0x00000070,
		189	0x5bc0, 0x00200000, 0x50100000,
		190	0x7030, 0x31000311, 0x00000011,
		191	0x2ae4, 0x00073ffe, 0x000022a2,
		192	0x2ae4, 0x00073ffe, 0x000022a2,
		193	0x2ae4, 0x00073ffe, 0x000022a2,
		194	0x240c, 0x000007ff, 0x00000000,
		195	0x240c, 0x000007ff, 0x00000000,
		196	0x240c, 0x000007ff, 0x00000000,
		197	0x8a14, 0xf000001f, 0x00000007,
		198	0x8a14, 0xf000001f, 0x00000007,
		199	0x8a14, 0xf000001f, 0x00000007,
		200	0x8b24, 0xffffffff, 0x00ffffff,
		201	0x8b10, 0x0000ff0f, 0x00000000,
		202	0x28a4c, 0x07ffffff, 0x4e000000,
		203	0x28350, 0x3f3f3fff, 0x0000124a,
		204	0x28350, 0x3f3f3fff, 0x0000124a,
		205	0x28350, 0x3f3f3fff, 0x0000124a,
		206	0x30, 0x000000ff, 0x0040,
		207	0x34, 0x00000040, 0x00004040,
		208	0x9100, 0x07ffffff, 0x03000000,
		209	0x9100, 0x07ffffff, 0x03000000,
		210	0x8e88, 0x01ff1f3f, 0x00000000,
		211	0x8e88, 0x01ff1f3f, 0x00000000,
		212	0x8e88, 0x01ff1f3f, 0x00000000,
		213	0x8e84, 0x01ff1f3f, 0x00000000,
		214	0x8e84, 0x01ff1f3f, 0x00000000,
		215	0x8e84, 0x01ff1f3f, 0x00000000,
		216	0x9060, 0x0000007f, 0x00000020,
		217	0x9508, 0x00010000, 0x00010000,
		218	0xac14, 0x000003ff, 0x00000003,
		219	0xac14, 0x000003ff, 0x00000003,
		220	0xac14, 0x000003ff, 0x00000003,
		221	0xac10, 0xffffffff, 0x00000000,
		222	0xac10, 0xffffffff, 0x00000000,
		223	0xac10, 0xffffffff, 0x00000000,
		224	0xac0c, 0xffffffff, 0x00001032,
		225	0xac0c, 0xffffffff, 0x00001032,
		226	0xac0c, 0xffffffff, 0x00001032,
		227	0x88d4, 0x0000001f, 0x00000010,
		228	0x88d4, 0x0000001f, 0x00000010,
		229	0x88d4, 0x0000001f, 0x00000010,
		230	0x15c0, 0x000c0fc0, 0x000c0400
		231	};
		232
		233	static const u32 oland_golden_rlc_registers[] =
		234	{
		235	0xc424, 0xffffffff, 0x00601005,
		236	0xc47c, 0xffffffff, 0x10104040,
		237	0xc488, 0xffffffff, 0x0100000a,
		238	0xc314, 0xffffffff, 0x00000800,
		239	0xc30c, 0xffffffff, 0x800000f4
		240	};
		241
		242	static const u32 oland_golden_registers[] =
		243	{
		244	0x9a10, 0x00010000, 0x00018208,
		245	0x9830, 0xffffffff, 0x00000000,
		246	0x9834, 0xf00fffff, 0x00000400,
		247	0x9838, 0x0002021c, 0x00020200,
		248	0xc78, 0x00000080, 0x00000000,
		249	0xd030, 0x000300c0, 0x00800040,
		250	0xd830, 0x000300c0, 0x00800040,
		251	0x5bb0, 0x000000f0, 0x00000070,
		252	0x5bc0, 0x00200000, 0x50100000,
		253	0x7030, 0x31000311, 0x00000011,
		254	0x2ae4, 0x00073ffe, 0x000022a2,
		255	0x240c, 0x000007ff, 0x00000000,
		256	0x8a14, 0xf000001f, 0x00000007,
		257	0x8b24, 0xffffffff, 0x00ffffff,
		258	0x8b10, 0x0000ff0f, 0x00000000,
		259	0x28a4c, 0x07ffffff, 0x4e000000,
		260	0x28350, 0x3f3f3fff, 0x00000082,
		261	0x30, 0x000000ff, 0x0040,
		262	0x34, 0x00000040, 0x00004040,
		263	0x9100, 0x07ffffff, 0x03000000,
		264	0x9060, 0x0000007f, 0x00000020,
		265	0x9508, 0x00010000, 0x00010000,
		266	0xac14, 0x000003ff, 0x000000f3,
		267	0xac10, 0xffffffff, 0x00000000,
		268	0xac0c, 0xffffffff, 0x00003210,
		269	0x88d4, 0x0000001f, 0x00000010,
		270	0x15c0, 0x000c0fc0, 0x000c0400
		271	};
		272
		273	static const u32 hainan_golden_registers[] =
		274	{
		275	0x9a10, 0x00010000, 0x00018208,
		276	0x9830, 0xffffffff, 0x00000000,
		277	0x9834, 0xf00fffff, 0x00000400,
		278	0x9838, 0x0002021c, 0x00020200,
		279	0xd0c0, 0xff000fff, 0x00000100,
		280	0xd030, 0x000300c0, 0x00800040,
		281	0xd8c0, 0xff000fff, 0x00000100,
		282	0xd830, 0x000300c0, 0x00800040,
		283	0x2ae4, 0x00073ffe, 0x000022a2,
		284	0x240c, 0x000007ff, 0x00000000,
		285	0x8a14, 0xf000001f, 0x00000007,
		286	0x8b24, 0xffffffff, 0x00ffffff,
		287	0x8b10, 0x0000ff0f, 0x00000000,
		288	0x28a4c, 0x07ffffff, 0x4e000000,
		289	0x28350, 0x3f3f3fff, 0x00000000,
		290	0x30, 0x000000ff, 0x0040,
		291	0x34, 0x00000040, 0x00004040,
		292	0x9100, 0x03e00000, 0x03600000,
		293	0x9060, 0x0000007f, 0x00000020,
		294	0x9508, 0x00010000, 0x00010000,
		295	0xac14, 0x000003ff, 0x000000f1,
		296	0xac10, 0xffffffff, 0x00000000,
		297	0xac0c, 0xffffffff, 0x00003210,
		298	0x88d4, 0x0000001f, 0x00000010,
		299	0x15c0, 0x000c0fc0, 0x000c0400
		300	};
		301
		302	static const u32 hainan_golden_registers2[] =
		303	{
		304	0x98f8, 0xffffffff, 0x02010001
		305	};
		306
		307	static const u32 tahiti_mgcg_cgcg_init[] =
		308	{
		309	0xc400, 0xffffffff, 0xfffffffc,
		310	0x802c, 0xffffffff, 0xe0000000,
		311	0x9a60, 0xffffffff, 0x00000100,
		312	0x92a4, 0xffffffff, 0x00000100,
		313	0xc164, 0xffffffff, 0x00000100,
		314	0x9774, 0xffffffff, 0x00000100,
		315	0x8984, 0xffffffff, 0x06000100,
		316	0x8a18, 0xffffffff, 0x00000100,
		317	0x92a0, 0xffffffff, 0x00000100,
		318	0xc380, 0xffffffff, 0x00000100,
		319	0x8b28, 0xffffffff, 0x00000100,
		320	0x9144, 0xffffffff, 0x00000100,
		321	0x8d88, 0xffffffff, 0x00000100,
		322	0x8d8c, 0xffffffff, 0x00000100,
		323	0x9030, 0xffffffff, 0x00000100,
		324	0x9034, 0xffffffff, 0x00000100,
		325	0x9038, 0xffffffff, 0x00000100,
		326	0x903c, 0xffffffff, 0x00000100,
		327	0xad80, 0xffffffff, 0x00000100,
		328	0xac54, 0xffffffff, 0x00000100,
		329	0x897c, 0xffffffff, 0x06000100,
		330	0x9868, 0xffffffff, 0x00000100,
		331	0x9510, 0xffffffff, 0x00000100,
		332	0xaf04, 0xffffffff, 0x00000100,
		333	0xae04, 0xffffffff, 0x00000100,
		334	0x949c, 0xffffffff, 0x00000100,
		335	0x802c, 0xffffffff, 0xe0000000,
		336	0x9160, 0xffffffff, 0x00010000,
		337	0x9164, 0xffffffff, 0x00030002,
		338	0x9168, 0xffffffff, 0x00040007,
		339	0x916c, 0xffffffff, 0x00060005,
		340	0x9170, 0xffffffff, 0x00090008,
		341	0x9174, 0xffffffff, 0x00020001,
		342	0x9178, 0xffffffff, 0x00040003,
		343	0x917c, 0xffffffff, 0x00000007,
		344	0x9180, 0xffffffff, 0x00060005,
		345	0x9184, 0xffffffff, 0x00090008,
		346	0x9188, 0xffffffff, 0x00030002,
		347	0x918c, 0xffffffff, 0x00050004,
		348	0x9190, 0xffffffff, 0x00000008,
		349	0x9194, 0xffffffff, 0x00070006,
		350	0x9198, 0xffffffff, 0x000a0009,
		351	0x919c, 0xffffffff, 0x00040003,
		352	0x91a0, 0xffffffff, 0x00060005,
		353	0x91a4, 0xffffffff, 0x00000009,
		354	0x91a8, 0xffffffff, 0x00080007,
		355	0x91ac, 0xffffffff, 0x000b000a,
		356	0x91b0, 0xffffffff, 0x00050004,
		357	0x91b4, 0xffffffff, 0x00070006,
		358	0x91b8, 0xffffffff, 0x0008000b,
		359	0x91bc, 0xffffffff, 0x000a0009,
		360	0x91c0, 0xffffffff, 0x000d000c,
		361	0x91c4, 0xffffffff, 0x00060005,
		362	0x91c8, 0xffffffff, 0x00080007,
		363	0x91cc, 0xffffffff, 0x0000000b,
		364	0x91d0, 0xffffffff, 0x000a0009,
		365	0x91d4, 0xffffffff, 0x000d000c,
		366	0x91d8, 0xffffffff, 0x00070006,
		367	0x91dc, 0xffffffff, 0x00090008,
		368	0x91e0, 0xffffffff, 0x0000000c,
		369	0x91e4, 0xffffffff, 0x000b000a,
		370	0x91e8, 0xffffffff, 0x000e000d,
		371	0x91ec, 0xffffffff, 0x00080007,
		372	0x91f0, 0xffffffff, 0x000a0009,
		373	0x91f4, 0xffffffff, 0x0000000d,
		374	0x91f8, 0xffffffff, 0x000c000b,
		375	0x91fc, 0xffffffff, 0x000f000e,
		376	0x9200, 0xffffffff, 0x00090008,
		377	0x9204, 0xffffffff, 0x000b000a,
		378	0x9208, 0xffffffff, 0x000c000f,
		379	0x920c, 0xffffffff, 0x000e000d,
		380	0x9210, 0xffffffff, 0x00110010,
		381	0x9214, 0xffffffff, 0x000a0009,
		382	0x9218, 0xffffffff, 0x000c000b,
		383	0x921c, 0xffffffff, 0x0000000f,
		384	0x9220, 0xffffffff, 0x000e000d,
		385	0x9224, 0xffffffff, 0x00110010,
		386	0x9228, 0xffffffff, 0x000b000a,
		387	0x922c, 0xffffffff, 0x000d000c,
		388	0x9230, 0xffffffff, 0x00000010,
		389	0x9234, 0xffffffff, 0x000f000e,
		390	0x9238, 0xffffffff, 0x00120011,
		391	0x923c, 0xffffffff, 0x000c000b,
		392	0x9240, 0xffffffff, 0x000e000d,
		393	0x9244, 0xffffffff, 0x00000011,
		394	0x9248, 0xffffffff, 0x0010000f,
		395	0x924c, 0xffffffff, 0x00130012,
		396	0x9250, 0xffffffff, 0x000d000c,
		397	0x9254, 0xffffffff, 0x000f000e,
		398	0x9258, 0xffffffff, 0x00100013,
		399	0x925c, 0xffffffff, 0x00120011,
		400	0x9260, 0xffffffff, 0x00150014,
		401	0x9264, 0xffffffff, 0x000e000d,
		402	0x9268, 0xffffffff, 0x0010000f,
		403	0x926c, 0xffffffff, 0x00000013,
		404	0x9270, 0xffffffff, 0x00120011,
		405	0x9274, 0xffffffff, 0x00150014,
		406	0x9278, 0xffffffff, 0x000f000e,
		407	0x927c, 0xffffffff, 0x00110010,
		408	0x9280, 0xffffffff, 0x00000014,
		409	0x9284, 0xffffffff, 0x00130012,
		410	0x9288, 0xffffffff, 0x00160015,
		411	0x928c, 0xffffffff, 0x0010000f,
		412	0x9290, 0xffffffff, 0x00120011,
		413	0x9294, 0xffffffff, 0x00000015,
		414	0x9298, 0xffffffff, 0x00140013,
		415	0x929c, 0xffffffff, 0x00170016,
		416	0x9150, 0xffffffff, 0x96940200,
		417	0x8708, 0xffffffff, 0x00900100,
		418	0xc478, 0xffffffff, 0x00000080,
		419	0xc404, 0xffffffff, 0x0020003f,
		420	0x30, 0xffffffff, 0x0000001c,
		421	0x34, 0x000f0000, 0x000f0000,
		422	0x160c, 0xffffffff, 0x00000100,
		423	0x1024, 0xffffffff, 0x00000100,
		424	0x102c, 0x00000101, 0x00000000,
		425	0x20a8, 0xffffffff, 0x00000104,
		426	0x264c, 0x000c0000, 0x000c0000,
		427	0x2648, 0x000c0000, 0x000c0000,
		428	0x55e4, 0xff000fff, 0x00000100,
		429	0x55e8, 0x00000001, 0x00000001,
		430	0x2f50, 0x00000001, 0x00000001,
		431	0x30cc, 0xc0000fff, 0x00000104,
		432	0xc1e4, 0x00000001, 0x00000001,
		433	0xd0c0, 0xfffffff0, 0x00000100,
		434	0xd8c0, 0xfffffff0, 0x00000100
		435	};
		436
		437	static const u32 pitcairn_mgcg_cgcg_init[] =
		438	{
		439	0xc400, 0xffffffff, 0xfffffffc,
		440	0x802c, 0xffffffff, 0xe0000000,
		441	0x9a60, 0xffffffff, 0x00000100,
		442	0x92a4, 0xffffffff, 0x00000100,
		443	0xc164, 0xffffffff, 0x00000100,
		444	0x9774, 0xffffffff, 0x00000100,
		445	0x8984, 0xffffffff, 0x06000100,
		446	0x8a18, 0xffffffff, 0x00000100,
		447	0x92a0, 0xffffffff, 0x00000100,
		448	0xc380, 0xffffffff, 0x00000100,
		449	0x8b28, 0xffffffff, 0x00000100,
		450	0x9144, 0xffffffff, 0x00000100,
		451	0x8d88, 0xffffffff, 0x00000100,
		452	0x8d8c, 0xffffffff, 0x00000100,
		453	0x9030, 0xffffffff, 0x00000100,
		454	0x9034, 0xffffffff, 0x00000100,
		455	0x9038, 0xffffffff, 0x00000100,
		456	0x903c, 0xffffffff, 0x00000100,
		457	0xad80, 0xffffffff, 0x00000100,
		458	0xac54, 0xffffffff, 0x00000100,
		459	0x897c, 0xffffffff, 0x06000100,
		460	0x9868, 0xffffffff, 0x00000100,
		461	0x9510, 0xffffffff, 0x00000100,
		462	0xaf04, 0xffffffff, 0x00000100,
		463	0xae04, 0xffffffff, 0x00000100,
		464	0x949c, 0xffffffff, 0x00000100,
		465	0x802c, 0xffffffff, 0xe0000000,
		466	0x9160, 0xffffffff, 0x00010000,
		467	0x9164, 0xffffffff, 0x00030002,
		468	0x9168, 0xffffffff, 0x00040007,
		469	0x916c, 0xffffffff, 0x00060005,
		470	0x9170, 0xffffffff, 0x00090008,
		471	0x9174, 0xffffffff, 0x00020001,
		472	0x9178, 0xffffffff, 0x00040003,
		473	0x917c, 0xffffffff, 0x00000007,
		474	0x9180, 0xffffffff, 0x00060005,
		475	0x9184, 0xffffffff, 0x00090008,
		476	0x9188, 0xffffffff, 0x00030002,
		477	0x918c, 0xffffffff, 0x00050004,
		478	0x9190, 0xffffffff, 0x00000008,
		479	0x9194, 0xffffffff, 0x00070006,
		480	0x9198, 0xffffffff, 0x000a0009,
		481	0x919c, 0xffffffff, 0x00040003,
		482	0x91a0, 0xffffffff, 0x00060005,
		483	0x91a4, 0xffffffff, 0x00000009,
		484	0x91a8, 0xffffffff, 0x00080007,
		485	0x91ac, 0xffffffff, 0x000b000a,
		486	0x91b0, 0xffffffff, 0x00050004,
		487	0x91b4, 0xffffffff, 0x00070006,
		488	0x91b8, 0xffffffff, 0x0008000b,
		489	0x91bc, 0xffffffff, 0x000a0009,
		490	0x91c0, 0xffffffff, 0x000d000c,
		491	0x9200, 0xffffffff, 0x00090008,
		492	0x9204, 0xffffffff, 0x000b000a,
		493	0x9208, 0xffffffff, 0x000c000f,
		494	0x920c, 0xffffffff, 0x000e000d,
		495	0x9210, 0xffffffff, 0x00110010,
		496	0x9214, 0xffffffff, 0x000a0009,
		497	0x9218, 0xffffffff, 0x000c000b,
		498	0x921c, 0xffffffff, 0x0000000f,
		499	0x9220, 0xffffffff, 0x000e000d,
		500	0x9224, 0xffffffff, 0x00110010,
		501	0x9228, 0xffffffff, 0x000b000a,
		502	0x922c, 0xffffffff, 0x000d000c,
		503	0x9230, 0xffffffff, 0x00000010,
		504	0x9234, 0xffffffff, 0x000f000e,
		505	0x9238, 0xffffffff, 0x00120011,
		506	0x923c, 0xffffffff, 0x000c000b,
		507	0x9240, 0xffffffff, 0x000e000d,
		508	0x9244, 0xffffffff, 0x00000011,
		509	0x9248, 0xffffffff, 0x0010000f,
		510	0x924c, 0xffffffff, 0x00130012,
		511	0x9250, 0xffffffff, 0x000d000c,
		512	0x9254, 0xffffffff, 0x000f000e,
		513	0x9258, 0xffffffff, 0x00100013,
		514	0x925c, 0xffffffff, 0x00120011,
		515	0x9260, 0xffffffff, 0x00150014,
		516	0x9150, 0xffffffff, 0x96940200,
		517	0x8708, 0xffffffff, 0x00900100,
		518	0xc478, 0xffffffff, 0x00000080,
		519	0xc404, 0xffffffff, 0x0020003f,
		520	0x30, 0xffffffff, 0x0000001c,
		521	0x34, 0x000f0000, 0x000f0000,
		522	0x160c, 0xffffffff, 0x00000100,
		523	0x1024, 0xffffffff, 0x00000100,
		524	0x102c, 0x00000101, 0x00000000,
		525	0x20a8, 0xffffffff, 0x00000104,
		526	0x55e4, 0xff000fff, 0x00000100,
		527	0x55e8, 0x00000001, 0x00000001,
		528	0x2f50, 0x00000001, 0x00000001,
		529	0x30cc, 0xc0000fff, 0x00000104,
		530	0xc1e4, 0x00000001, 0x00000001,
		531	0xd0c0, 0xfffffff0, 0x00000100,
		532	0xd8c0, 0xfffffff0, 0x00000100
		533	};
		534
		535	static const u32 verde_mgcg_cgcg_init[] =
		536	{
		537	0xc400, 0xffffffff, 0xfffffffc,
		538	0x802c, 0xffffffff, 0xe0000000,
		539	0x9a60, 0xffffffff, 0x00000100,
		540	0x92a4, 0xffffffff, 0x00000100,
		541	0xc164, 0xffffffff, 0x00000100,
		542	0x9774, 0xffffffff, 0x00000100,
		543	0x8984, 0xffffffff, 0x06000100,
		544	0x8a18, 0xffffffff, 0x00000100,
		545	0x92a0, 0xffffffff, 0x00000100,
		546	0xc380, 0xffffffff, 0x00000100,
		547	0x8b28, 0xffffffff, 0x00000100,
		548	0x9144, 0xffffffff, 0x00000100,
		549	0x8d88, 0xffffffff, 0x00000100,
		550	0x8d8c, 0xffffffff, 0x00000100,
		551	0x9030, 0xffffffff, 0x00000100,
		552	0x9034, 0xffffffff, 0x00000100,
		553	0x9038, 0xffffffff, 0x00000100,
		554	0x903c, 0xffffffff, 0x00000100,
		555	0xad80, 0xffffffff, 0x00000100,
		556	0xac54, 0xffffffff, 0x00000100,
		557	0x897c, 0xffffffff, 0x06000100,
		558	0x9868, 0xffffffff, 0x00000100,
		559	0x9510, 0xffffffff, 0x00000100,
		560	0xaf04, 0xffffffff, 0x00000100,
		561	0xae04, 0xffffffff, 0x00000100,
		562	0x949c, 0xffffffff, 0x00000100,
		563	0x802c, 0xffffffff, 0xe0000000,
		564	0x9160, 0xffffffff, 0x00010000,
		565	0x9164, 0xffffffff, 0x00030002,
		566	0x9168, 0xffffffff, 0x00040007,
		567	0x916c, 0xffffffff, 0x00060005,
		568	0x9170, 0xffffffff, 0x00090008,
		569	0x9174, 0xffffffff, 0x00020001,
		570	0x9178, 0xffffffff, 0x00040003,
		571	0x917c, 0xffffffff, 0x00000007,
		572	0x9180, 0xffffffff, 0x00060005,
		573	0x9184, 0xffffffff, 0x00090008,
		574	0x9188, 0xffffffff, 0x00030002,
		575	0x918c, 0xffffffff, 0x00050004,
		576	0x9190, 0xffffffff, 0x00000008,
		577	0x9194, 0xffffffff, 0x00070006,
		578	0x9198, 0xffffffff, 0x000a0009,
		579	0x919c, 0xffffffff, 0x00040003,
		580	0x91a0, 0xffffffff, 0x00060005,
		581	0x91a4, 0xffffffff, 0x00000009,
		582	0x91a8, 0xffffffff, 0x00080007,
		583	0x91ac, 0xffffffff, 0x000b000a,
		584	0x91b0, 0xffffffff, 0x00050004,
		585	0x91b4, 0xffffffff, 0x00070006,
		586	0x91b8, 0xffffffff, 0x0008000b,
		587	0x91bc, 0xffffffff, 0x000a0009,
		588	0x91c0, 0xffffffff, 0x000d000c,
		589	0x9200, 0xffffffff, 0x00090008,
		590	0x9204, 0xffffffff, 0x000b000a,
		591	0x9208, 0xffffffff, 0x000c000f,
		592	0x920c, 0xffffffff, 0x000e000d,
		593	0x9210, 0xffffffff, 0x00110010,
		594	0x9214, 0xffffffff, 0x000a0009,
		595	0x9218, 0xffffffff, 0x000c000b,
		596	0x921c, 0xffffffff, 0x0000000f,
		597	0x9220, 0xffffffff, 0x000e000d,
		598	0x9224, 0xffffffff, 0x00110010,
		599	0x9228, 0xffffffff, 0x000b000a,
		600	0x922c, 0xffffffff, 0x000d000c,
		601	0x9230, 0xffffffff, 0x00000010,
		602	0x9234, 0xffffffff, 0x000f000e,
		603	0x9238, 0xffffffff, 0x00120011,
		604	0x923c, 0xffffffff, 0x000c000b,
		605	0x9240, 0xffffffff, 0x000e000d,
		606	0x9244, 0xffffffff, 0x00000011,
		607	0x9248, 0xffffffff, 0x0010000f,
		608	0x924c, 0xffffffff, 0x00130012,
		609	0x9250, 0xffffffff, 0x000d000c,
		610	0x9254, 0xffffffff, 0x000f000e,
		611	0x9258, 0xffffffff, 0x00100013,
		612	0x925c, 0xffffffff, 0x00120011,
		613	0x9260, 0xffffffff, 0x00150014,
		614	0x9150, 0xffffffff, 0x96940200,
		615	0x8708, 0xffffffff, 0x00900100,
		616	0xc478, 0xffffffff, 0x00000080,
		617	0xc404, 0xffffffff, 0x0020003f,
		618	0x30, 0xffffffff, 0x0000001c,
		619	0x34, 0x000f0000, 0x000f0000,
		620	0x160c, 0xffffffff, 0x00000100,
		621	0x1024, 0xffffffff, 0x00000100,
		622	0x102c, 0x00000101, 0x00000000,
		623	0x20a8, 0xffffffff, 0x00000104,
		624	0x264c, 0x000c0000, 0x000c0000,
		625	0x2648, 0x000c0000, 0x000c0000,
		626	0x55e4, 0xff000fff, 0x00000100,
		627	0x55e8, 0x00000001, 0x00000001,
		628	0x2f50, 0x00000001, 0x00000001,
		629	0x30cc, 0xc0000fff, 0x00000104,
		630	0xc1e4, 0x00000001, 0x00000001,
		631	0xd0c0, 0xfffffff0, 0x00000100,
		632	0xd8c0, 0xfffffff0, 0x00000100
		633	};
		634
		635	static const u32 oland_mgcg_cgcg_init[] =
		636	{
		637	0xc400, 0xffffffff, 0xfffffffc,
		638	0x802c, 0xffffffff, 0xe0000000,
		639	0x9a60, 0xffffffff, 0x00000100,
		640	0x92a4, 0xffffffff, 0x00000100,
		641	0xc164, 0xffffffff, 0x00000100,
		642	0x9774, 0xffffffff, 0x00000100,
		643	0x8984, 0xffffffff, 0x06000100,
		644	0x8a18, 0xffffffff, 0x00000100,
		645	0x92a0, 0xffffffff, 0x00000100,
		646	0xc380, 0xffffffff, 0x00000100,
		647	0x8b28, 0xffffffff, 0x00000100,
		648	0x9144, 0xffffffff, 0x00000100,
		649	0x8d88, 0xffffffff, 0x00000100,
		650	0x8d8c, 0xffffffff, 0x00000100,
		651	0x9030, 0xffffffff, 0x00000100,
		652	0x9034, 0xffffffff, 0x00000100,
		653	0x9038, 0xffffffff, 0x00000100,
		654	0x903c, 0xffffffff, 0x00000100,
		655	0xad80, 0xffffffff, 0x00000100,
		656	0xac54, 0xffffffff, 0x00000100,
		657	0x897c, 0xffffffff, 0x06000100,
		658	0x9868, 0xffffffff, 0x00000100,
		659	0x9510, 0xffffffff, 0x00000100,
		660	0xaf04, 0xffffffff, 0x00000100,
		661	0xae04, 0xffffffff, 0x00000100,
		662	0x949c, 0xffffffff, 0x00000100,
		663	0x802c, 0xffffffff, 0xe0000000,
		664	0x9160, 0xffffffff, 0x00010000,
		665	0x9164, 0xffffffff, 0x00030002,
		666	0x9168, 0xffffffff, 0x00040007,
		667	0x916c, 0xffffffff, 0x00060005,
		668	0x9170, 0xffffffff, 0x00090008,
		669	0x9174, 0xffffffff, 0x00020001,
		670	0x9178, 0xffffffff, 0x00040003,
		671	0x917c, 0xffffffff, 0x00000007,
		672	0x9180, 0xffffffff, 0x00060005,
		673	0x9184, 0xffffffff, 0x00090008,
		674	0x9188, 0xffffffff, 0x00030002,
		675	0x918c, 0xffffffff, 0x00050004,
		676	0x9190, 0xffffffff, 0x00000008,
		677	0x9194, 0xffffffff, 0x00070006,
		678	0x9198, 0xffffffff, 0x000a0009,
		679	0x919c, 0xffffffff, 0x00040003,
		680	0x91a0, 0xffffffff, 0x00060005,
		681	0x91a4, 0xffffffff, 0x00000009,
		682	0x91a8, 0xffffffff, 0x00080007,
		683	0x91ac, 0xffffffff, 0x000b000a,
		684	0x91b0, 0xffffffff, 0x00050004,
		685	0x91b4, 0xffffffff, 0x00070006,
		686	0x91b8, 0xffffffff, 0x0008000b,
		687	0x91bc, 0xffffffff, 0x000a0009,
		688	0x91c0, 0xffffffff, 0x000d000c,
		689	0x91c4, 0xffffffff, 0x00060005,
		690	0x91c8, 0xffffffff, 0x00080007,
		691	0x91cc, 0xffffffff, 0x0000000b,
		692	0x91d0, 0xffffffff, 0x000a0009,
		693	0x91d4, 0xffffffff, 0x000d000c,
		694	0x9150, 0xffffffff, 0x96940200,
		695	0x8708, 0xffffffff, 0x00900100,
		696	0xc478, 0xffffffff, 0x00000080,
		697	0xc404, 0xffffffff, 0x0020003f,
		698	0x30, 0xffffffff, 0x0000001c,
		699	0x34, 0x000f0000, 0x000f0000,
		700	0x160c, 0xffffffff, 0x00000100,
		701	0x1024, 0xffffffff, 0x00000100,
		702	0x102c, 0x00000101, 0x00000000,
		703	0x20a8, 0xffffffff, 0x00000104,
		704	0x264c, 0x000c0000, 0x000c0000,
		705	0x2648, 0x000c0000, 0x000c0000,
		706	0x55e4, 0xff000fff, 0x00000100,
		707	0x55e8, 0x00000001, 0x00000001,
		708	0x2f50, 0x00000001, 0x00000001,
		709	0x30cc, 0xc0000fff, 0x00000104,
		710	0xc1e4, 0x00000001, 0x00000001,
		711	0xd0c0, 0xfffffff0, 0x00000100,
		712	0xd8c0, 0xfffffff0, 0x00000100
		713	};
		714
		715	static const u32 hainan_mgcg_cgcg_init[] =
		716	{
		717	0xc400, 0xffffffff, 0xfffffffc,
		718	0x802c, 0xffffffff, 0xe0000000,
		719	0x9a60, 0xffffffff, 0x00000100,
		720	0x92a4, 0xffffffff, 0x00000100,
		721	0xc164, 0xffffffff, 0x00000100,
		722	0x9774, 0xffffffff, 0x00000100,
		723	0x8984, 0xffffffff, 0x06000100,
		724	0x8a18, 0xffffffff, 0x00000100,
		725	0x92a0, 0xffffffff, 0x00000100,
		726	0xc380, 0xffffffff, 0x00000100,
		727	0x8b28, 0xffffffff, 0x00000100,
		728	0x9144, 0xffffffff, 0x00000100,
		729	0x8d88, 0xffffffff, 0x00000100,
		730	0x8d8c, 0xffffffff, 0x00000100,
		731	0x9030, 0xffffffff, 0x00000100,
		732	0x9034, 0xffffffff, 0x00000100,
		733	0x9038, 0xffffffff, 0x00000100,
		734	0x903c, 0xffffffff, 0x00000100,
		735	0xad80, 0xffffffff, 0x00000100,
		736	0xac54, 0xffffffff, 0x00000100,
		737	0x897c, 0xffffffff, 0x06000100,
		738	0x9868, 0xffffffff, 0x00000100,
		739	0x9510, 0xffffffff, 0x00000100,
		740	0xaf04, 0xffffffff, 0x00000100,
		741	0xae04, 0xffffffff, 0x00000100,
		742	0x949c, 0xffffffff, 0x00000100,
		743	0x802c, 0xffffffff, 0xe0000000,
		744	0x9160, 0xffffffff, 0x00010000,
		745	0x9164, 0xffffffff, 0x00030002,
		746	0x9168, 0xffffffff, 0x00040007,
		747	0x916c, 0xffffffff, 0x00060005,
		748	0x9170, 0xffffffff, 0x00090008,
		749	0x9174, 0xffffffff, 0x00020001,
		750	0x9178, 0xffffffff, 0x00040003,
		751	0x917c, 0xffffffff, 0x00000007,
		752	0x9180, 0xffffffff, 0x00060005,
		753	0x9184, 0xffffffff, 0x00090008,
		754	0x9188, 0xffffffff, 0x00030002,
		755	0x918c, 0xffffffff, 0x00050004,
		756	0x9190, 0xffffffff, 0x00000008,
		757	0x9194, 0xffffffff, 0x00070006,
		758	0x9198, 0xffffffff, 0x000a0009,
		759	0x919c, 0xffffffff, 0x00040003,
		760	0x91a0, 0xffffffff, 0x00060005,
		761	0x91a4, 0xffffffff, 0x00000009,
		762	0x91a8, 0xffffffff, 0x00080007,
		763	0x91ac, 0xffffffff, 0x000b000a,
		764	0x91b0, 0xffffffff, 0x00050004,
		765	0x91b4, 0xffffffff, 0x00070006,
		766	0x91b8, 0xffffffff, 0x0008000b,
		767	0x91bc, 0xffffffff, 0x000a0009,
		768	0x91c0, 0xffffffff, 0x000d000c,
		769	0x91c4, 0xffffffff, 0x00060005,
		770	0x91c8, 0xffffffff, 0x00080007,
		771	0x91cc, 0xffffffff, 0x0000000b,
		772	0x91d0, 0xffffffff, 0x000a0009,
		773	0x91d4, 0xffffffff, 0x000d000c,
		774	0x9150, 0xffffffff, 0x96940200,
		775	0x8708, 0xffffffff, 0x00900100,
		776	0xc478, 0xffffffff, 0x00000080,
		777	0xc404, 0xffffffff, 0x0020003f,
		778	0x30, 0xffffffff, 0x0000001c,
		779	0x34, 0x000f0000, 0x000f0000,
		780	0x160c, 0xffffffff, 0x00000100,
		781	0x1024, 0xffffffff, 0x00000100,
		782	0x20a8, 0xffffffff, 0x00000104,
		783	0x264c, 0x000c0000, 0x000c0000,
		784	0x2648, 0x000c0000, 0x000c0000,
		785	0x2f50, 0x00000001, 0x00000001,
		786	0x30cc, 0xc0000fff, 0x00000104,
		787	0xc1e4, 0x00000001, 0x00000001,
		788	0xd0c0, 0xfffffff0, 0x00000100,
		789	0xd8c0, 0xfffffff0, 0x00000100
		790	};
		791
		792	static u32 verde_pg_init[] =
		793	{
		794	0x353c, 0xffffffff, 0x40000,
		795	0x3538, 0xffffffff, 0x200010ff,
		796	0x353c, 0xffffffff, 0x0,
		797	0x353c, 0xffffffff, 0x0,
		798	0x353c, 0xffffffff, 0x0,
		799	0x353c, 0xffffffff, 0x0,
		800	0x353c, 0xffffffff, 0x0,
		801	0x353c, 0xffffffff, 0x7007,
		802	0x3538, 0xffffffff, 0x300010ff,
		803	0x353c, 0xffffffff, 0x0,
		804	0x353c, 0xffffffff, 0x0,
		805	0x353c, 0xffffffff, 0x0,
		806	0x353c, 0xffffffff, 0x0,
		807	0x353c, 0xffffffff, 0x0,
		808	0x353c, 0xffffffff, 0x400000,
		809	0x3538, 0xffffffff, 0x100010ff,
		810	0x353c, 0xffffffff, 0x0,
		811	0x353c, 0xffffffff, 0x0,
		812	0x353c, 0xffffffff, 0x0,
		813	0x353c, 0xffffffff, 0x0,
		814	0x353c, 0xffffffff, 0x0,
		815	0x353c, 0xffffffff, 0x120200,
		816	0x3538, 0xffffffff, 0x500010ff,
		817	0x353c, 0xffffffff, 0x0,
		818	0x353c, 0xffffffff, 0x0,
		819	0x353c, 0xffffffff, 0x0,
		820	0x353c, 0xffffffff, 0x0,
		821	0x353c, 0xffffffff, 0x0,
		822	0x353c, 0xffffffff, 0x1e1e16,
		823	0x3538, 0xffffffff, 0x600010ff,
		824	0x353c, 0xffffffff, 0x0,
		825	0x353c, 0xffffffff, 0x0,
		826	0x353c, 0xffffffff, 0x0,
		827	0x353c, 0xffffffff, 0x0,
		828	0x353c, 0xffffffff, 0x0,
		829	0x353c, 0xffffffff, 0x171f1e,
		830	0x3538, 0xffffffff, 0x700010ff,
		831	0x353c, 0xffffffff, 0x0,
		832	0x353c, 0xffffffff, 0x0,
		833	0x353c, 0xffffffff, 0x0,
		834	0x353c, 0xffffffff, 0x0,
		835	0x353c, 0xffffffff, 0x0,
		836	0x353c, 0xffffffff, 0x0,
		837	0x3538, 0xffffffff, 0x9ff,
		838	0x3500, 0xffffffff, 0x0,
		839	0x3504, 0xffffffff, 0x10000800,
		840	0x3504, 0xffffffff, 0xf,
		841	0x3504, 0xffffffff, 0xf,
		842	0x3500, 0xffffffff, 0x4,
		843	0x3504, 0xffffffff, 0x1000051e,
		844	0x3504, 0xffffffff, 0xffff,
		845	0x3504, 0xffffffff, 0xffff,
		846	0x3500, 0xffffffff, 0x8,
		847	0x3504, 0xffffffff, 0x80500,
		848	0x3500, 0xffffffff, 0x12,
		849	0x3504, 0xffffffff, 0x9050c,
		850	0x3500, 0xffffffff, 0x1d,
		851	0x3504, 0xffffffff, 0xb052c,
		852	0x3500, 0xffffffff, 0x2a,
		853	0x3504, 0xffffffff, 0x1053e,
		854	0x3500, 0xffffffff, 0x2d,
		855	0x3504, 0xffffffff, 0x10546,
		856	0x3500, 0xffffffff, 0x30,
		857	0x3504, 0xffffffff, 0xa054e,
		858	0x3500, 0xffffffff, 0x3c,
		859	0x3504, 0xffffffff, 0x1055f,
		860	0x3500, 0xffffffff, 0x3f,
		861	0x3504, 0xffffffff, 0x10567,
		862	0x3500, 0xffffffff, 0x42,
		863	0x3504, 0xffffffff, 0x1056f,
		864	0x3500, 0xffffffff, 0x45,
		865	0x3504, 0xffffffff, 0x10572,
		866	0x3500, 0xffffffff, 0x48,
		867	0x3504, 0xffffffff, 0x20575,
		868	0x3500, 0xffffffff, 0x4c,
		869	0x3504, 0xffffffff, 0x190801,
		870	0x3500, 0xffffffff, 0x67,
		871	0x3504, 0xffffffff, 0x1082a,
		872	0x3500, 0xffffffff, 0x6a,
		873	0x3504, 0xffffffff, 0x1b082d,
		874	0x3500, 0xffffffff, 0x87,
		875	0x3504, 0xffffffff, 0x310851,
		876	0x3500, 0xffffffff, 0xba,
		877	0x3504, 0xffffffff, 0x891,
		878	0x3500, 0xffffffff, 0xbc,
		879	0x3504, 0xffffffff, 0x893,
		880	0x3500, 0xffffffff, 0xbe,
		881	0x3504, 0xffffffff, 0x20895,
		882	0x3500, 0xffffffff, 0xc2,
		883	0x3504, 0xffffffff, 0x20899,
		884	0x3500, 0xffffffff, 0xc6,
		885	0x3504, 0xffffffff, 0x2089d,
		886	0x3500, 0xffffffff, 0xca,
		887	0x3504, 0xffffffff, 0x8a1,
		888	0x3500, 0xffffffff, 0xcc,
		889	0x3504, 0xffffffff, 0x8a3,
		890	0x3500, 0xffffffff, 0xce,
		891	0x3504, 0xffffffff, 0x308a5,
		892	0x3500, 0xffffffff, 0xd3,
		893	0x3504, 0xffffffff, 0x6d08cd,
		894	0x3500, 0xffffffff, 0x142,
		895	0x3504, 0xffffffff, 0x2000095a,
		896	0x3504, 0xffffffff, 0x1,
		897	0x3500, 0xffffffff, 0x144,
		898	0x3504, 0xffffffff, 0x301f095b,
		899	0x3500, 0xffffffff, 0x165,
		900	0x3504, 0xffffffff, 0xc094d,
		901	0x3500, 0xffffffff, 0x173,
		902	0x3504, 0xffffffff, 0xf096d,
		903	0x3500, 0xffffffff, 0x184,
		904	0x3504, 0xffffffff, 0x15097f,
		905	0x3500, 0xffffffff, 0x19b,
		906	0x3504, 0xffffffff, 0xc0998,
		907	0x3500, 0xffffffff, 0x1a9,
		908	0x3504, 0xffffffff, 0x409a7,
		909	0x3500, 0xffffffff, 0x1af,
		910	0x3504, 0xffffffff, 0xcdc,
		911	0x3500, 0xffffffff, 0x1b1,
		912	0x3504, 0xffffffff, 0x800,
		913	0x3508, 0xffffffff, 0x6c9b2000,
		914	0x3510, 0xfc00, 0x2000,
		915	0x3544, 0xffffffff, 0xfc0,
		916	0x28d4, 0x00000100, 0x100
		917	};
		918
		919	static void si_init_golden_registers(struct radeon_device *rdev)
		920	{
		921	switch (rdev->family) {
		922	case CHIP_TAHITI:
		923	radeon_program_register_sequence(rdev,
		924	tahiti_golden_registers,
		925	(const u32)ARRAY_SIZE(tahiti_golden_registers));
		926	radeon_program_register_sequence(rdev,
		927	tahiti_golden_rlc_registers,
		928	(const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
		929	radeon_program_register_sequence(rdev,
		930	tahiti_mgcg_cgcg_init,
		931	(const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
		932	radeon_program_register_sequence(rdev,
		933	tahiti_golden_registers2,
		934	(const u32)ARRAY_SIZE(tahiti_golden_registers2));
		935	break;
		936	case CHIP_PITCAIRN:
		937	radeon_program_register_sequence(rdev,
		938	pitcairn_golden_registers,
		939	(const u32)ARRAY_SIZE(pitcairn_golden_registers));
		940	radeon_program_register_sequence(rdev,
		941	pitcairn_golden_rlc_registers,
		942	(const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
		943	radeon_program_register_sequence(rdev,
		944	pitcairn_mgcg_cgcg_init,
		945	(const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
		946	break;
		947	case CHIP_VERDE:
		948	radeon_program_register_sequence(rdev,
		949	verde_golden_registers,
		950	(const u32)ARRAY_SIZE(verde_golden_registers));
		951	radeon_program_register_sequence(rdev,
		952	verde_golden_rlc_registers,
		953	(const u32)ARRAY_SIZE(verde_golden_rlc_registers));
		954	radeon_program_register_sequence(rdev,
		955	verde_mgcg_cgcg_init,
		956	(const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
		957	radeon_program_register_sequence(rdev,
		958	verde_pg_init,
		959	(const u32)ARRAY_SIZE(verde_pg_init));
		960	break;
		961	case CHIP_OLAND:
		962	radeon_program_register_sequence(rdev,
		963	oland_golden_registers,
		964	(const u32)ARRAY_SIZE(oland_golden_registers));
		965	radeon_program_register_sequence(rdev,
		966	oland_golden_rlc_registers,
		967	(const u32)ARRAY_SIZE(oland_golden_rlc_registers));
		968	radeon_program_register_sequence(rdev,
		969	oland_mgcg_cgcg_init,
		970	(const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
		971	break;
		972	case CHIP_HAINAN:
		973	radeon_program_register_sequence(rdev,
		974	hainan_golden_registers,
		975	(const u32)ARRAY_SIZE(hainan_golden_registers));
		976	radeon_program_register_sequence(rdev,
		977	hainan_golden_registers2,
		978	(const u32)ARRAY_SIZE(hainan_golden_registers2));
		979	radeon_program_register_sequence(rdev,
		980	hainan_mgcg_cgcg_init,
		981	(const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
		982	break;
		983	default:
		984	break;
		985	}
		986	}
		987
		988	#define PCIE_BUS_CLK 10000
		989	#define TCLK (PCIE_BUS_CLK / 10)
		990
		991	/**
		992	* si_get_xclk - get the xclk
		993	*
		994	* @rdev: radeon_device pointer
		995	*
		996	* Returns the reference clock used by the gfx engine
		997	* (SI).
		998	*/
		999	u32 si_get_xclk(struct radeon_device *rdev)
		1000	{
		1001	u32 reference_clock = rdev->clock.spll.reference_freq;
		1002	u32 tmp;
		1003
		1004	tmp = RREG32(CG_CLKPIN_CNTL_2);
		1005	if (tmp & MUX_TCLK_TO_XCLK)
		1006	return TCLK;
		1007
		1008	tmp = RREG32(CG_CLKPIN_CNTL);
		1009	if (tmp & XTALIN_DIVIDE)
		1010	return reference_clock / 4;
		1011
		1012	return reference_clock;
		1013	}
		1014
2997	Serge	1015	/* get temperature in millidegrees */
		1016	int si_get_temp(struct radeon_device *rdev)
		1017	{
		1018	u32 temp;
		1019	int actual_temp = 0;
		1020
		1021	temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
		1022	CTF_TEMP_SHIFT;
		1023
		1024	if (temp & 0x200)
		1025	actual_temp = 255;
		1026	else
		1027	actual_temp = temp & 0x1ff;
		1028
		1029	actual_temp = (actual_temp * 1000);
		1030
		1031	return actual_temp;
		1032	}
		1033
		1034	#define TAHITI_IO_MC_REGS_SIZE 36
		1035
		1036	static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1037	{0x0000006f, 0x03044000},
		1038	{0x00000070, 0x0480c018},
		1039	{0x00000071, 0x00000040},
		1040	{0x00000072, 0x01000000},
		1041	{0x00000074, 0x000000ff},
		1042	{0x00000075, 0x00143400},
		1043	{0x00000076, 0x08ec0800},
		1044	{0x00000077, 0x040000cc},
		1045	{0x00000079, 0x00000000},
		1046	{0x0000007a, 0x21000409},
		1047	{0x0000007c, 0x00000000},
		1048	{0x0000007d, 0xe8000000},
		1049	{0x0000007e, 0x044408a8},
		1050	{0x0000007f, 0x00000003},
		1051	{0x00000080, 0x00000000},
		1052	{0x00000081, 0x01000000},
		1053	{0x00000082, 0x02000000},
		1054	{0x00000083, 0x00000000},
		1055	{0x00000084, 0xe3f3e4f4},
		1056	{0x00000085, 0x00052024},
		1057	{0x00000087, 0x00000000},
		1058	{0x00000088, 0x66036603},
		1059	{0x00000089, 0x01000000},
		1060	{0x0000008b, 0x1c0a0000},
		1061	{0x0000008c, 0xff010000},
		1062	{0x0000008e, 0xffffefff},
		1063	{0x0000008f, 0xfff3efff},
		1064	{0x00000090, 0xfff3efbf},
		1065	{0x00000094, 0x00101101},
		1066	{0x00000095, 0x00000fff},
		1067	{0x00000096, 0x00116fff},
		1068	{0x00000097, 0x60010000},
		1069	{0x00000098, 0x10010000},
		1070	{0x00000099, 0x00006000},
		1071	{0x0000009a, 0x00001000},
		1072	{0x0000009f, 0x00a77400}
		1073	};
		1074
		1075	static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1076	{0x0000006f, 0x03044000},
		1077	{0x00000070, 0x0480c018},
		1078	{0x00000071, 0x00000040},
		1079	{0x00000072, 0x01000000},
		1080	{0x00000074, 0x000000ff},
		1081	{0x00000075, 0x00143400},
		1082	{0x00000076, 0x08ec0800},
		1083	{0x00000077, 0x040000cc},
		1084	{0x00000079, 0x00000000},
		1085	{0x0000007a, 0x21000409},
		1086	{0x0000007c, 0x00000000},
		1087	{0x0000007d, 0xe8000000},
		1088	{0x0000007e, 0x044408a8},
		1089	{0x0000007f, 0x00000003},
		1090	{0x00000080, 0x00000000},
		1091	{0x00000081, 0x01000000},
		1092	{0x00000082, 0x02000000},
		1093	{0x00000083, 0x00000000},
		1094	{0x00000084, 0xe3f3e4f4},
		1095	{0x00000085, 0x00052024},
		1096	{0x00000087, 0x00000000},
		1097	{0x00000088, 0x66036603},
		1098	{0x00000089, 0x01000000},
		1099	{0x0000008b, 0x1c0a0000},
		1100	{0x0000008c, 0xff010000},
		1101	{0x0000008e, 0xffffefff},
		1102	{0x0000008f, 0xfff3efff},
		1103	{0x00000090, 0xfff3efbf},
		1104	{0x00000094, 0x00101101},
		1105	{0x00000095, 0x00000fff},
		1106	{0x00000096, 0x00116fff},
		1107	{0x00000097, 0x60010000},
		1108	{0x00000098, 0x10010000},
		1109	{0x00000099, 0x00006000},
		1110	{0x0000009a, 0x00001000},
		1111	{0x0000009f, 0x00a47400}
		1112	};
		1113
		1114	static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1115	{0x0000006f, 0x03044000},
		1116	{0x00000070, 0x0480c018},
		1117	{0x00000071, 0x00000040},
		1118	{0x00000072, 0x01000000},
		1119	{0x00000074, 0x000000ff},
		1120	{0x00000075, 0x00143400},
		1121	{0x00000076, 0x08ec0800},
		1122	{0x00000077, 0x040000cc},
		1123	{0x00000079, 0x00000000},
		1124	{0x0000007a, 0x21000409},
		1125	{0x0000007c, 0x00000000},
		1126	{0x0000007d, 0xe8000000},
		1127	{0x0000007e, 0x044408a8},
		1128	{0x0000007f, 0x00000003},
		1129	{0x00000080, 0x00000000},
		1130	{0x00000081, 0x01000000},
		1131	{0x00000082, 0x02000000},
		1132	{0x00000083, 0x00000000},
		1133	{0x00000084, 0xe3f3e4f4},
		1134	{0x00000085, 0x00052024},
		1135	{0x00000087, 0x00000000},
		1136	{0x00000088, 0x66036603},
		1137	{0x00000089, 0x01000000},
		1138	{0x0000008b, 0x1c0a0000},
		1139	{0x0000008c, 0xff010000},
		1140	{0x0000008e, 0xffffefff},
		1141	{0x0000008f, 0xfff3efff},
		1142	{0x00000090, 0xfff3efbf},
		1143	{0x00000094, 0x00101101},
		1144	{0x00000095, 0x00000fff},
		1145	{0x00000096, 0x00116fff},
		1146	{0x00000097, 0x60010000},
		1147	{0x00000098, 0x10010000},
		1148	{0x00000099, 0x00006000},
		1149	{0x0000009a, 0x00001000},
		1150	{0x0000009f, 0x00a37400}
		1151	};
		1152
3764	Serge	1153	static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1154	{0x0000006f, 0x03044000},
		1155	{0x00000070, 0x0480c018},
		1156	{0x00000071, 0x00000040},
		1157	{0x00000072, 0x01000000},
		1158	{0x00000074, 0x000000ff},
		1159	{0x00000075, 0x00143400},
		1160	{0x00000076, 0x08ec0800},
		1161	{0x00000077, 0x040000cc},
		1162	{0x00000079, 0x00000000},
		1163	{0x0000007a, 0x21000409},
		1164	{0x0000007c, 0x00000000},
		1165	{0x0000007d, 0xe8000000},
		1166	{0x0000007e, 0x044408a8},
		1167	{0x0000007f, 0x00000003},
		1168	{0x00000080, 0x00000000},
		1169	{0x00000081, 0x01000000},
		1170	{0x00000082, 0x02000000},
		1171	{0x00000083, 0x00000000},
		1172	{0x00000084, 0xe3f3e4f4},
		1173	{0x00000085, 0x00052024},
		1174	{0x00000087, 0x00000000},
		1175	{0x00000088, 0x66036603},
		1176	{0x00000089, 0x01000000},
		1177	{0x0000008b, 0x1c0a0000},
		1178	{0x0000008c, 0xff010000},
		1179	{0x0000008e, 0xffffefff},
		1180	{0x0000008f, 0xfff3efff},
		1181	{0x00000090, 0xfff3efbf},
		1182	{0x00000094, 0x00101101},
		1183	{0x00000095, 0x00000fff},
		1184	{0x00000096, 0x00116fff},
		1185	{0x00000097, 0x60010000},
		1186	{0x00000098, 0x10010000},
		1187	{0x00000099, 0x00006000},
		1188	{0x0000009a, 0x00001000},
		1189	{0x0000009f, 0x00a17730}
		1190	};
		1191
		1192	static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
		1193	{0x0000006f, 0x03044000},
		1194	{0x00000070, 0x0480c018},
		1195	{0x00000071, 0x00000040},
		1196	{0x00000072, 0x01000000},
		1197	{0x00000074, 0x000000ff},
		1198	{0x00000075, 0x00143400},
		1199	{0x00000076, 0x08ec0800},
		1200	{0x00000077, 0x040000cc},
		1201	{0x00000079, 0x00000000},
		1202	{0x0000007a, 0x21000409},
		1203	{0x0000007c, 0x00000000},
		1204	{0x0000007d, 0xe8000000},
		1205	{0x0000007e, 0x044408a8},
		1206	{0x0000007f, 0x00000003},
		1207	{0x00000080, 0x00000000},
		1208	{0x00000081, 0x01000000},
		1209	{0x00000082, 0x02000000},
		1210	{0x00000083, 0x00000000},
		1211	{0x00000084, 0xe3f3e4f4},
		1212	{0x00000085, 0x00052024},
		1213	{0x00000087, 0x00000000},
		1214	{0x00000088, 0x66036603},
		1215	{0x00000089, 0x01000000},
		1216	{0x0000008b, 0x1c0a0000},
		1217	{0x0000008c, 0xff010000},
		1218	{0x0000008e, 0xffffefff},
		1219	{0x0000008f, 0xfff3efff},
		1220	{0x00000090, 0xfff3efbf},
		1221	{0x00000094, 0x00101101},
		1222	{0x00000095, 0x00000fff},
		1223	{0x00000096, 0x00116fff},
		1224	{0x00000097, 0x60010000},
		1225	{0x00000098, 0x10010000},
		1226	{0x00000099, 0x00006000},
		1227	{0x0000009a, 0x00001000},
		1228	{0x0000009f, 0x00a07730}
		1229	};
		1230
2997	Serge	1231	/* ucode loading */
		1232	static int si_mc_load_microcode(struct radeon_device *rdev)
		1233	{
		1234	const __be32 *fw_data;
		1235	u32 running, blackout = 0;
		1236	u32 *io_mc_regs;
		1237	int i, ucode_size, regs_size;
		1238
		1239	if (!rdev->mc_fw)
		1240	return -EINVAL;
		1241
		1242	switch (rdev->family) {
		1243	case CHIP_TAHITI:
		1244	io_mc_regs = (u32 *)&tahiti_io_mc_regs;
		1245	ucode_size = SI_MC_UCODE_SIZE;
		1246	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1247	break;
		1248	case CHIP_PITCAIRN:
		1249	io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
		1250	ucode_size = SI_MC_UCODE_SIZE;
		1251	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1252	break;
		1253	case CHIP_VERDE:
		1254	default:
		1255	io_mc_regs = (u32 *)&verde_io_mc_regs;
		1256	ucode_size = SI_MC_UCODE_SIZE;
		1257	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1258	break;
3764	Serge	1259	case CHIP_OLAND:
		1260	io_mc_regs = (u32 *)&oland_io_mc_regs;
		1261	ucode_size = OLAND_MC_UCODE_SIZE;
		1262	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1263	break;
		1264	case CHIP_HAINAN:
		1265	io_mc_regs = (u32 *)&hainan_io_mc_regs;
		1266	ucode_size = OLAND_MC_UCODE_SIZE;
		1267	regs_size = TAHITI_IO_MC_REGS_SIZE;
		1268	break;
2997	Serge	1269	}
		1270
		1271	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
		1272
		1273	if (running == 0) {
		1274	if (running) {
		1275	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
		1276	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout \| 1);
		1277	}
		1278
		1279	/* reset the engine and set to writable */
		1280	WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
		1281	WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
		1282
		1283	/* load mc io regs */
		1284	for (i = 0; i < regs_size; i++) {
		1285	WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
		1286	WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
		1287	}
		1288	/* load the MC ucode */
		1289	fw_data = (const __be32 *)rdev->mc_fw->data;
		1290	for (i = 0; i < ucode_size; i++)
		1291	WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
		1292
		1293	/* put the engine back into the active state */
		1294	WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
		1295	WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
		1296	WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
		1297
		1298	/* wait for training to complete */
		1299	for (i = 0; i < rdev->usec_timeout; i++) {
		1300	if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
		1301	break;
		1302	udelay(1);
		1303	}
		1304	for (i = 0; i < rdev->usec_timeout; i++) {
		1305	if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
		1306	break;
		1307	udelay(1);
		1308	}
		1309
		1310	if (running)
		1311	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
		1312	}
		1313
		1314	return 0;
		1315	}
		1316
		1317	static int si_init_microcode(struct radeon_device *rdev)
		1318	{
		1319	struct platform_device *pdev;
		1320	const char *chip_name;
		1321	const char *rlc_chip_name;
		1322	size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
		1323	char fw_name[30];
		1324	int err;
		1325
		1326	DRM_DEBUG("\n");
		1327
		1328	pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
		1329	err = IS_ERR(pdev);
		1330	if (err) {
		1331	printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
		1332	return -EINVAL;
		1333	}
		1334
		1335	switch (rdev->family) {
		1336	case CHIP_TAHITI:
		1337	chip_name = "TAHITI";
		1338	rlc_chip_name = "TAHITI";
		1339	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1340	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1341	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1342	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1343	mc_req_size = SI_MC_UCODE_SIZE * 4;
		1344	break;
		1345	case CHIP_PITCAIRN:
		1346	chip_name = "PITCAIRN";
		1347	rlc_chip_name = "PITCAIRN";
		1348	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1349	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1350	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1351	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1352	mc_req_size = SI_MC_UCODE_SIZE * 4;
		1353	break;
		1354	case CHIP_VERDE:
		1355	chip_name = "VERDE";
		1356	rlc_chip_name = "VERDE";
		1357	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1358	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1359	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1360	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1361	mc_req_size = SI_MC_UCODE_SIZE * 4;
		1362	break;
3764	Serge	1363	case CHIP_OLAND:
		1364	chip_name = "OLAND";
		1365	rlc_chip_name = "OLAND";
		1366	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1367	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1368	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1369	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1370	mc_req_size = OLAND_MC_UCODE_SIZE * 4;
		1371	break;
		1372	case CHIP_HAINAN:
		1373	chip_name = "HAINAN";
		1374	rlc_chip_name = "HAINAN";
		1375	pfp_req_size = SI_PFP_UCODE_SIZE * 4;
		1376	me_req_size = SI_PM4_UCODE_SIZE * 4;
		1377	ce_req_size = SI_CE_UCODE_SIZE * 4;
		1378	rlc_req_size = SI_RLC_UCODE_SIZE * 4;
		1379	mc_req_size = OLAND_MC_UCODE_SIZE * 4;
		1380	break;
2997	Serge	1381	default: BUG();
		1382	}
		1383
		1384	DRM_INFO("Loading %s Microcode\n", chip_name);
		1385
		1386	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
		1387	err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
		1388	if (err)
		1389	goto out;
		1390	if (rdev->pfp_fw->size != pfp_req_size) {
		1391	printk(KERN_ERR
		1392	"si_cp: Bogus length %zu in firmware \"%s\"\n",
		1393	rdev->pfp_fw->size, fw_name);
		1394	err = -EINVAL;
		1395	goto out;
		1396	}
		1397
		1398	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
		1399	err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
		1400	if (err)
		1401	goto out;
		1402	if (rdev->me_fw->size != me_req_size) {
		1403	printk(KERN_ERR
		1404	"si_cp: Bogus length %zu in firmware \"%s\"\n",
		1405	rdev->me_fw->size, fw_name);
		1406	err = -EINVAL;
		1407	}
		1408
		1409	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
		1410	err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
		1411	if (err)
		1412	goto out;
		1413	if (rdev->ce_fw->size != ce_req_size) {
		1414	printk(KERN_ERR
		1415	"si_cp: Bogus length %zu in firmware \"%s\"\n",
		1416	rdev->ce_fw->size, fw_name);
		1417	err = -EINVAL;
		1418	}
		1419
		1420	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
		1421	err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
		1422	if (err)
		1423	goto out;
		1424	if (rdev->rlc_fw->size != rlc_req_size) {
		1425	printk(KERN_ERR
		1426	"si_rlc: Bogus length %zu in firmware \"%s\"\n",
		1427	rdev->rlc_fw->size, fw_name);
		1428	err = -EINVAL;
		1429	}
		1430
		1431	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
		1432	err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
		1433	if (err)
		1434	goto out;
		1435	if (rdev->mc_fw->size != mc_req_size) {
		1436	printk(KERN_ERR
		1437	"si_mc: Bogus length %zu in firmware \"%s\"\n",
		1438	rdev->mc_fw->size, fw_name);
		1439	err = -EINVAL;
		1440	}
		1441
		1442	out:
		1443	platform_device_unregister(pdev);
		1444
		1445	if (err) {
		1446	if (err != -EINVAL)
		1447	printk(KERN_ERR
		1448	"si_cp: Failed to load firmware \"%s\"\n",
		1449	fw_name);
		1450	release_firmware(rdev->pfp_fw);
		1451	rdev->pfp_fw = NULL;
		1452	release_firmware(rdev->me_fw);
		1453	rdev->me_fw = NULL;
		1454	release_firmware(rdev->ce_fw);
		1455	rdev->ce_fw = NULL;
		1456	release_firmware(rdev->rlc_fw);
		1457	rdev->rlc_fw = NULL;
		1458	release_firmware(rdev->mc_fw);
		1459	rdev->mc_fw = NULL;
		1460	}
		1461	return err;
		1462	}
		1463
		1464	/* watermark setup */
		1465	static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
		1466	struct radeon_crtc *radeon_crtc,
		1467	struct drm_display_mode *mode,
		1468	struct drm_display_mode *other_mode)
		1469	{
		1470	u32 tmp;
		1471	/*
		1472	* Line Buffer Setup
		1473	* There are 3 line buffers, each one shared by 2 display controllers.
		1474	* DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
		1475	* the display controllers. The paritioning is done via one of four
		1476	* preset allocations specified in bits 21:20:
		1477	* 0 - half lb
		1478	* 2 - whole lb, other crtc must be disabled
		1479	*/
		1480	/* this can get tricky if we have two large displays on a paired group
		1481	* of crtcs. Ideally for multiple large displays we'd assign them to
		1482	* non-linked crtcs for maximum line buffer allocation.
		1483	*/
		1484	if (radeon_crtc->base.enabled && mode) {
		1485	if (other_mode)
		1486	tmp = 0; /* 1/2 */
		1487	else
		1488	tmp = 2; /* whole */
		1489	} else
		1490	tmp = 0;
		1491
		1492	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
		1493	DC_LB_MEMORY_CONFIG(tmp));
		1494
		1495	if (radeon_crtc->base.enabled && mode) {
		1496	switch (tmp) {
		1497	case 0:
		1498	default:
		1499	return 4096 * 2;
		1500	case 2:
		1501	return 8192 * 2;
		1502	}
		1503	}
		1504
		1505	/* controller not enabled, so no lb used */
		1506	return 0;
		1507	}
		1508
		1509	static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
		1510	{
		1511	u32 tmp = RREG32(MC_SHARED_CHMAP);
		1512
		1513	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		1514	case 0:
		1515	default:
		1516	return 1;
		1517	case 1:
		1518	return 2;
		1519	case 2:
		1520	return 4;
		1521	case 3:
		1522	return 8;
		1523	case 4:
		1524	return 3;
		1525	case 5:
		1526	return 6;
		1527	case 6:
		1528	return 10;
		1529	case 7:
		1530	return 12;
		1531	case 8:
		1532	return 16;
		1533	}
		1534	}
		1535
		1536	struct dce6_wm_params {
		1537	u32 dram_channels; /* number of dram channels */
		1538	u32 yclk; /* bandwidth per dram data pin in kHz */
		1539	u32 sclk; /* engine clock in kHz */
		1540	u32 disp_clk; /* display clock in kHz */
		1541	u32 src_width; /* viewport width */
		1542	u32 active_time; /* active display time in ns */
		1543	u32 blank_time; /* blank time in ns */
		1544	bool interlaced; /* mode is interlaced */
		1545	fixed20_12 vsc; /* vertical scale ratio */
		1546	u32 num_heads; /* number of active crtcs */
		1547	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
		1548	u32 lb_size; /* line buffer allocated to pipe */
		1549	u32 vtaps; /* vertical scaler taps */
		1550	};
		1551
		1552	static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
		1553	{
		1554	/* Calculate raw DRAM Bandwidth */
		1555	fixed20_12 dram_efficiency; /* 0.7 */
		1556	fixed20_12 yclk, dram_channels, bandwidth;
		1557	fixed20_12 a;
		1558
		1559	a.full = dfixed_const(1000);
		1560	yclk.full = dfixed_const(wm->yclk);
		1561	yclk.full = dfixed_div(yclk, a);
		1562	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		1563	a.full = dfixed_const(10);
		1564	dram_efficiency.full = dfixed_const(7);
		1565	dram_efficiency.full = dfixed_div(dram_efficiency, a);
		1566	bandwidth.full = dfixed_mul(dram_channels, yclk);
		1567	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
		1568
		1569	return dfixed_trunc(bandwidth);
		1570	}
		1571
		1572	static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
		1573	{
		1574	/* Calculate DRAM Bandwidth and the part allocated to display. */
		1575	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
		1576	fixed20_12 yclk, dram_channels, bandwidth;
		1577	fixed20_12 a;
		1578
		1579	a.full = dfixed_const(1000);
		1580	yclk.full = dfixed_const(wm->yclk);
		1581	yclk.full = dfixed_div(yclk, a);
		1582	dram_channels.full = dfixed_const(wm->dram_channels * 4);
		1583	a.full = dfixed_const(10);
		1584	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
		1585	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
		1586	bandwidth.full = dfixed_mul(dram_channels, yclk);
		1587	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
		1588
		1589	return dfixed_trunc(bandwidth);
		1590	}
		1591
		1592	static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
		1593	{
		1594	/* Calculate the display Data return Bandwidth */
		1595	fixed20_12 return_efficiency; /* 0.8 */
		1596	fixed20_12 sclk, bandwidth;
		1597	fixed20_12 a;
		1598
		1599	a.full = dfixed_const(1000);
		1600	sclk.full = dfixed_const(wm->sclk);
		1601	sclk.full = dfixed_div(sclk, a);
		1602	a.full = dfixed_const(10);
		1603	return_efficiency.full = dfixed_const(8);
		1604	return_efficiency.full = dfixed_div(return_efficiency, a);
		1605	a.full = dfixed_const(32);
		1606	bandwidth.full = dfixed_mul(a, sclk);
		1607	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
		1608
		1609	return dfixed_trunc(bandwidth);
		1610	}
		1611
		1612	static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
		1613	{
		1614	return 32;
		1615	}
		1616
		1617	static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
		1618	{
		1619	/* Calculate the DMIF Request Bandwidth */
		1620	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
		1621	fixed20_12 disp_clk, sclk, bandwidth;
		1622	fixed20_12 a, b1, b2;
		1623	u32 min_bandwidth;
		1624
		1625	a.full = dfixed_const(1000);
		1626	disp_clk.full = dfixed_const(wm->disp_clk);
		1627	disp_clk.full = dfixed_div(disp_clk, a);
		1628	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
		1629	b1.full = dfixed_mul(a, disp_clk);
		1630
		1631	a.full = dfixed_const(1000);
		1632	sclk.full = dfixed_const(wm->sclk);
		1633	sclk.full = dfixed_div(sclk, a);
		1634	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
		1635	b2.full = dfixed_mul(a, sclk);
		1636
		1637	a.full = dfixed_const(10);
		1638	disp_clk_request_efficiency.full = dfixed_const(8);
		1639	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
		1640
		1641	min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
		1642
		1643	a.full = dfixed_const(min_bandwidth);
		1644	bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
		1645
		1646	return dfixed_trunc(bandwidth);
		1647	}
		1648
		1649	static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
		1650	{
		1651	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
		1652	u32 dram_bandwidth = dce6_dram_bandwidth(wm);
		1653	u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
		1654	u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
		1655
		1656	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
		1657	}
		1658
		1659	static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
		1660	{
		1661	/* Calculate the display mode Average Bandwidth
		1662	* DisplayMode should contain the source and destination dimensions,
		1663	* timing, etc.
		1664	*/
		1665	fixed20_12 bpp;
		1666	fixed20_12 line_time;
		1667	fixed20_12 src_width;
		1668	fixed20_12 bandwidth;
		1669	fixed20_12 a;
		1670
		1671	a.full = dfixed_const(1000);
		1672	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
		1673	line_time.full = dfixed_div(line_time, a);
		1674	bpp.full = dfixed_const(wm->bytes_per_pixel);
		1675	src_width.full = dfixed_const(wm->src_width);
		1676	bandwidth.full = dfixed_mul(src_width, bpp);
		1677	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
		1678	bandwidth.full = dfixed_div(bandwidth, line_time);
		1679
		1680	return dfixed_trunc(bandwidth);
		1681	}
		1682
		1683	static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
		1684	{
		1685	/* First calcualte the latency in ns */
		1686	u32 mc_latency = 2000; /* 2000 ns. */
		1687	u32 available_bandwidth = dce6_available_bandwidth(wm);
		1688	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
		1689	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
		1690	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
		1691	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
		1692	(wm->num_heads * cursor_line_pair_return_time);
		1693	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
		1694	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
		1695	u32 tmp, dmif_size = 12288;
		1696	fixed20_12 a, b, c;
		1697
		1698	if (wm->num_heads == 0)
		1699	return 0;
		1700
		1701	a.full = dfixed_const(2);
		1702	b.full = dfixed_const(1);
		1703	if ((wm->vsc.full > a.full) \|\|
		1704	((wm->vsc.full > b.full) && (wm->vtaps >= 3)) \|\|
		1705	(wm->vtaps >= 5) \|\|
		1706	((wm->vsc.full >= a.full) && wm->interlaced))
		1707	max_src_lines_per_dst_line = 4;
		1708	else
		1709	max_src_lines_per_dst_line = 2;
		1710
		1711	a.full = dfixed_const(available_bandwidth);
		1712	b.full = dfixed_const(wm->num_heads);
		1713	a.full = dfixed_div(a, b);
		1714
		1715	b.full = dfixed_const(mc_latency + 512);
		1716	c.full = dfixed_const(wm->disp_clk);
		1717	b.full = dfixed_div(b, c);
		1718
		1719	c.full = dfixed_const(dmif_size);
		1720	b.full = dfixed_div(c, b);
		1721
		1722	tmp = min(dfixed_trunc(a), dfixed_trunc(b));
		1723
		1724	b.full = dfixed_const(1000);
		1725	c.full = dfixed_const(wm->disp_clk);
		1726	b.full = dfixed_div(c, b);
		1727	c.full = dfixed_const(wm->bytes_per_pixel);
		1728	b.full = dfixed_mul(b, c);
		1729
		1730	lb_fill_bw = min(tmp, dfixed_trunc(b));
		1731
		1732	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
		1733	b.full = dfixed_const(1000);
		1734	c.full = dfixed_const(lb_fill_bw);
		1735	b.full = dfixed_div(c, b);
		1736	a.full = dfixed_div(a, b);
		1737	line_fill_time = dfixed_trunc(a);
		1738
		1739	if (line_fill_time < wm->active_time)
		1740	return latency;
		1741	else
		1742	return latency + (line_fill_time - wm->active_time);
		1743
		1744	}
		1745
		1746	static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
		1747	{
		1748	if (dce6_average_bandwidth(wm) <=
		1749	(dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
		1750	return true;
		1751	else
		1752	return false;
		1753	};
		1754
		1755	static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
		1756	{
		1757	if (dce6_average_bandwidth(wm) <=
		1758	(dce6_available_bandwidth(wm) / wm->num_heads))
		1759	return true;
		1760	else
		1761	return false;
		1762	};
		1763
		1764	static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
		1765	{
		1766	u32 lb_partitions = wm->lb_size / wm->src_width;
		1767	u32 line_time = wm->active_time + wm->blank_time;
		1768	u32 latency_tolerant_lines;
		1769	u32 latency_hiding;
		1770	fixed20_12 a;
		1771
		1772	a.full = dfixed_const(1);
		1773	if (wm->vsc.full > a.full)
		1774	latency_tolerant_lines = 1;
		1775	else {
		1776	if (lb_partitions <= (wm->vtaps + 1))
		1777	latency_tolerant_lines = 1;
		1778	else
		1779	latency_tolerant_lines = 2;
		1780	}
		1781
		1782	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
		1783
		1784	if (dce6_latency_watermark(wm) <= latency_hiding)
		1785	return true;
		1786	else
		1787	return false;
		1788	}
		1789
		1790	static void dce6_program_watermarks(struct radeon_device *rdev,
		1791	struct radeon_crtc *radeon_crtc,
		1792	u32 lb_size, u32 num_heads)
		1793	{
		1794	struct drm_display_mode *mode = &radeon_crtc->base.mode;
		1795	struct dce6_wm_params wm;
		1796	u32 pixel_period;
		1797	u32 line_time = 0;
		1798	u32 latency_watermark_a = 0, latency_watermark_b = 0;
		1799	u32 priority_a_mark = 0, priority_b_mark = 0;
		1800	u32 priority_a_cnt = PRIORITY_OFF;
		1801	u32 priority_b_cnt = PRIORITY_OFF;
		1802	u32 tmp, arb_control3;
		1803	fixed20_12 a, b, c;
		1804
		1805	if (radeon_crtc->base.enabled && num_heads && mode) {
		1806	pixel_period = 1000000 / (u32)mode->clock;
		1807	line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
		1808	priority_a_cnt = 0;
		1809	priority_b_cnt = 0;
		1810
		1811	wm.yclk = rdev->pm.current_mclk * 10;
		1812	wm.sclk = rdev->pm.current_sclk * 10;
		1813	wm.disp_clk = mode->clock;
		1814	wm.src_width = mode->crtc_hdisplay;
		1815	wm.active_time = mode->crtc_hdisplay * pixel_period;
		1816	wm.blank_time = line_time - wm.active_time;
		1817	wm.interlaced = false;
		1818	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		1819	wm.interlaced = true;
		1820	wm.vsc = radeon_crtc->vsc;
		1821	wm.vtaps = 1;
		1822	if (radeon_crtc->rmx_type != RMX_OFF)
		1823	wm.vtaps = 2;
		1824	wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
		1825	wm.lb_size = lb_size;
		1826	if (rdev->family == CHIP_ARUBA)
		1827	wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
		1828	else
		1829	wm.dram_channels = si_get_number_of_dram_channels(rdev);
		1830	wm.num_heads = num_heads;
		1831
		1832	/* set for high clocks */
		1833	latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535);
		1834	/* set for low clocks */
		1835	/* wm.yclk = low clk; wm.sclk = low clk */
		1836	latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535);
		1837
		1838	/* possibly force display priority to high */
		1839	/* should really do this at mode validation time... */
		1840	if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) \|\|
		1841	!dce6_average_bandwidth_vs_available_bandwidth(&wm) \|\|
		1842	!dce6_check_latency_hiding(&wm) \|\|
		1843	(rdev->disp_priority == 2)) {
		1844	DRM_DEBUG_KMS("force priority to high\n");
		1845	priority_a_cnt \|= PRIORITY_ALWAYS_ON;
		1846	priority_b_cnt \|= PRIORITY_ALWAYS_ON;
		1847	}
		1848
		1849	a.full = dfixed_const(1000);
		1850	b.full = dfixed_const(mode->clock);
		1851	b.full = dfixed_div(b, a);
		1852	c.full = dfixed_const(latency_watermark_a);
		1853	c.full = dfixed_mul(c, b);
		1854	c.full = dfixed_mul(c, radeon_crtc->hsc);
		1855	c.full = dfixed_div(c, a);
		1856	a.full = dfixed_const(16);
		1857	c.full = dfixed_div(c, a);
		1858	priority_a_mark = dfixed_trunc(c);
		1859	priority_a_cnt \|= priority_a_mark & PRIORITY_MARK_MASK;
		1860
		1861	a.full = dfixed_const(1000);
		1862	b.full = dfixed_const(mode->clock);
		1863	b.full = dfixed_div(b, a);
		1864	c.full = dfixed_const(latency_watermark_b);
		1865	c.full = dfixed_mul(c, b);
		1866	c.full = dfixed_mul(c, radeon_crtc->hsc);
		1867	c.full = dfixed_div(c, a);
		1868	a.full = dfixed_const(16);
		1869	c.full = dfixed_div(c, a);
		1870	priority_b_mark = dfixed_trunc(c);
		1871	priority_b_cnt \|= priority_b_mark & PRIORITY_MARK_MASK;
		1872	}
		1873
		1874	/* select wm A */
		1875	arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
		1876	tmp = arb_control3;
		1877	tmp &= ~LATENCY_WATERMARK_MASK(3);
		1878	tmp \|= LATENCY_WATERMARK_MASK(1);
		1879	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
		1880	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
		1881	(LATENCY_LOW_WATERMARK(latency_watermark_a) \|
		1882	LATENCY_HIGH_WATERMARK(line_time)));
		1883	/* select wm B */
		1884	tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
		1885	tmp &= ~LATENCY_WATERMARK_MASK(3);
		1886	tmp \|= LATENCY_WATERMARK_MASK(2);
		1887	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
		1888	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
		1889	(LATENCY_LOW_WATERMARK(latency_watermark_b) \|
		1890	LATENCY_HIGH_WATERMARK(line_time)));
		1891	/* restore original selection */
		1892	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
		1893
		1894	/* write the priority marks */
		1895	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
		1896	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
		1897
		1898	}
		1899
		1900	void dce6_bandwidth_update(struct radeon_device *rdev)
		1901	{
		1902	struct drm_display_mode *mode0 = NULL;
		1903	struct drm_display_mode *mode1 = NULL;
		1904	u32 num_heads = 0, lb_size;
		1905	int i;
		1906
		1907	radeon_update_display_priority(rdev);
		1908
		1909	for (i = 0; i < rdev->num_crtc; i++) {
		1910	if (rdev->mode_info.crtcs[i]->base.enabled)
		1911	num_heads++;
		1912	}
		1913	for (i = 0; i < rdev->num_crtc; i += 2) {
		1914	mode0 = &rdev->mode_info.crtcs[i]->base.mode;
		1915	mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
		1916	lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
		1917	dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
		1918	lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
		1919	dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
		1920	}
		1921	}
		1922
		1923	/*
		1924	* Core functions
		1925	*/
		1926	static void si_tiling_mode_table_init(struct radeon_device *rdev)
		1927	{
		1928	const u32 num_tile_mode_states = 32;
		1929	u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
		1930
		1931	switch (rdev->config.si.mem_row_size_in_kb) {
		1932	case 1:
		1933	split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
		1934	break;
		1935	case 2:
		1936	default:
		1937	split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
		1938	break;
		1939	case 4:
		1940	split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
		1941	break;
		1942	}
		1943
		1944	if ((rdev->family == CHIP_TAHITI) \|\|
		1945	(rdev->family == CHIP_PITCAIRN)) {
		1946	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
		1947	switch (reg_offset) {
		1948	case 0: /* non-AA compressed depth or any compressed stencil */
		1949	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		1950	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		1951	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		1952	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		1953	NUM_BANKS(ADDR_SURF_16_BANK) \|
		1954	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		1955	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		1956	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		1957	break;
		1958	case 1: /* 2xAA/4xAA compressed depth only */
		1959	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		1960	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		1961	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		1962	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		1963	NUM_BANKS(ADDR_SURF_16_BANK) \|
		1964	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		1965	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		1966	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		1967	break;
		1968	case 2: /* 8xAA compressed depth only */
		1969	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		1970	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		1971	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		1972	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		1973	NUM_BANKS(ADDR_SURF_16_BANK) \|
		1974	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		1975	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		1976	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		1977	break;
		1978	case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
		1979	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		1980	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		1981	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		1982	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		1983	NUM_BANKS(ADDR_SURF_16_BANK) \|
		1984	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		1985	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		1986	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		1987	break;
		1988	case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
		1989	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		1990	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		1991	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		1992	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		1993	NUM_BANKS(ADDR_SURF_16_BANK) \|
		1994	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		1995	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		1996	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		1997	break;
		1998	case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
		1999	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2000	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2001	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2002	TILE_SPLIT(split_equal_to_row_size) \|
		2003	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2004	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2005	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2006	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2007	break;
		2008	case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
		2009	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2010	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2011	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2012	TILE_SPLIT(split_equal_to_row_size) \|
		2013	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2014	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2015	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2016	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2017	break;
		2018	case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
		2019	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2020	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2021	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2022	TILE_SPLIT(split_equal_to_row_size) \|
		2023	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2024	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2025	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2026	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2027	break;
		2028	case 8: /* 1D and 1D Array Surfaces */
		2029	gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) \|
		2030	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2031	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2032	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2033	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2034	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2035	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2036	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2037	break;
		2038	case 9: /* Displayable maps. */
		2039	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2040	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2041	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2042	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2043	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2044	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2045	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2046	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2047	break;
		2048	case 10: /* Display 8bpp. */
		2049	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2050	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2051	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2052	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2053	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2054	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2055	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2056	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2057	break;
		2058	case 11: /* Display 16bpp. */
		2059	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2060	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2061	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2062	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2063	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2064	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2065	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2066	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2067	break;
		2068	case 12: /* Display 32bpp. */
		2069	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2070	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2071	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2072	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2073	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2074	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2075	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2076	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2077	break;
		2078	case 13: /* Thin. */
		2079	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2080	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2081	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2082	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2083	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2084	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2085	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2086	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2087	break;
		2088	case 14: /* Thin 8 bpp. */
		2089	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2090	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2091	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2092	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2093	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2094	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2095	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2096	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2097	break;
		2098	case 15: /* Thin 16 bpp. */
		2099	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2100	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2101	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2102	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2103	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2104	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2105	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2106	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2107	break;
		2108	case 16: /* Thin 32 bpp. */
		2109	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2110	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2111	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2112	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2113	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2114	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2115	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2116	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2117	break;
		2118	case 17: /* Thin 64 bpp. */
		2119	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2120	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2121	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2122	TILE_SPLIT(split_equal_to_row_size) \|
		2123	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2124	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2125	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2126	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2127	break;
		2128	case 21: /* 8 bpp PRT. */
		2129	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2130	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2131	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2132	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2133	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2134	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) \|
		2135	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2136	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2137	break;
		2138	case 22: /* 16 bpp PRT */
		2139	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2140	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2141	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2142	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2143	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2144	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2145	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2146	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2147	break;
		2148	case 23: /* 32 bpp PRT */
		2149	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2150	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2151	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2152	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2153	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2154	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2155	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2156	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2157	break;
		2158	case 24: /* 64 bpp PRT */
		2159	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2160	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2161	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2162	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2163	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2164	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2165	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2166	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2167	break;
		2168	case 25: /* 128 bpp PRT */
		2169	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2170	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2171	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2172	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) \|
		2173	NUM_BANKS(ADDR_SURF_8_BANK) \|
		2174	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2175	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2176	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2177	break;
		2178	default:
		2179	gb_tile_moden = 0;
		2180	break;
		2181	}
3764	Serge	2182	rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2997	Serge	2183	WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
		2184	}
3764	Serge	2185	} else if ((rdev->family == CHIP_VERDE) \|\|
		2186	(rdev->family == CHIP_OLAND) \|\|
		2187	(rdev->family == CHIP_HAINAN)) {
2997	Serge	2188	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
		2189	switch (reg_offset) {
		2190	case 0: /* non-AA compressed depth or any compressed stencil */
		2191	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2192	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2193	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2194	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2195	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2196	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2197	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2198	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2199	break;
		2200	case 1: /* 2xAA/4xAA compressed depth only */
		2201	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2202	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2203	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2204	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		2205	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2206	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2207	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2208	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2209	break;
		2210	case 2: /* 8xAA compressed depth only */
		2211	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2212	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2213	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2214	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2215	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2216	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2217	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2218	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2219	break;
		2220	case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
		2221	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2222	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2223	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2224	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) \|
		2225	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2226	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2227	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2228	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2229	break;
		2230	case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
		2231	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2232	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2233	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2234	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2235	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2236	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2237	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2238	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2239	break;
		2240	case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
		2241	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2242	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2243	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2244	TILE_SPLIT(split_equal_to_row_size) \|
		2245	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2246	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2247	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2248	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2249	break;
		2250	case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
		2251	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2252	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2253	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2254	TILE_SPLIT(split_equal_to_row_size) \|
		2255	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2256	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2257	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2258	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2259	break;
		2260	case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
		2261	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2262	MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) \|
		2263	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2264	TILE_SPLIT(split_equal_to_row_size) \|
		2265	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2266	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2267	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2268	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2269	break;
		2270	case 8: /* 1D and 1D Array Surfaces */
		2271	gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) \|
		2272	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2273	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2274	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2275	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2276	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2277	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2278	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2279	break;
		2280	case 9: /* Displayable maps. */
		2281	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2282	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2283	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2284	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2285	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2286	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2287	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2288	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2289	break;
		2290	case 10: /* Display 8bpp. */
		2291	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2292	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2293	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2294	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2295	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2296	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2297	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2298	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2299	break;
		2300	case 11: /* Display 16bpp. */
		2301	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2302	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2303	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2304	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2305	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2306	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2307	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2308	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2309	break;
		2310	case 12: /* Display 32bpp. */
		2311	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2312	MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) \|
		2313	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2314	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2315	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2316	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2317	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2318	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2319	break;
		2320	case 13: /* Thin. */
		2321	gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) \|
		2322	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2323	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2324	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) \|
		2325	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2326	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2327	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2328	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2329	break;
		2330	case 14: /* Thin 8 bpp. */
		2331	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2332	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2333	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2334	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2335	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2336	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2337	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2338	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2339	break;
		2340	case 15: /* Thin 16 bpp. */
		2341	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2342	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2343	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2344	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2345	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2346	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2347	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2348	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2349	break;
		2350	case 16: /* Thin 32 bpp. */
		2351	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2352	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2353	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2354	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2355	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2356	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2357	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2358	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2359	break;
		2360	case 17: /* Thin 64 bpp. */
		2361	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2362	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2363	PIPE_CONFIG(ADDR_SURF_P4_8x16) \|
		2364	TILE_SPLIT(split_equal_to_row_size) \|
		2365	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2366	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2367	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2368	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2369	break;
		2370	case 21: /* 8 bpp PRT. */
		2371	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2372	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2373	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2374	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2375	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2376	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) \|
		2377	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2378	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2379	break;
		2380	case 22: /* 16 bpp PRT */
		2381	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2382	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2383	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2384	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2385	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2386	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2387	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) \|
		2388	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
		2389	break;
		2390	case 23: /* 32 bpp PRT */
		2391	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2392	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2393	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2394	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) \|
		2395	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2396	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2397	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) \|
		2398	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2399	break;
		2400	case 24: /* 64 bpp PRT */
		2401	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2402	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2403	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2404	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) \|
		2405	NUM_BANKS(ADDR_SURF_16_BANK) \|
		2406	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2407	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2408	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
		2409	break;
		2410	case 25: /* 128 bpp PRT */
		2411	gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) \|
		2412	MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) \|
		2413	PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) \|
		2414	TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) \|
		2415	NUM_BANKS(ADDR_SURF_8_BANK) \|
		2416	BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) \|
		2417	BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) \|
		2418	MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
		2419	break;
		2420	default:
		2421	gb_tile_moden = 0;
		2422	break;
		2423	}
3764	Serge	2424	rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2997	Serge	2425	WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
		2426	}
		2427	} else
		2428	DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
		2429	}
		2430
		2431	static void si_select_se_sh(struct radeon_device *rdev,
		2432	u32 se_num, u32 sh_num)
		2433	{
		2434	u32 data = INSTANCE_BROADCAST_WRITES;
		2435
		2436	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3764	Serge	2437	data \|= SH_BROADCAST_WRITES \| SE_BROADCAST_WRITES;
2997	Serge	2438	else if (se_num == 0xffffffff)
		2439	data \|= SE_BROADCAST_WRITES \| SH_INDEX(sh_num);
		2440	else if (sh_num == 0xffffffff)
		2441	data \|= SH_BROADCAST_WRITES \| SE_INDEX(se_num);
		2442	else
		2443	data \|= SH_INDEX(sh_num) \| SE_INDEX(se_num);
		2444	WREG32(GRBM_GFX_INDEX, data);
		2445	}
		2446
		2447	static u32 si_create_bitmask(u32 bit_width)
		2448	{
		2449	u32 i, mask = 0;
		2450
		2451	for (i = 0; i < bit_width; i++) {
		2452	mask <<= 1;
		2453	mask \|= 1;
		2454	}
		2455	return mask;
		2456	}
		2457
		2458	static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
		2459	{
		2460	u32 data, mask;
		2461
		2462	data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
		2463	if (data & 1)
		2464	data &= INACTIVE_CUS_MASK;
		2465	else
		2466	data = 0;
		2467	data \|= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
		2468
		2469	data >>= INACTIVE_CUS_SHIFT;
		2470
		2471	mask = si_create_bitmask(cu_per_sh);
		2472
		2473	return ~data & mask;
		2474	}
		2475
		2476	static void si_setup_spi(struct radeon_device *rdev,
		2477	u32 se_num, u32 sh_per_se,
		2478	u32 cu_per_sh)
		2479	{
		2480	int i, j, k;
		2481	u32 data, mask, active_cu;
		2482
		2483	for (i = 0; i < se_num; i++) {
		2484	for (j = 0; j < sh_per_se; j++) {
		2485	si_select_se_sh(rdev, i, j);
		2486	data = RREG32(SPI_STATIC_THREAD_MGMT_3);
		2487	active_cu = si_get_cu_enabled(rdev, cu_per_sh);
		2488
		2489	mask = 1;
		2490	for (k = 0; k < 16; k++) {
		2491	mask <<= k;
		2492	if (active_cu & mask) {
		2493	data &= ~mask;
		2494	WREG32(SPI_STATIC_THREAD_MGMT_3, data);
		2495	break;
		2496	}
		2497	}
		2498	}
		2499	}
		2500	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		2501	}
		2502
		2503	static u32 si_get_rb_disabled(struct radeon_device *rdev,
		2504	u32 max_rb_num, u32 se_num,
		2505	u32 sh_per_se)
		2506	{
		2507	u32 data, mask;
		2508
		2509	data = RREG32(CC_RB_BACKEND_DISABLE);
		2510	if (data & 1)
		2511	data &= BACKEND_DISABLE_MASK;
		2512	else
		2513	data = 0;
		2514	data \|= RREG32(GC_USER_RB_BACKEND_DISABLE);
		2515
		2516	data >>= BACKEND_DISABLE_SHIFT;
		2517
		2518	mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
		2519
		2520	return data & mask;
		2521	}
		2522
		2523	static void si_setup_rb(struct radeon_device *rdev,
		2524	u32 se_num, u32 sh_per_se,
		2525	u32 max_rb_num)
		2526	{
		2527	int i, j;
		2528	u32 data, mask;
		2529	u32 disabled_rbs = 0;
		2530	u32 enabled_rbs = 0;
		2531
		2532	for (i = 0; i < se_num; i++) {
		2533	for (j = 0; j < sh_per_se; j++) {
		2534	si_select_se_sh(rdev, i, j);
		2535	data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
		2536	disabled_rbs \|= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
		2537	}
		2538	}
		2539	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		2540
		2541	mask = 1;
		2542	for (i = 0; i < max_rb_num; i++) {
		2543	if (!(disabled_rbs & mask))
		2544	enabled_rbs \|= mask;
		2545	mask <<= 1;
		2546	}
		2547
		2548	for (i = 0; i < se_num; i++) {
		2549	si_select_se_sh(rdev, i, 0xffffffff);
		2550	data = 0;
		2551	for (j = 0; j < sh_per_se; j++) {
		2552	switch (enabled_rbs & 3) {
		2553	case 1:
		2554	data \|= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
		2555	break;
		2556	case 2:
		2557	data \|= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
		2558	break;
		2559	case 3:
		2560	default:
		2561	data \|= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
		2562	break;
		2563	}
		2564	enabled_rbs >>= 2;
		2565	}
		2566	WREG32(PA_SC_RASTER_CONFIG, data);
		2567	}
		2568	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
		2569	}
		2570
		2571	static void si_gpu_init(struct radeon_device *rdev)
		2572	{
		2573	u32 gb_addr_config = 0;
		2574	u32 mc_shared_chmap, mc_arb_ramcfg;
		2575	u32 sx_debug_1;
		2576	u32 hdp_host_path_cntl;
		2577	u32 tmp;
		2578	int i, j;
		2579
		2580	switch (rdev->family) {
		2581	case CHIP_TAHITI:
		2582	rdev->config.si.max_shader_engines = 2;
		2583	rdev->config.si.max_tile_pipes = 12;
		2584	rdev->config.si.max_cu_per_sh = 8;
		2585	rdev->config.si.max_sh_per_se = 2;
		2586	rdev->config.si.max_backends_per_se = 4;
		2587	rdev->config.si.max_texture_channel_caches = 12;
		2588	rdev->config.si.max_gprs = 256;
		2589	rdev->config.si.max_gs_threads = 32;
		2590	rdev->config.si.max_hw_contexts = 8;
		2591
		2592	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		2593	rdev->config.si.sc_prim_fifo_size_backend = 0x100;
		2594	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		2595	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		2596	gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
		2597	break;
		2598	case CHIP_PITCAIRN:
		2599	rdev->config.si.max_shader_engines = 2;
		2600	rdev->config.si.max_tile_pipes = 8;
		2601	rdev->config.si.max_cu_per_sh = 5;
		2602	rdev->config.si.max_sh_per_se = 2;
		2603	rdev->config.si.max_backends_per_se = 4;
		2604	rdev->config.si.max_texture_channel_caches = 8;
		2605	rdev->config.si.max_gprs = 256;
		2606	rdev->config.si.max_gs_threads = 32;
		2607	rdev->config.si.max_hw_contexts = 8;
		2608
		2609	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		2610	rdev->config.si.sc_prim_fifo_size_backend = 0x100;
		2611	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		2612	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		2613	gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
		2614	break;
		2615	case CHIP_VERDE:
		2616	default:
		2617	rdev->config.si.max_shader_engines = 1;
		2618	rdev->config.si.max_tile_pipes = 4;
3764	Serge	2619	rdev->config.si.max_cu_per_sh = 5;
2997	Serge	2620	rdev->config.si.max_sh_per_se = 2;
		2621	rdev->config.si.max_backends_per_se = 4;
		2622	rdev->config.si.max_texture_channel_caches = 4;
		2623	rdev->config.si.max_gprs = 256;
		2624	rdev->config.si.max_gs_threads = 32;
		2625	rdev->config.si.max_hw_contexts = 8;
		2626
		2627	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		2628	rdev->config.si.sc_prim_fifo_size_backend = 0x40;
		2629	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		2630	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		2631	gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
		2632	break;
3764	Serge	2633	case CHIP_OLAND:
		2634	rdev->config.si.max_shader_engines = 1;
		2635	rdev->config.si.max_tile_pipes = 4;
		2636	rdev->config.si.max_cu_per_sh = 6;
		2637	rdev->config.si.max_sh_per_se = 1;
		2638	rdev->config.si.max_backends_per_se = 2;
		2639	rdev->config.si.max_texture_channel_caches = 4;
		2640	rdev->config.si.max_gprs = 256;
		2641	rdev->config.si.max_gs_threads = 16;
		2642	rdev->config.si.max_hw_contexts = 8;
		2643
		2644	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		2645	rdev->config.si.sc_prim_fifo_size_backend = 0x40;
		2646	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		2647	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		2648	gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
		2649	break;
		2650	case CHIP_HAINAN:
		2651	rdev->config.si.max_shader_engines = 1;
		2652	rdev->config.si.max_tile_pipes = 4;
		2653	rdev->config.si.max_cu_per_sh = 5;
		2654	rdev->config.si.max_sh_per_se = 1;
		2655	rdev->config.si.max_backends_per_se = 1;
		2656	rdev->config.si.max_texture_channel_caches = 2;
		2657	rdev->config.si.max_gprs = 256;
		2658	rdev->config.si.max_gs_threads = 16;
		2659	rdev->config.si.max_hw_contexts = 8;
		2660
		2661	rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
		2662	rdev->config.si.sc_prim_fifo_size_backend = 0x40;
		2663	rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
		2664	rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
		2665	gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
		2666	break;
2997	Serge	2667	}
		2668
		2669	/* Initialize HDP */
		2670	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		2671	WREG32((0x2c14 + j), 0x00000000);
		2672	WREG32((0x2c18 + j), 0x00000000);
		2673	WREG32((0x2c1c + j), 0x00000000);
		2674	WREG32((0x2c20 + j), 0x00000000);
		2675	WREG32((0x2c24 + j), 0x00000000);
		2676	}
		2677
		2678	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		2679
		2680	evergreen_fix_pci_max_read_req_size(rdev);
		2681
		2682	WREG32(BIF_FB_EN, FB_READ_EN \| FB_WRITE_EN);
		2683
		2684	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
		2685	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
		2686
		2687	rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
		2688	rdev->config.si.mem_max_burst_length_bytes = 256;
		2689	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
		2690	rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
		2691	if (rdev->config.si.mem_row_size_in_kb > 4)
		2692	rdev->config.si.mem_row_size_in_kb = 4;
		2693	/* XXX use MC settings? */
		2694	rdev->config.si.shader_engine_tile_size = 32;
		2695	rdev->config.si.num_gpus = 1;
		2696	rdev->config.si.multi_gpu_tile_size = 64;
		2697
		2698	/* fix up row size */
		2699	gb_addr_config &= ~ROW_SIZE_MASK;
		2700	switch (rdev->config.si.mem_row_size_in_kb) {
		2701	case 1:
		2702	default:
		2703	gb_addr_config \|= ROW_SIZE(0);
		2704	break;
		2705	case 2:
		2706	gb_addr_config \|= ROW_SIZE(1);
		2707	break;
		2708	case 4:
		2709	gb_addr_config \|= ROW_SIZE(2);
		2710	break;
		2711	}
		2712
		2713	/* setup tiling info dword. gb_addr_config is not adequate since it does
		2714	* not have bank info, so create a custom tiling dword.
		2715	* bits 3:0 num_pipes
		2716	* bits 7:4 num_banks
		2717	* bits 11:8 group_size
		2718	* bits 15:12 row_size
		2719	*/
		2720	rdev->config.si.tile_config = 0;
		2721	switch (rdev->config.si.num_tile_pipes) {
		2722	case 1:
		2723	rdev->config.si.tile_config \|= (0 << 0);
		2724	break;
		2725	case 2:
		2726	rdev->config.si.tile_config \|= (1 << 0);
		2727	break;
		2728	case 4:
		2729	rdev->config.si.tile_config \|= (2 << 0);
		2730	break;
		2731	case 8:
		2732	default:
		2733	/* XXX what about 12? */
		2734	rdev->config.si.tile_config \|= (3 << 0);
		2735	break;
		2736	}
		2737	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
		2738	case 0: /* four banks */
		2739	rdev->config.si.tile_config \|= 0 << 4;
		2740	break;
		2741	case 1: /* eight banks */
		2742	rdev->config.si.tile_config \|= 1 << 4;
		2743	break;
		2744	case 2: /* sixteen banks */
		2745	default:
		2746	rdev->config.si.tile_config \|= 2 << 4;
		2747	break;
		2748	}
		2749	rdev->config.si.tile_config \|=
		2750	((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
		2751	rdev->config.si.tile_config \|=
		2752	((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
		2753
		2754	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		2755	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3764	Serge	2756	WREG32(DMIF_ADDR_CALC, gb_addr_config);
2997	Serge	2757	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3192	Serge	2758	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
		2759	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3764	Serge	2760	if (rdev->has_uvd) {
		2761	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
		2762	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
		2763	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
		2764	}
2997	Serge	2765
		2766	si_tiling_mode_table_init(rdev);
		2767
		2768	si_setup_rb(rdev, rdev->config.si.max_shader_engines,
		2769	rdev->config.si.max_sh_per_se,
		2770	rdev->config.si.max_backends_per_se);
		2771
		2772	si_setup_spi(rdev, rdev->config.si.max_shader_engines,
		2773	rdev->config.si.max_sh_per_se,
		2774	rdev->config.si.max_cu_per_sh);
		2775
		2776
		2777	/* set HW defaults for 3D engine */
		2778	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \|
		2779	ROQ_IB2_START(0x2b)));
		2780	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) \| MEQ2_START(0x60));
		2781
		2782	sx_debug_1 = RREG32(SX_DEBUG_1);
		2783	WREG32(SX_DEBUG_1, sx_debug_1);
		2784
		2785	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
		2786
		2787	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) \|
		2788	SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) \|
		2789	SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) \|
		2790	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
		2791
		2792	WREG32(VGT_NUM_INSTANCES, 1);
		2793
		2794	WREG32(CP_PERFMON_CNTL, 0);
		2795
		2796	WREG32(SQ_CONFIG, 0);
		2797
		2798	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		2799	FORCE_EOV_MAX_REZ_CNT(255)));
		2800
		2801	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) \|
		2802	AUTO_INVLD_EN(ES_AND_GS_AUTO));
		2803
		2804	WREG32(VGT_GS_VERTEX_REUSE, 16);
		2805	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		2806
		2807	WREG32(CB_PERFCOUNTER0_SELECT0, 0);
		2808	WREG32(CB_PERFCOUNTER0_SELECT1, 0);
		2809	WREG32(CB_PERFCOUNTER1_SELECT0, 0);
		2810	WREG32(CB_PERFCOUNTER1_SELECT1, 0);
		2811	WREG32(CB_PERFCOUNTER2_SELECT0, 0);
		2812	WREG32(CB_PERFCOUNTER2_SELECT1, 0);
		2813	WREG32(CB_PERFCOUNTER3_SELECT0, 0);
		2814	WREG32(CB_PERFCOUNTER3_SELECT1, 0);
		2815
		2816	tmp = RREG32(HDP_MISC_CNTL);
		2817	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		2818	WREG32(HDP_MISC_CNTL, tmp);
		2819
		2820	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		2821	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		2822
		2823	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		2824
		2825	udelay(50);
		2826	}
		2827
		2828	/*
		2829	* GPU scratch registers helpers function.
		2830	*/
		2831	static void si_scratch_init(struct radeon_device *rdev)
		2832	{
		2833	int i;
		2834
		2835	rdev->scratch.num_reg = 7;
		2836	rdev->scratch.reg_base = SCRATCH_REG0;
		2837	for (i = 0; i < rdev->scratch.num_reg; i++) {
		2838	rdev->scratch.free[i] = true;
		2839	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
		2840	}
		2841	}
		2842
		2843	void si_fence_ring_emit(struct radeon_device *rdev,
		2844	struct radeon_fence *fence)
		2845	{
		2846	struct radeon_ring *ring = &rdev->ring[fence->ring];
		2847	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		2848
		2849	/* flush read cache over gart */
		2850	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2851	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
		2852	radeon_ring_write(ring, 0);
		2853	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		2854	radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA \|
		2855	PACKET3_TC_ACTION_ENA \|
		2856	PACKET3_SH_KCACHE_ACTION_ENA \|
		2857	PACKET3_SH_ICACHE_ACTION_ENA);
		2858	radeon_ring_write(ring, 0xFFFFFFFF);
		2859	radeon_ring_write(ring, 0);
		2860	radeon_ring_write(ring, 10); /* poll interval */
		2861	/* EVENT_WRITE_EOP - flush caches, send int */
		2862	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		2863	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) \| EVENT_INDEX(5));
		2864	radeon_ring_write(ring, addr & 0xffffffff);
		2865	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		2866	radeon_ring_write(ring, fence->seq);
		2867	radeon_ring_write(ring, 0);
		2868	}
		2869
		2870	/*
		2871	* IB stuff
		2872	*/
		2873	void si_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		2874	{
		2875	struct radeon_ring *ring = &rdev->ring[ib->ring];
		2876	u32 header;
		2877
		2878	if (ib->is_const_ib) {
		2879	/* set switch buffer packet before const IB */
		2880	radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
		2881	radeon_ring_write(ring, 0);
		2882
		2883	header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
		2884	} else {
		2885	u32 next_rptr;
		2886	if (ring->rptr_save_reg) {
		2887	next_rptr = ring->wptr + 3 + 4 + 8;
		2888	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2889	radeon_ring_write(ring, ((ring->rptr_save_reg -
		2890	PACKET3_SET_CONFIG_REG_START) >> 2));
		2891	radeon_ring_write(ring, next_rptr);
		2892	} else if (rdev->wb.enabled) {
		2893	next_rptr = ring->wptr + 5 + 4 + 8;
		2894	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
		2895	radeon_ring_write(ring, (1 << 8));
		2896	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		2897	radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
		2898	radeon_ring_write(ring, next_rptr);
		2899	}
		2900
		2901	header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
		2902	}
		2903
		2904	radeon_ring_write(ring, header);
		2905	radeon_ring_write(ring,
		2906	#ifdef __BIG_ENDIAN
		2907	(2 << 0) \|
		2908	#endif
		2909	(ib->gpu_addr & 0xFFFFFFFC));
		2910	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
		2911	radeon_ring_write(ring, ib->length_dw \|
		2912	(ib->vm ? (ib->vm->id << 24) : 0));
		2913
		2914	if (!ib->is_const_ib) {
		2915	/* flush read cache over gart for this vmid */
		2916	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2917	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
		2918	radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
		2919	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		2920	radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA \|
		2921	PACKET3_TC_ACTION_ENA \|
		2922	PACKET3_SH_KCACHE_ACTION_ENA \|
		2923	PACKET3_SH_ICACHE_ACTION_ENA);
		2924	radeon_ring_write(ring, 0xFFFFFFFF);
		2925	radeon_ring_write(ring, 0);
		2926	radeon_ring_write(ring, 10); /* poll interval */
		2927	}
		2928	}
		2929
		2930	/*
		2931	* CP.
		2932	*/
		2933	static void si_cp_enable(struct radeon_device *rdev, bool enable)
		2934	{
		2935	if (enable)
		2936	WREG32(CP_ME_CNTL, 0);
		2937	else {
3192	Serge	2938	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2997	Serge	2939	WREG32(CP_ME_CNTL, (CP_ME_HALT \| CP_PFP_HALT \| CP_CE_HALT));
		2940	WREG32(SCRATCH_UMSK, 0);
3192	Serge	2941	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
		2942	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		2943	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2997	Serge	2944	}
		2945	udelay(50);
		2946	}
		2947
		2948	static int si_cp_load_microcode(struct radeon_device *rdev)
		2949	{
		2950	const __be32 *fw_data;
		2951	int i;
		2952
		2953	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		2954	return -EINVAL;
		2955
		2956	si_cp_enable(rdev, false);
		2957
		2958	/* PFP */
		2959	fw_data = (const __be32 *)rdev->pfp_fw->data;
		2960	WREG32(CP_PFP_UCODE_ADDR, 0);
		2961	for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
		2962	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		2963	WREG32(CP_PFP_UCODE_ADDR, 0);
		2964
		2965	/* CE */
		2966	fw_data = (const __be32 *)rdev->ce_fw->data;
		2967	WREG32(CP_CE_UCODE_ADDR, 0);
		2968	for (i = 0; i < SI_CE_UCODE_SIZE; i++)
		2969	WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
		2970	WREG32(CP_CE_UCODE_ADDR, 0);
		2971
		2972	/* ME */
		2973	fw_data = (const __be32 *)rdev->me_fw->data;
		2974	WREG32(CP_ME_RAM_WADDR, 0);
		2975	for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
		2976	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		2977	WREG32(CP_ME_RAM_WADDR, 0);
		2978
		2979	WREG32(CP_PFP_UCODE_ADDR, 0);
		2980	WREG32(CP_CE_UCODE_ADDR, 0);
		2981	WREG32(CP_ME_RAM_WADDR, 0);
		2982	WREG32(CP_ME_RAM_RADDR, 0);
		2983	return 0;
		2984	}
		2985
		2986	static int si_cp_start(struct radeon_device *rdev)
		2987	{
		2988	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		2989	int r, i;
		2990
		2991	r = radeon_ring_lock(rdev, ring, 7 + 4);
		2992	if (r) {
		2993	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		2994	return r;
		2995	}
		2996	/* init the CP */
		2997	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		2998	radeon_ring_write(ring, 0x1);
		2999	radeon_ring_write(ring, 0x0);
		3000	radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
		3001	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		3002	radeon_ring_write(ring, 0);
		3003	radeon_ring_write(ring, 0);
		3004
		3005	/* init the CE partitions */
		3006	radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
		3007	radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
		3008	radeon_ring_write(ring, 0xc000);
		3009	radeon_ring_write(ring, 0xe000);
		3010	radeon_ring_unlock_commit(rdev, ring);
		3011
		3012	si_cp_enable(rdev, true);
		3013
		3014	r = radeon_ring_lock(rdev, ring, si_default_size + 10);
		3015	if (r) {
		3016	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		3017	return r;
		3018	}
		3019
		3020	/* setup clear context state */
		3021	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		3022	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
		3023
		3024	for (i = 0; i < si_default_size; i++)
		3025	radeon_ring_write(ring, si_default_state[i]);
		3026
		3027	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		3028	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
		3029
		3030	/* set clear context state */
		3031	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
		3032	radeon_ring_write(ring, 0);
		3033
		3034	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
		3035	radeon_ring_write(ring, 0x00000316);
		3036	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		3037	radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
		3038
		3039	radeon_ring_unlock_commit(rdev, ring);
		3040
		3041	for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
		3042	ring = &rdev->ring[i];
		3043	r = radeon_ring_lock(rdev, ring, 2);
		3044
		3045	/* clear the compute context state */
		3046	radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
		3047	radeon_ring_write(ring, 0);
		3048
		3049	radeon_ring_unlock_commit(rdev, ring);
		3050	}
		3051
		3052	return 0;
		3053	}
		3054
		3055	static void si_cp_fini(struct radeon_device *rdev)
		3056	{
		3057	struct radeon_ring *ring;
		3058	si_cp_enable(rdev, false);
		3059
		3060	// ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		3061	// radeon_ring_fini(rdev, ring);
		3062	// radeon_scratch_free(rdev, ring->rptr_save_reg);
		3063
		3064	// ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
		3065	// radeon_ring_fini(rdev, ring);
		3066	// radeon_scratch_free(rdev, ring->rptr_save_reg);
		3067
		3068	// ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
		3069	// radeon_ring_fini(rdev, ring);
		3070	// radeon_scratch_free(rdev, ring->rptr_save_reg);
		3071	}
		3072
		3073	static int si_cp_resume(struct radeon_device *rdev)
		3074	{
		3075	struct radeon_ring *ring;
		3076	u32 tmp;
		3077	u32 rb_bufsz;
		3078	int r;
		3079
		3080	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
		3081	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP \|
		3082	SOFT_RESET_PA \|
		3083	SOFT_RESET_VGT \|
		3084	SOFT_RESET_SPI \|
		3085	SOFT_RESET_SX));
		3086	RREG32(GRBM_SOFT_RESET);
		3087	mdelay(15);
		3088	WREG32(GRBM_SOFT_RESET, 0);
		3089	RREG32(GRBM_SOFT_RESET);
		3090
		3091	WREG32(CP_SEM_WAIT_TIMER, 0x0);
		3092	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
		3093
		3094	/* Set the write pointer delay */
		3095	WREG32(CP_RB_WPTR_DELAY, 0);
		3096
		3097	WREG32(CP_DEBUG, 0);
		3098	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		3099
		3100	/* ring 0 - compute and gfx */
		3101	/* Set ring buffer size */
		3102	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		3103	rb_bufsz = drm_order(ring->ring_size / 8);
		3104	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
		3105	#ifdef __BIG_ENDIAN
		3106	tmp \|= BUF_SWAP_32BIT;
		3107	#endif
		3108	WREG32(CP_RB0_CNTL, tmp);
		3109
		3110	/* Initialize the ring buffer's read and write pointers */
		3111	WREG32(CP_RB0_CNTL, tmp \| RB_RPTR_WR_ENA);
		3112	ring->wptr = 0;
		3113	WREG32(CP_RB0_WPTR, ring->wptr);
		3114
3120	serge	3115	/* set the wb address whether it's enabled or not */
2997	Serge	3116	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
		3117	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		3118
		3119	if (rdev->wb.enabled)
		3120	WREG32(SCRATCH_UMSK, 0xff);
		3121	else {
		3122	tmp \|= RB_NO_UPDATE;
		3123	WREG32(SCRATCH_UMSK, 0);
		3124	}
		3125
		3126	mdelay(1);
		3127	WREG32(CP_RB0_CNTL, tmp);
		3128
		3129	WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
		3130
		3131	ring->rptr = RREG32(CP_RB0_RPTR);
		3132
		3133	/* ring1 - compute only */
		3134	/* Set ring buffer size */
		3135	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
		3136	rb_bufsz = drm_order(ring->ring_size / 8);
		3137	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
		3138	#ifdef __BIG_ENDIAN
		3139	tmp \|= BUF_SWAP_32BIT;
		3140	#endif
		3141	WREG32(CP_RB1_CNTL, tmp);
		3142
		3143	/* Initialize the ring buffer's read and write pointers */
		3144	WREG32(CP_RB1_CNTL, tmp \| RB_RPTR_WR_ENA);
		3145	ring->wptr = 0;
		3146	WREG32(CP_RB1_WPTR, ring->wptr);
		3147
3120	serge	3148	/* set the wb address whether it's enabled or not */
2997	Serge	3149	WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
		3150	WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
		3151
		3152	mdelay(1);
		3153	WREG32(CP_RB1_CNTL, tmp);
		3154
		3155	WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
		3156
		3157	ring->rptr = RREG32(CP_RB1_RPTR);
		3158
		3159	/* ring2 - compute only */
		3160	/* Set ring buffer size */
		3161	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
		3162	rb_bufsz = drm_order(ring->ring_size / 8);
		3163	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
		3164	#ifdef __BIG_ENDIAN
		3165	tmp \|= BUF_SWAP_32BIT;
		3166	#endif
		3167	WREG32(CP_RB2_CNTL, tmp);
		3168
		3169	/* Initialize the ring buffer's read and write pointers */
		3170	WREG32(CP_RB2_CNTL, tmp \| RB_RPTR_WR_ENA);
		3171	ring->wptr = 0;
		3172	WREG32(CP_RB2_WPTR, ring->wptr);
		3173
3120	serge	3174	/* set the wb address whether it's enabled or not */
2997	Serge	3175	WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
		3176	WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
		3177
		3178	mdelay(1);
		3179	WREG32(CP_RB2_CNTL, tmp);
		3180
		3181	WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
		3182
		3183	ring->rptr = RREG32(CP_RB2_RPTR);
		3184
		3185	/* start the rings */
		3186	si_cp_start(rdev);
		3187	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
		3188	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
		3189	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
		3190	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
		3191	if (r) {
		3192	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
		3193	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		3194	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
		3195	return r;
		3196	}
		3197	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
		3198	if (r) {
		3199	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		3200	}
		3201	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
		3202	if (r) {
		3203	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
		3204	}
		3205
		3206	return 0;
		3207	}
		3208
3764	Serge	3209	static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
2997	Serge	3210	{
3764	Serge	3211	u32 reset_mask = 0;
		3212	u32 tmp;
2997	Serge	3213
3764	Serge	3214	/* GRBM_STATUS */
		3215	tmp = RREG32(GRBM_STATUS);
		3216	if (tmp & (PA_BUSY \| SC_BUSY \|
		3217	BCI_BUSY \| SX_BUSY \|
		3218	TA_BUSY \| VGT_BUSY \|
		3219	DB_BUSY \| CB_BUSY \|
		3220	GDS_BUSY \| SPI_BUSY \|
		3221	IA_BUSY \| IA_BUSY_NO_DMA))
		3222	reset_mask \|= RADEON_RESET_GFX;
		3223
		3224	if (tmp & (CF_RQ_PENDING \| PF_RQ_PENDING \|
		3225	CP_BUSY \| CP_COHERENCY_BUSY))
		3226	reset_mask \|= RADEON_RESET_CP;
		3227
		3228	if (tmp & GRBM_EE_BUSY)
		3229	reset_mask \|= RADEON_RESET_GRBM \| RADEON_RESET_GFX \| RADEON_RESET_CP;
		3230
		3231	/* GRBM_STATUS2 */
		3232	tmp = RREG32(GRBM_STATUS2);
		3233	if (tmp & (RLC_RQ_PENDING \| RLC_BUSY))
		3234	reset_mask \|= RADEON_RESET_RLC;
		3235
		3236	/* DMA_STATUS_REG 0 */
		3237	tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
		3238	if (!(tmp & DMA_IDLE))
		3239	reset_mask \|= RADEON_RESET_DMA;
		3240
		3241	/* DMA_STATUS_REG 1 */
		3242	tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
		3243	if (!(tmp & DMA_IDLE))
		3244	reset_mask \|= RADEON_RESET_DMA1;
		3245
		3246	/* SRBM_STATUS2 */
		3247	tmp = RREG32(SRBM_STATUS2);
		3248	if (tmp & DMA_BUSY)
		3249	reset_mask \|= RADEON_RESET_DMA;
		3250
		3251	if (tmp & DMA1_BUSY)
		3252	reset_mask \|= RADEON_RESET_DMA1;
		3253
		3254	/* SRBM_STATUS */
		3255	tmp = RREG32(SRBM_STATUS);
		3256
		3257	if (tmp & IH_BUSY)
		3258	reset_mask \|= RADEON_RESET_IH;
		3259
		3260	if (tmp & SEM_BUSY)
		3261	reset_mask \|= RADEON_RESET_SEM;
		3262
		3263	if (tmp & GRBM_RQ_PENDING)
		3264	reset_mask \|= RADEON_RESET_GRBM;
		3265
		3266	if (tmp & VMC_BUSY)
		3267	reset_mask \|= RADEON_RESET_VMC;
		3268
		3269	if (tmp & (MCB_BUSY \| MCB_NON_DISPLAY_BUSY \|
		3270	MCC_BUSY \| MCD_BUSY))
		3271	reset_mask \|= RADEON_RESET_MC;
		3272
		3273	if (evergreen_is_display_hung(rdev))
		3274	reset_mask \|= RADEON_RESET_DISPLAY;
		3275
		3276	/* VM_L2_STATUS */
		3277	tmp = RREG32(VM_L2_STATUS);
		3278	if (tmp & L2_BUSY)
		3279	reset_mask \|= RADEON_RESET_VMC;
		3280
		3281	/* Skip MC reset as it's mostly likely not hung, just busy */
		3282	if (reset_mask & RADEON_RESET_MC) {
		3283	DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
		3284	reset_mask &= ~RADEON_RESET_MC;
2997	Serge	3285	}
3764	Serge	3286
		3287	return reset_mask;
2997	Serge	3288	}
		3289
3764	Serge	3290	static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
2997	Serge	3291	{
3764	Serge	3292	struct evergreen_mc_save save;
		3293	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
		3294	u32 tmp;
2997	Serge	3295
3764	Serge	3296	if (reset_mask == 0)
3192	Serge	3297	return;
2997	Serge	3298
3764	Serge	3299	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3192	Serge	3300
3764	Serge	3301	evergreen_print_gpu_status_regs(rdev);
		3302	dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
		3303	RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
		3304	dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
		3305	RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
		3306
2997	Serge	3307	/* Disable CP parsing/prefetching */
		3308	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT \| CP_CE_HALT);
		3309
3764	Serge	3310	if (reset_mask & RADEON_RESET_DMA) {
		3311	/* dma0 */
		3312	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
		3313	tmp &= ~DMA_RB_ENABLE;
		3314	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
		3315	}
		3316	if (reset_mask & RADEON_RESET_DMA1) {
		3317	/* dma1 */
		3318	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
		3319	tmp &= ~DMA_RB_ENABLE;
		3320	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
		3321	}
		3322
		3323	udelay(50);
		3324
		3325	evergreen_mc_stop(rdev, &save);
		3326	if (evergreen_mc_wait_for_idle(rdev)) {
		3327	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		3328	}
		3329
		3330	if (reset_mask & (RADEON_RESET_GFX \| RADEON_RESET_COMPUTE \| RADEON_RESET_CP)) {
		3331	grbm_soft_reset = SOFT_RESET_CB \|
2997	Serge	3332	SOFT_RESET_DB \|
		3333	SOFT_RESET_GDS \|
		3334	SOFT_RESET_PA \|
		3335	SOFT_RESET_SC \|
		3336	SOFT_RESET_BCI \|
		3337	SOFT_RESET_SPI \|
		3338	SOFT_RESET_SX \|
		3339	SOFT_RESET_TC \|
		3340	SOFT_RESET_TA \|
		3341	SOFT_RESET_VGT \|
3764	Serge	3342	SOFT_RESET_IA;
		3343	}
2997	Serge	3344
3764	Serge	3345	if (reset_mask & RADEON_RESET_CP) {
		3346	grbm_soft_reset \|= SOFT_RESET_CP \| SOFT_RESET_VGT;
3192	Serge	3347
3764	Serge	3348	srbm_soft_reset \|= SOFT_RESET_GRBM;
		3349	}
3192	Serge	3350
3764	Serge	3351	if (reset_mask & RADEON_RESET_DMA)
		3352	srbm_soft_reset \|= SOFT_RESET_DMA;
3192	Serge	3353
3764	Serge	3354	if (reset_mask & RADEON_RESET_DMA1)
		3355	srbm_soft_reset \|= SOFT_RESET_DMA1;
3192	Serge	3356
3764	Serge	3357	if (reset_mask & RADEON_RESET_DISPLAY)
		3358	srbm_soft_reset \|= SOFT_RESET_DC;
3192	Serge	3359
3764	Serge	3360	if (reset_mask & RADEON_RESET_RLC)
		3361	grbm_soft_reset \|= SOFT_RESET_RLC;
3192	Serge	3362
3764	Serge	3363	if (reset_mask & RADEON_RESET_SEM)
		3364	srbm_soft_reset \|= SOFT_RESET_SEM;
3192	Serge	3365
3764	Serge	3366	if (reset_mask & RADEON_RESET_IH)
		3367	srbm_soft_reset \|= SOFT_RESET_IH;
3192	Serge	3368
3764	Serge	3369	if (reset_mask & RADEON_RESET_GRBM)
		3370	srbm_soft_reset \|= SOFT_RESET_GRBM;
3192	Serge	3371
3764	Serge	3372	if (reset_mask & RADEON_RESET_VMC)
		3373	srbm_soft_reset \|= SOFT_RESET_VMC;
3192	Serge	3374
3764	Serge	3375	if (reset_mask & RADEON_RESET_MC)
		3376	srbm_soft_reset \|= SOFT_RESET_MC;
3192	Serge	3377
3764	Serge	3378	if (grbm_soft_reset) {
		3379	tmp = RREG32(GRBM_SOFT_RESET);
		3380	tmp \|= grbm_soft_reset;
		3381	dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
		3382	WREG32(GRBM_SOFT_RESET, tmp);
		3383	tmp = RREG32(GRBM_SOFT_RESET);
3192	Serge	3384
3764	Serge	3385	udelay(50);
3192	Serge	3386
3764	Serge	3387	tmp &= ~grbm_soft_reset;
		3388	WREG32(GRBM_SOFT_RESET, tmp);
		3389	tmp = RREG32(GRBM_SOFT_RESET);
3192	Serge	3390	}
		3391
3764	Serge	3392	if (srbm_soft_reset) {
		3393	tmp = RREG32(SRBM_SOFT_RESET);
		3394	tmp \|= srbm_soft_reset;
		3395	dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
		3396	WREG32(SRBM_SOFT_RESET, tmp);
		3397	tmp = RREG32(SRBM_SOFT_RESET);
3192	Serge	3398
3764	Serge	3399	udelay(50);
3192	Serge	3400
3764	Serge	3401	tmp &= ~srbm_soft_reset;
		3402	WREG32(SRBM_SOFT_RESET, tmp);
		3403	tmp = RREG32(SRBM_SOFT_RESET);
		3404	}
		3405
3192	Serge	3406	/* Wait a little for things to settle down */
		3407	udelay(50);
		3408
2997	Serge	3409	evergreen_mc_resume(rdev, &save);
3764	Serge	3410	udelay(50);
		3411
		3412	evergreen_print_gpu_status_regs(rdev);
2997	Serge	3413	}
		3414
		3415	int si_asic_reset(struct radeon_device *rdev)
		3416	{
3764	Serge	3417	u32 reset_mask;
		3418
		3419	reset_mask = si_gpu_check_soft_reset(rdev);
		3420
		3421	if (reset_mask)
		3422	r600_set_bios_scratch_engine_hung(rdev, true);
		3423
		3424	si_gpu_soft_reset(rdev, reset_mask);
		3425
		3426	reset_mask = si_gpu_check_soft_reset(rdev);
		3427
		3428	if (!reset_mask)
		3429	r600_set_bios_scratch_engine_hung(rdev, false);
		3430
		3431	return 0;
		3432	}
		3433
		3434	/**
		3435	* si_gfx_is_lockup - Check if the GFX engine is locked up
		3436	*
		3437	* @rdev: radeon_device pointer
		3438	* @ring: radeon_ring structure holding ring information
		3439	*
		3440	* Check if the GFX engine is locked up.
		3441	* Returns true if the engine appears to be locked up, false if not.
		3442	*/
		3443	bool si_gfx_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
		3444	{
		3445	u32 reset_mask = si_gpu_check_soft_reset(rdev);
		3446
		3447	if (!(reset_mask & (RADEON_RESET_GFX \|
3192	Serge	3448	RADEON_RESET_COMPUTE \|
3764	Serge	3449	RADEON_RESET_CP))) {
		3450	radeon_ring_lockup_update(ring);
		3451	return false;
		3452	}
		3453	/* force CP activities */
		3454	radeon_ring_force_activity(rdev, ring);
		3455	return radeon_ring_test_lockup(rdev, ring);
2997	Serge	3456	}
		3457
3764	Serge	3458	/**
		3459	* si_dma_is_lockup - Check if the DMA engine is locked up
		3460	*
		3461	* @rdev: radeon_device pointer
		3462	* @ring: radeon_ring structure holding ring information
		3463	*
		3464	* Check if the async DMA engine is locked up.
		3465	* Returns true if the engine appears to be locked up, false if not.
		3466	*/
		3467	bool si_dma_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
		3468	{
		3469	u32 reset_mask = si_gpu_check_soft_reset(rdev);
		3470	u32 mask;
		3471
		3472	if (ring->idx == R600_RING_TYPE_DMA_INDEX)
		3473	mask = RADEON_RESET_DMA;
		3474	else
		3475	mask = RADEON_RESET_DMA1;
		3476
		3477	if (!(reset_mask & mask)) {
		3478	radeon_ring_lockup_update(ring);
		3479	return false;
		3480	}
		3481	/* force ring activities */
		3482	radeon_ring_force_activity(rdev, ring);
		3483	return radeon_ring_test_lockup(rdev, ring);
		3484	}
		3485
2997	Serge	3486	/* MC */
		3487	static void si_mc_program(struct radeon_device *rdev)
		3488	{
		3489	struct evergreen_mc_save save;
		3490	u32 tmp;
		3491	int i, j;
		3492
		3493	/* Initialize HDP */
		3494	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		3495	WREG32((0x2c14 + j), 0x00000000);
		3496	WREG32((0x2c18 + j), 0x00000000);
		3497	WREG32((0x2c1c + j), 0x00000000);
		3498	WREG32((0x2c20 + j), 0x00000000);
		3499	WREG32((0x2c24 + j), 0x00000000);
		3500	}
		3501	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		3502
		3503	evergreen_mc_stop(rdev, &save);
		3504	if (radeon_mc_wait_for_idle(rdev)) {
		3505	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		3506	}
3764	Serge	3507	if (!ASIC_IS_NODCE(rdev))
2997	Serge	3508	/* Lockout access through VGA aperture*/
		3509	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		3510	/* Update configuration */
		3511	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		3512	rdev->mc.vram_start >> 12);
		3513	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		3514	rdev->mc.vram_end >> 12);
		3515	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
		3516	rdev->vram_scratch.gpu_addr >> 12);
		3517	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		3518	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		3519	WREG32(MC_VM_FB_LOCATION, tmp);
		3520	/* XXX double check these! */
		3521	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
		3522	WREG32(HDP_NONSURFACE_INFO, (2 << 7) \| (1 << 30));
		3523	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
		3524	WREG32(MC_VM_AGP_BASE, 0);
		3525	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		3526	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		3527	if (radeon_mc_wait_for_idle(rdev)) {
		3528	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		3529	}
		3530	evergreen_mc_resume(rdev, &save);
3764	Serge	3531	if (!ASIC_IS_NODCE(rdev)) {
2997	Serge	3532	/* we need to own VRAM, so turn off the VGA renderer here
		3533	* to stop it overwriting our objects */
		3534	rv515_vga_render_disable(rdev);
		3535	}
		3536	}
		3537
		3538	static void si_vram_gtt_location(struct radeon_device *rdev,
		3539	struct radeon_mc *mc)
		3540	{
		3541	if (mc->mc_vram_size > 0xFFC0000000ULL) {
		3542	/* leave room for at least 1024M GTT */
		3543	dev_warn(rdev->dev, "limiting VRAM\n");
		3544	mc->real_vram_size = 0xFFC0000000ULL;
		3545	mc->mc_vram_size = 0xFFC0000000ULL;
		3546	}
3764	Serge	3547	radeon_vram_location(rdev, &rdev->mc, 0);
2997	Serge	3548	rdev->mc.gtt_base_align = 0;
3764	Serge	3549	radeon_gtt_location(rdev, mc);
2997	Serge	3550	}
		3551
		3552	static int si_mc_init(struct radeon_device *rdev)
		3553	{
		3554	u32 tmp;
		3555	int chansize, numchan;
		3556
		3557	/* Get VRAM informations */
		3558	rdev->mc.vram_is_ddr = true;
		3559	tmp = RREG32(MC_ARB_RAMCFG);
		3560	if (tmp & CHANSIZE_OVERRIDE) {
		3561	chansize = 16;
		3562	} else if (tmp & CHANSIZE_MASK) {
		3563	chansize = 64;
		3564	} else {
		3565	chansize = 32;
		3566	}
		3567	tmp = RREG32(MC_SHARED_CHMAP);
		3568	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		3569	case 0:
		3570	default:
		3571	numchan = 1;
		3572	break;
		3573	case 1:
		3574	numchan = 2;
		3575	break;
		3576	case 2:
		3577	numchan = 4;
		3578	break;
		3579	case 3:
		3580	numchan = 8;
		3581	break;
		3582	case 4:
		3583	numchan = 3;
		3584	break;
		3585	case 5:
		3586	numchan = 6;
		3587	break;
		3588	case 6:
		3589	numchan = 10;
		3590	break;
		3591	case 7:
		3592	numchan = 12;
		3593	break;
		3594	case 8:
		3595	numchan = 16;
		3596	break;
		3597	}
		3598	rdev->mc.vram_width = numchan * chansize;
		3599	/* Could aper size report 0 ? */
		3600	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		3601	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
		3602	/* size in MB on si */
3764	Serge	3603	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
		3604	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
2997	Serge	3605	rdev->mc.visible_vram_size = rdev->mc.aper_size;
		3606	si_vram_gtt_location(rdev, &rdev->mc);
		3607	radeon_update_bandwidth_info(rdev);
		3608
		3609	return 0;
		3610	}
		3611
		3612	/*
		3613	* GART
		3614	*/
		3615	void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
		3616	{
		3617	/* flush hdp cache */
		3618	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		3619
		3620	/* bits 0-15 are the VM contexts0-15 */
		3621	WREG32(VM_INVALIDATE_REQUEST, 1);
		3622	}
		3623
		3624	static int si_pcie_gart_enable(struct radeon_device *rdev)
		3625	{
		3626	int r, i;
		3627
		3628	if (rdev->gart.robj == NULL) {
		3629	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		3630	return -EINVAL;
		3631	}
		3632	r = radeon_gart_table_vram_pin(rdev);
		3633	if (r)
		3634	return r;
		3635	radeon_gart_restore(rdev);
		3636	/* Setup TLB control */
		3637	WREG32(MC_VM_MX_L1_TLB_CNTL,
		3638	(0xA << 7) \|
		3639	ENABLE_L1_TLB \|
		3640	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		3641	ENABLE_ADVANCED_DRIVER_MODEL \|
		3642	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
		3643	/* Setup L2 cache */
		3644	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \|
		3645	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		3646	ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE \|
		3647	EFFECTIVE_L2_QUEUE_SIZE(7) \|
		3648	CONTEXT1_IDENTITY_ACCESS_MODE(1));
		3649	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS \| INVALIDATE_L2_CACHE);
		3650	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY \|
		3651	L2_CACHE_BIGK_FRAGMENT_SIZE(0));
		3652	/* setup context0 */
		3653	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		3654	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		3655	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		3656	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		3657	(u32)(rdev->dummy_page.addr >> 12));
		3658	WREG32(VM_CONTEXT0_CNTL2, 0);
		3659	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		3660	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
		3661
		3662	WREG32(0x15D4, 0);
		3663	WREG32(0x15D8, 0);
		3664	WREG32(0x15DC, 0);
		3665
		3666	/* empty context1-15 */
		3667	/* set vm size, must be a multiple of 4 */
		3668	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
		3669	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
		3670	/* Assign the pt base to something valid for now; the pts used for
		3671	* the VMs are determined by the application and setup and assigned
		3672	* on the fly in the vm part of radeon_gart.c
		3673	*/
		3674	for (i = 1; i < 16; i++) {
		3675	if (i < 8)
		3676	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
		3677	rdev->gart.table_addr >> 12);
		3678	else
		3679	WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
		3680	rdev->gart.table_addr >> 12);
		3681	}
		3682
		3683	/* enable context1-15 */
		3684	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
		3685	(u32)(rdev->dummy_page.addr >> 12));
3192	Serge	3686	WREG32(VM_CONTEXT1_CNTL2, 4);
2997	Serge	3687	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(1) \|
3192	Serge	3688	RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		3689	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT \|
		3690	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		3691	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT \|
		3692	PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		3693	PDE0_PROTECTION_FAULT_ENABLE_DEFAULT \|
		3694	VALID_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		3695	VALID_PROTECTION_FAULT_ENABLE_DEFAULT \|
		3696	READ_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		3697	READ_PROTECTION_FAULT_ENABLE_DEFAULT \|
		3698	WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		3699	WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
2997	Serge	3700
		3701	si_pcie_gart_tlb_flush(rdev);
		3702	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		3703	(unsigned)(rdev->mc.gtt_size >> 20),
		3704	(unsigned long long)rdev->gart.table_addr);
		3705	rdev->gart.ready = true;
		3706	return 0;
		3707	}
		3708
		3709	static void si_pcie_gart_disable(struct radeon_device *rdev)
		3710	{
		3711	/* Disable all tables */
		3712	WREG32(VM_CONTEXT0_CNTL, 0);
		3713	WREG32(VM_CONTEXT1_CNTL, 0);
		3714	/* Setup TLB control */
		3715	WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		3716	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
		3717	/* Setup L2 cache */
		3718	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		3719	ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE \|
		3720	EFFECTIVE_L2_QUEUE_SIZE(7) \|
		3721	CONTEXT1_IDENTITY_ACCESS_MODE(1));
		3722	WREG32(VM_L2_CNTL2, 0);
		3723	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY \|
		3724	L2_CACHE_BIGK_FRAGMENT_SIZE(0));
		3725	radeon_gart_table_vram_unpin(rdev);
		3726	}
		3727
		3728	static void si_pcie_gart_fini(struct radeon_device *rdev)
		3729	{
		3730	si_pcie_gart_disable(rdev);
		3731	radeon_gart_table_vram_free(rdev);
		3732	// radeon_gart_fini(rdev);
		3733	}
		3734
		3735	/* vm parser */
		3736	static bool si_vm_reg_valid(u32 reg)
		3737	{
		3738	/* context regs are fine */
		3739	if (reg >= 0x28000)
		3740	return true;
		3741
		3742	/* check config regs */
		3743	switch (reg) {
		3744	case GRBM_GFX_INDEX:
3031	serge	3745	case CP_STRMOUT_CNTL:
2997	Serge	3746	case VGT_VTX_VECT_EJECT_REG:
		3747	case VGT_CACHE_INVALIDATION:
		3748	case VGT_ESGS_RING_SIZE:
		3749	case VGT_GSVS_RING_SIZE:
		3750	case VGT_GS_VERTEX_REUSE:
		3751	case VGT_PRIMITIVE_TYPE:
		3752	case VGT_INDEX_TYPE:
		3753	case VGT_NUM_INDICES:
		3754	case VGT_NUM_INSTANCES:
		3755	case VGT_TF_RING_SIZE:
		3756	case VGT_HS_OFFCHIP_PARAM:
		3757	case VGT_TF_MEMORY_BASE:
		3758	case PA_CL_ENHANCE:
		3759	case PA_SU_LINE_STIPPLE_VALUE:
		3760	case PA_SC_LINE_STIPPLE_STATE:
		3761	case PA_SC_ENHANCE:
		3762	case SQC_CACHES:
		3763	case SPI_STATIC_THREAD_MGMT_1:
		3764	case SPI_STATIC_THREAD_MGMT_2:
		3765	case SPI_STATIC_THREAD_MGMT_3:
		3766	case SPI_PS_MAX_WAVE_ID:
		3767	case SPI_CONFIG_CNTL:
		3768	case SPI_CONFIG_CNTL_1:
		3769	case TA_CNTL_AUX:
		3770	return true;
		3771	default:
		3772	DRM_ERROR("Invalid register 0x%x in CS\n", reg);
		3773	return false;
		3774	}
		3775	}
		3776
		3777	static int si_vm_packet3_ce_check(struct radeon_device *rdev,
		3778	u32 ib, struct radeon_cs_packet pkt)
		3779	{
		3780	switch (pkt->opcode) {
		3781	case PACKET3_NOP:
		3782	case PACKET3_SET_BASE:
		3783	case PACKET3_SET_CE_DE_COUNTERS:
		3784	case PACKET3_LOAD_CONST_RAM:
		3785	case PACKET3_WRITE_CONST_RAM:
		3786	case PACKET3_WRITE_CONST_RAM_OFFSET:
		3787	case PACKET3_DUMP_CONST_RAM:
		3788	case PACKET3_INCREMENT_CE_COUNTER:
		3789	case PACKET3_WAIT_ON_DE_COUNTER:
		3790	case PACKET3_CE_WRITE:
		3791	break;
		3792	default:
		3793	DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
		3794	return -EINVAL;
		3795	}
		3796	return 0;
		3797	}
		3798
		3799	static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
		3800	u32 ib, struct radeon_cs_packet pkt)
		3801	{
		3802	u32 idx = pkt->idx + 1;
		3803	u32 idx_value = ib[idx];
		3804	u32 start_reg, end_reg, reg, i;
3192	Serge	3805	u32 command, info;
2997	Serge	3806
		3807	switch (pkt->opcode) {
		3808	case PACKET3_NOP:
		3809	case PACKET3_SET_BASE:
		3810	case PACKET3_CLEAR_STATE:
		3811	case PACKET3_INDEX_BUFFER_SIZE:
		3812	case PACKET3_DISPATCH_DIRECT:
		3813	case PACKET3_DISPATCH_INDIRECT:
		3814	case PACKET3_ALLOC_GDS:
		3815	case PACKET3_WRITE_GDS_RAM:
		3816	case PACKET3_ATOMIC_GDS:
		3817	case PACKET3_ATOMIC:
		3818	case PACKET3_OCCLUSION_QUERY:
		3819	case PACKET3_SET_PREDICATION:
		3820	case PACKET3_COND_EXEC:
		3821	case PACKET3_PRED_EXEC:
		3822	case PACKET3_DRAW_INDIRECT:
		3823	case PACKET3_DRAW_INDEX_INDIRECT:
		3824	case PACKET3_INDEX_BASE:
		3825	case PACKET3_DRAW_INDEX_2:
		3826	case PACKET3_CONTEXT_CONTROL:
		3827	case PACKET3_INDEX_TYPE:
		3828	case PACKET3_DRAW_INDIRECT_MULTI:
		3829	case PACKET3_DRAW_INDEX_AUTO:
		3830	case PACKET3_DRAW_INDEX_IMMD:
		3831	case PACKET3_NUM_INSTANCES:
		3832	case PACKET3_DRAW_INDEX_MULTI_AUTO:
		3833	case PACKET3_STRMOUT_BUFFER_UPDATE:
		3834	case PACKET3_DRAW_INDEX_OFFSET_2:
		3835	case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
		3836	case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
		3837	case PACKET3_MPEG_INDEX:
		3838	case PACKET3_WAIT_REG_MEM:
		3839	case PACKET3_MEM_WRITE:
		3840	case PACKET3_PFP_SYNC_ME:
		3841	case PACKET3_SURFACE_SYNC:
		3842	case PACKET3_EVENT_WRITE:
		3843	case PACKET3_EVENT_WRITE_EOP:
		3844	case PACKET3_EVENT_WRITE_EOS:
		3845	case PACKET3_SET_CONTEXT_REG:
		3846	case PACKET3_SET_CONTEXT_REG_INDIRECT:
		3847	case PACKET3_SET_SH_REG:
		3848	case PACKET3_SET_SH_REG_OFFSET:
		3849	case PACKET3_INCREMENT_DE_COUNTER:
		3850	case PACKET3_WAIT_ON_CE_COUNTER:
		3851	case PACKET3_WAIT_ON_AVAIL_BUFFER:
		3852	case PACKET3_ME_WRITE:
		3853	break;
		3854	case PACKET3_COPY_DATA:
		3855	if ((idx_value & 0xf00) == 0) {
		3856	reg = ib[idx + 3] * 4;
		3857	if (!si_vm_reg_valid(reg))
		3858	return -EINVAL;
		3859	}
		3860	break;
		3861	case PACKET3_WRITE_DATA:
		3862	if ((idx_value & 0xf00) == 0) {
		3863	start_reg = ib[idx + 1] * 4;
		3864	if (idx_value & 0x10000) {
		3865	if (!si_vm_reg_valid(start_reg))
		3866	return -EINVAL;
		3867	} else {
		3868	for (i = 0; i < (pkt->count - 2); i++) {
		3869	reg = start_reg + (4 * i);
		3870	if (!si_vm_reg_valid(reg))
		3871	return -EINVAL;
		3872	}
		3873	}
		3874	}
		3875	break;
		3876	case PACKET3_COND_WRITE:
		3877	if (idx_value & 0x100) {
		3878	reg = ib[idx + 5] * 4;
		3879	if (!si_vm_reg_valid(reg))
		3880	return -EINVAL;
		3881	}
		3882	break;
		3883	case PACKET3_COPY_DW:
		3884	if (idx_value & 0x2) {
		3885	reg = ib[idx + 3] * 4;
		3886	if (!si_vm_reg_valid(reg))
		3887	return -EINVAL;
		3888	}
		3889	break;
		3890	case PACKET3_SET_CONFIG_REG:
		3891	start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
		3892	end_reg = 4 * pkt->count + start_reg - 4;
		3893	if ((start_reg < PACKET3_SET_CONFIG_REG_START) \|\|
		3894	(start_reg >= PACKET3_SET_CONFIG_REG_END) \|\|
		3895	(end_reg >= PACKET3_SET_CONFIG_REG_END)) {
		3896	DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
		3897	return -EINVAL;
		3898	}
		3899	for (i = 0; i < pkt->count; i++) {
		3900	reg = start_reg + (4 * i);
		3901	if (!si_vm_reg_valid(reg))
		3902	return -EINVAL;
		3903	}
		3904	break;
3192	Serge	3905	case PACKET3_CP_DMA:
		3906	command = ib[idx + 4];
		3907	info = ib[idx + 1];
		3908	if (command & PACKET3_CP_DMA_CMD_SAS) {
		3909	/* src address space is register */
		3910	if (((info & 0x60000000) >> 29) == 0) {
		3911	start_reg = idx_value << 2;
		3912	if (command & PACKET3_CP_DMA_CMD_SAIC) {
		3913	reg = start_reg;
		3914	if (!si_vm_reg_valid(reg)) {
		3915	DRM_ERROR("CP DMA Bad SRC register\n");
		3916	return -EINVAL;
		3917	}
		3918	} else {
		3919	for (i = 0; i < (command & 0x1fffff); i++) {
		3920	reg = start_reg + (4 * i);
		3921	if (!si_vm_reg_valid(reg)) {
		3922	DRM_ERROR("CP DMA Bad SRC register\n");
		3923	return -EINVAL;
		3924	}
		3925	}
		3926	}
		3927	}
		3928	}
		3929	if (command & PACKET3_CP_DMA_CMD_DAS) {
		3930	/* dst address space is register */
		3931	if (((info & 0x00300000) >> 20) == 0) {
		3932	start_reg = ib[idx + 2];
		3933	if (command & PACKET3_CP_DMA_CMD_DAIC) {
		3934	reg = start_reg;
		3935	if (!si_vm_reg_valid(reg)) {
		3936	DRM_ERROR("CP DMA Bad DST register\n");
		3937	return -EINVAL;
		3938	}
		3939	} else {
		3940	for (i = 0; i < (command & 0x1fffff); i++) {
		3941	reg = start_reg + (4 * i);
		3942	if (!si_vm_reg_valid(reg)) {
		3943	DRM_ERROR("CP DMA Bad DST register\n");
		3944	return -EINVAL;
		3945	}
		3946	}
		3947	}
		3948	}
		3949	}
		3950	break;
2997	Serge	3951	default:
		3952	DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
		3953	return -EINVAL;
		3954	}
		3955	return 0;
		3956	}
		3957
		3958	static int si_vm_packet3_compute_check(struct radeon_device *rdev,
		3959	u32 ib, struct radeon_cs_packet pkt)
		3960	{
		3961	u32 idx = pkt->idx + 1;
		3962	u32 idx_value = ib[idx];
		3963	u32 start_reg, reg, i;
		3964
		3965	switch (pkt->opcode) {
		3966	case PACKET3_NOP:
		3967	case PACKET3_SET_BASE:
		3968	case PACKET3_CLEAR_STATE:
		3969	case PACKET3_DISPATCH_DIRECT:
		3970	case PACKET3_DISPATCH_INDIRECT:
		3971	case PACKET3_ALLOC_GDS:
		3972	case PACKET3_WRITE_GDS_RAM:
		3973	case PACKET3_ATOMIC_GDS:
		3974	case PACKET3_ATOMIC:
		3975	case PACKET3_OCCLUSION_QUERY:
		3976	case PACKET3_SET_PREDICATION:
		3977	case PACKET3_COND_EXEC:
		3978	case PACKET3_PRED_EXEC:
		3979	case PACKET3_CONTEXT_CONTROL:
		3980	case PACKET3_STRMOUT_BUFFER_UPDATE:
		3981	case PACKET3_WAIT_REG_MEM:
		3982	case PACKET3_MEM_WRITE:
		3983	case PACKET3_PFP_SYNC_ME:
		3984	case PACKET3_SURFACE_SYNC:
		3985	case PACKET3_EVENT_WRITE:
		3986	case PACKET3_EVENT_WRITE_EOP:
		3987	case PACKET3_EVENT_WRITE_EOS:
		3988	case PACKET3_SET_CONTEXT_REG:
		3989	case PACKET3_SET_CONTEXT_REG_INDIRECT:
		3990	case PACKET3_SET_SH_REG:
		3991	case PACKET3_SET_SH_REG_OFFSET:
		3992	case PACKET3_INCREMENT_DE_COUNTER:
		3993	case PACKET3_WAIT_ON_CE_COUNTER:
		3994	case PACKET3_WAIT_ON_AVAIL_BUFFER:
		3995	case PACKET3_ME_WRITE:
		3996	break;
		3997	case PACKET3_COPY_DATA:
		3998	if ((idx_value & 0xf00) == 0) {
		3999	reg = ib[idx + 3] * 4;
		4000	if (!si_vm_reg_valid(reg))
		4001	return -EINVAL;
		4002	}
		4003	break;
		4004	case PACKET3_WRITE_DATA:
		4005	if ((idx_value & 0xf00) == 0) {
		4006	start_reg = ib[idx + 1] * 4;
		4007	if (idx_value & 0x10000) {
		4008	if (!si_vm_reg_valid(start_reg))
		4009	return -EINVAL;
		4010	} else {
		4011	for (i = 0; i < (pkt->count - 2); i++) {
		4012	reg = start_reg + (4 * i);
		4013	if (!si_vm_reg_valid(reg))
		4014	return -EINVAL;
		4015	}
		4016	}
		4017	}
		4018	break;
		4019	case PACKET3_COND_WRITE:
		4020	if (idx_value & 0x100) {
		4021	reg = ib[idx + 5] * 4;
		4022	if (!si_vm_reg_valid(reg))
		4023	return -EINVAL;
		4024	}
		4025	break;
		4026	case PACKET3_COPY_DW:
		4027	if (idx_value & 0x2) {
		4028	reg = ib[idx + 3] * 4;
		4029	if (!si_vm_reg_valid(reg))
		4030	return -EINVAL;
		4031	}
		4032	break;
		4033	default:
		4034	DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
		4035	return -EINVAL;
		4036	}
		4037	return 0;
		4038	}
		4039
		4040	int si_ib_parse(struct radeon_device rdev, struct radeon_ib ib)
		4041	{
		4042	int ret = 0;
		4043	u32 idx = 0;
		4044	struct radeon_cs_packet pkt;
		4045
		4046	do {
		4047	pkt.idx = idx;
3764	Serge	4048	pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
		4049	pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
2997	Serge	4050	pkt.one_reg_wr = 0;
		4051	switch (pkt.type) {
3764	Serge	4052	case RADEON_PACKET_TYPE0:
2997	Serge	4053	dev_err(rdev->dev, "Packet0 not allowed!\n");
		4054	ret = -EINVAL;
		4055	break;
3764	Serge	4056	case RADEON_PACKET_TYPE2:
2997	Serge	4057	idx += 1;
		4058	break;
3764	Serge	4059	case RADEON_PACKET_TYPE3:
		4060	pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
2997	Serge	4061	if (ib->is_const_ib)
		4062	ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
		4063	else {
		4064	switch (ib->ring) {
		4065	case RADEON_RING_TYPE_GFX_INDEX:
		4066	ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
		4067	break;
		4068	case CAYMAN_RING_TYPE_CP1_INDEX:
		4069	case CAYMAN_RING_TYPE_CP2_INDEX:
		4070	ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
		4071	break;
		4072	default:
		4073	dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
		4074	ret = -EINVAL;
		4075	break;
		4076	}
		4077	}
		4078	idx += pkt.count + 2;
		4079	break;
		4080	default:
		4081	dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
		4082	ret = -EINVAL;
		4083	break;
		4084	}
		4085	if (ret)
		4086	break;
		4087	} while (idx < ib->length_dw);
		4088
		4089	return ret;
		4090	}
		4091
		4092	/*
		4093	* vm
		4094	*/
		4095	int si_vm_init(struct radeon_device *rdev)
		4096	{
		4097	/* number of VMs */
		4098	rdev->vm_manager.nvm = 16;
		4099	/* base offset of vram pages */
		4100	rdev->vm_manager.vram_base_offset = 0;
		4101
		4102	return 0;
		4103	}
		4104
		4105	void si_vm_fini(struct radeon_device *rdev)
		4106	{
		4107	}
		4108
		4109	/**
		4110	* si_vm_set_page - update the page tables using the CP
		4111	*
		4112	* @rdev: radeon_device pointer
3764	Serge	4113	* @ib: indirect buffer to fill with commands
2997	Serge	4114	* @pe: addr of the page entry
		4115	* @addr: dst addr to write into pe
		4116	* @count: number of page entries to update
		4117	* @incr: increase next addr by incr bytes
		4118	* @flags: access flags
		4119	*
3764	Serge	4120	* Update the page tables using the CP (SI).
2997	Serge	4121	*/
3764	Serge	4122	void si_vm_set_page(struct radeon_device *rdev,
		4123	struct radeon_ib *ib,
		4124	uint64_t pe,
2997	Serge	4125	uint64_t addr, unsigned count,
		4126	uint32_t incr, uint32_t flags)
		4127	{
		4128	uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
3192	Serge	4129	uint64_t value;
		4130	unsigned ndw;
2997	Serge	4131
3192	Serge	4132	if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
2997	Serge	4133	while (count) {
3192	Serge	4134	ndw = 2 + count * 2;
2997	Serge	4135	if (ndw > 0x3FFE)
		4136	ndw = 0x3FFE;
		4137
3764	Serge	4138	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
		4139	ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) \|
		4140	WRITE_DATA_DST_SEL(1));
		4141	ib->ptr[ib->length_dw++] = pe;
		4142	ib->ptr[ib->length_dw++] = upper_32_bits(pe);
2997	Serge	4143	for (; ndw > 2; ndw -= 2, --count, pe += 8) {
		4144	if (flags & RADEON_VM_PAGE_SYSTEM) {
		4145	value = radeon_vm_map_gart(rdev, addr);
		4146	value &= 0xFFFFFFFFFFFFF000ULL;
3192	Serge	4147	} else if (flags & RADEON_VM_PAGE_VALID) {
2997	Serge	4148	value = addr;
3192	Serge	4149	} else {
2997	Serge	4150	value = 0;
3192	Serge	4151	}
3764	Serge	4152	addr += incr;
		4153	value \|= r600_flags;
		4154	ib->ptr[ib->length_dw++] = value;
		4155	ib->ptr[ib->length_dw++] = upper_32_bits(value);
		4156	}
2997	Serge	4157	}
3192	Serge	4158	} else {
		4159	/* DMA */
		4160	if (flags & RADEON_VM_PAGE_SYSTEM) {
		4161	while (count) {
		4162	ndw = count * 2;
		4163	if (ndw > 0xFFFFE)
		4164	ndw = 0xFFFFE;
		4165
		4166	/* for non-physically contiguous pages (system) */
3764	Serge	4167	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
		4168	ib->ptr[ib->length_dw++] = pe;
		4169	ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
3192	Serge	4170	for (; ndw > 0; ndw -= 2, --count, pe += 8) {
		4171	if (flags & RADEON_VM_PAGE_SYSTEM) {
		4172	value = radeon_vm_map_gart(rdev, addr);
		4173	value &= 0xFFFFFFFFFFFFF000ULL;
		4174	} else if (flags & RADEON_VM_PAGE_VALID) {
		4175	value = addr;
		4176	} else {
		4177	value = 0;
		4178	}
		4179	addr += incr;
		4180	value \|= r600_flags;
3764	Serge	4181	ib->ptr[ib->length_dw++] = value;
		4182	ib->ptr[ib->length_dw++] = upper_32_bits(value);
3192	Serge	4183	}
		4184	}
		4185	} else {
		4186	while (count) {
		4187	ndw = count * 2;
		4188	if (ndw > 0xFFFFE)
		4189	ndw = 0xFFFFE;
		4190
		4191	if (flags & RADEON_VM_PAGE_VALID)
		4192	value = addr;
		4193	else
		4194	value = 0;
		4195	/* for physically contiguous pages (vram) */
3764	Serge	4196	ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
		4197	ib->ptr[ib->length_dw++] = pe; /* dst addr */
		4198	ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
		4199	ib->ptr[ib->length_dw++] = r600_flags; /* mask */
		4200	ib->ptr[ib->length_dw++] = 0;
		4201	ib->ptr[ib->length_dw++] = value; /* value */
		4202	ib->ptr[ib->length_dw++] = upper_32_bits(value);
		4203	ib->ptr[ib->length_dw++] = incr; /* increment size */
		4204	ib->ptr[ib->length_dw++] = 0;
3192	Serge	4205	pe += ndw * 4;
		4206	addr += (ndw / 2) * incr;
		4207	count -= ndw / 2;
		4208	}
		4209	}
3764	Serge	4210	while (ib->length_dw & 0x7)
		4211	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
3192	Serge	4212	}
2997	Serge	4213	}
		4214
		4215	void si_vm_flush(struct radeon_device rdev, int ridx, struct radeon_vm vm)
		4216	{
		4217	struct radeon_ring *ring = &rdev->ring[ridx];
		4218
		4219	if (vm == NULL)
		4220	return;
		4221
		4222	/* write new base address */
		4223	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
		4224	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) \|
		4225	WRITE_DATA_DST_SEL(0)));
		4226
		4227	if (vm->id < 8) {
		4228	radeon_ring_write(ring,
		4229	(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
		4230	} else {
		4231	radeon_ring_write(ring,
		4232	(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
		4233	}
		4234	radeon_ring_write(ring, 0);
		4235	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
		4236
		4237	/* flush hdp cache */
		4238	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
		4239	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) \|
		4240	WRITE_DATA_DST_SEL(0)));
		4241	radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
		4242	radeon_ring_write(ring, 0);
		4243	radeon_ring_write(ring, 0x1);
		4244
		4245	/* bits 0-15 are the VM contexts0-15 */
		4246	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
		4247	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) \|
		4248	WRITE_DATA_DST_SEL(0)));
		4249	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
		4250	radeon_ring_write(ring, 0);
		4251	radeon_ring_write(ring, 1 << vm->id);
		4252
		4253	/* sync PFP to ME, otherwise we might get invalid PFP reads */
		4254	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
		4255	radeon_ring_write(ring, 0x0);
		4256	}
		4257
3192	Serge	4258	void si_dma_vm_flush(struct radeon_device rdev, int ridx, struct radeon_vm vm)
		4259	{
		4260	struct radeon_ring *ring = &rdev->ring[ridx];
		4261
		4262	if (vm == NULL)
		4263	return;
		4264
		4265	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
		4266	if (vm->id < 8) {
		4267	radeon_ring_write(ring, (0xf << 16) \| ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
		4268	} else {
		4269	radeon_ring_write(ring, (0xf << 16) \| ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
		4270	}
		4271	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
		4272
		4273	/* flush hdp cache */
		4274	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
		4275	radeon_ring_write(ring, (0xf << 16) \| (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
		4276	radeon_ring_write(ring, 1);
		4277
		4278	/* bits 0-7 are the VM contexts0-7 */
		4279	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
		4280	radeon_ring_write(ring, (0xf << 16) \| (VM_INVALIDATE_REQUEST >> 2));
		4281	radeon_ring_write(ring, 1 << vm->id);
		4282	}
		4283
2997	Serge	4284	/*
		4285	* RLC
		4286	*/
		4287	void si_rlc_fini(struct radeon_device *rdev)
		4288	{
		4289	int r;
		4290
		4291	/* save restore block */
		4292	if (rdev->rlc.save_restore_obj) {
		4293	r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
		4294	if (unlikely(r != 0))
		4295	dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
		4296	radeon_bo_unpin(rdev->rlc.save_restore_obj);
		4297	radeon_bo_unreserve(rdev->rlc.save_restore_obj);
		4298
		4299	radeon_bo_unref(&rdev->rlc.save_restore_obj);
		4300	rdev->rlc.save_restore_obj = NULL;
		4301	}
		4302
		4303	/* clear state block */
		4304	if (rdev->rlc.clear_state_obj) {
		4305	r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
		4306	if (unlikely(r != 0))
		4307	dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
		4308	radeon_bo_unpin(rdev->rlc.clear_state_obj);
		4309	radeon_bo_unreserve(rdev->rlc.clear_state_obj);
		4310
		4311	radeon_bo_unref(&rdev->rlc.clear_state_obj);
		4312	rdev->rlc.clear_state_obj = NULL;
		4313	}
		4314	}
		4315
		4316	int si_rlc_init(struct radeon_device *rdev)
		4317	{
		4318	int r;
		4319
		4320	/* save restore block */
		4321	if (rdev->rlc.save_restore_obj == NULL) {
		4322	r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
		4323	RADEON_GEM_DOMAIN_VRAM, NULL,
		4324	&rdev->rlc.save_restore_obj);
		4325	if (r) {
		4326	dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
		4327	return r;
		4328	}
		4329	}
		4330
		4331	r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
		4332	if (unlikely(r != 0)) {
		4333	si_rlc_fini(rdev);
		4334	return r;
		4335	}
		4336	r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
		4337	&rdev->rlc.save_restore_gpu_addr);
		4338	radeon_bo_unreserve(rdev->rlc.save_restore_obj);
		4339	if (r) {
		4340	dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
		4341	si_rlc_fini(rdev);
		4342	return r;
		4343	}
		4344
		4345	/* clear state block */
		4346	if (rdev->rlc.clear_state_obj == NULL) {
		4347	r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
		4348	RADEON_GEM_DOMAIN_VRAM, NULL,
		4349	&rdev->rlc.clear_state_obj);
		4350	if (r) {
		4351	dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
		4352	si_rlc_fini(rdev);
		4353	return r;
		4354	}
		4355	}
		4356	r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
		4357	if (unlikely(r != 0)) {
		4358	si_rlc_fini(rdev);
		4359	return r;
		4360	}
		4361	r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
		4362	&rdev->rlc.clear_state_gpu_addr);
		4363	radeon_bo_unreserve(rdev->rlc.clear_state_obj);
		4364	if (r) {
		4365	dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
		4366	si_rlc_fini(rdev);
		4367	return r;
		4368	}
		4369
		4370	return 0;
		4371	}
		4372
		4373	static void si_rlc_stop(struct radeon_device *rdev)
		4374	{
		4375	WREG32(RLC_CNTL, 0);
		4376	}
		4377
		4378	static void si_rlc_start(struct radeon_device *rdev)
		4379	{
		4380	WREG32(RLC_CNTL, RLC_ENABLE);
		4381	}
		4382
		4383	static int si_rlc_resume(struct radeon_device *rdev)
		4384	{
		4385	u32 i;
		4386	const __be32 *fw_data;
		4387
		4388	if (!rdev->rlc_fw)
		4389	return -EINVAL;
		4390
		4391	si_rlc_stop(rdev);
		4392
		4393	WREG32(RLC_RL_BASE, 0);
		4394	WREG32(RLC_RL_SIZE, 0);
		4395	WREG32(RLC_LB_CNTL, 0);
		4396	WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
		4397	WREG32(RLC_LB_CNTR_INIT, 0);
		4398
		4399	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		4400	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		4401
		4402	WREG32(RLC_MC_CNTL, 0);
		4403	WREG32(RLC_UCODE_CNTL, 0);
		4404
		4405	fw_data = (const __be32 *)rdev->rlc_fw->data;
		4406	for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
		4407	WREG32(RLC_UCODE_ADDR, i);
		4408	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		4409	}
		4410	WREG32(RLC_UCODE_ADDR, 0);
		4411
		4412	si_rlc_start(rdev);
		4413
		4414	return 0;
		4415	}
		4416
		4417	static void si_enable_interrupts(struct radeon_device *rdev)
		4418	{
		4419	u32 ih_cntl = RREG32(IH_CNTL);
		4420	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		4421
		4422	ih_cntl \|= ENABLE_INTR;
		4423	ih_rb_cntl \|= IH_RB_ENABLE;
		4424	WREG32(IH_CNTL, ih_cntl);
		4425	WREG32(IH_RB_CNTL, ih_rb_cntl);
		4426	rdev->ih.enabled = true;
		4427	}
		4428
		4429	static void si_disable_interrupts(struct radeon_device *rdev)
		4430	{
		4431	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		4432	u32 ih_cntl = RREG32(IH_CNTL);
		4433
		4434	ih_rb_cntl &= ~IH_RB_ENABLE;
		4435	ih_cntl &= ~ENABLE_INTR;
		4436	WREG32(IH_RB_CNTL, ih_rb_cntl);
		4437	WREG32(IH_CNTL, ih_cntl);
		4438	/* set rptr, wptr to 0 */
		4439	WREG32(IH_RB_RPTR, 0);
		4440	WREG32(IH_RB_WPTR, 0);
		4441	rdev->ih.enabled = false;
		4442	rdev->ih.rptr = 0;
		4443	}
		4444
		4445	static void si_disable_interrupt_state(struct radeon_device *rdev)
		4446	{
		4447	u32 tmp;
		4448
		4449	WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		4450	WREG32(CP_INT_CNTL_RING1, 0);
		4451	WREG32(CP_INT_CNTL_RING2, 0);
3192	Serge	4452	tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
		4453	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
		4454	tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
		4455	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
2997	Serge	4456	WREG32(GRBM_INT_CNTL, 0);
3764	Serge	4457	if (rdev->num_crtc >= 2) {
2997	Serge	4458	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		4459	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
3764	Serge	4460	}
2997	Serge	4461	if (rdev->num_crtc >= 4) {
		4462	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		4463	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		4464	}
		4465	if (rdev->num_crtc >= 6) {
		4466	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		4467	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		4468	}
		4469
3764	Serge	4470	if (rdev->num_crtc >= 2) {
2997	Serge	4471	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
		4472	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
3764	Serge	4473	}
2997	Serge	4474	if (rdev->num_crtc >= 4) {
		4475	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
		4476	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
		4477	}
		4478	if (rdev->num_crtc >= 6) {
		4479	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
		4480	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
		4481	}
		4482
3764	Serge	4483	if (!ASIC_IS_NODCE(rdev)) {
2997	Serge	4484	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		4485
		4486	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4487	WREG32(DC_HPD1_INT_CONTROL, tmp);
		4488	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4489	WREG32(DC_HPD2_INT_CONTROL, tmp);
		4490	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4491	WREG32(DC_HPD3_INT_CONTROL, tmp);
		4492	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4493	WREG32(DC_HPD4_INT_CONTROL, tmp);
		4494	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4495	WREG32(DC_HPD5_INT_CONTROL, tmp);
		4496	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		4497	WREG32(DC_HPD6_INT_CONTROL, tmp);
3764	Serge	4498	}
2997	Serge	4499	}
		4500
		4501	static int si_irq_init(struct radeon_device *rdev)
		4502	{
		4503	int ret = 0;
		4504	int rb_bufsz;
		4505	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
		4506
		4507	/* allocate ring */
		4508	ret = r600_ih_ring_alloc(rdev);
		4509	if (ret)
		4510	return ret;
		4511
		4512	/* disable irqs */
		4513	si_disable_interrupts(rdev);
		4514
		4515	/* init rlc */
		4516	ret = si_rlc_resume(rdev);
		4517	if (ret) {
		4518	r600_ih_ring_fini(rdev);
		4519	return ret;
		4520	}
		4521
		4522	/* setup interrupt control */
		4523	/* set dummy read address to ring address */
		4524	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
		4525	interrupt_cntl = RREG32(INTERRUPT_CNTL);
		4526	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
		4527	* IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
		4528	*/
		4529	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
		4530	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
		4531	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
		4532	WREG32(INTERRUPT_CNTL, interrupt_cntl);
		4533
		4534	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
		4535	rb_bufsz = drm_order(rdev->ih.ring_size / 4);
		4536
		4537	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE \|
		4538	IH_WPTR_OVERFLOW_CLEAR \|
		4539	(rb_bufsz << 1));
		4540
		4541	if (rdev->wb.enabled)
		4542	ih_rb_cntl \|= IH_WPTR_WRITEBACK_ENABLE;
		4543
		4544	/* set the writeback address whether it's enabled or not */
		4545	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
		4546	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
		4547
		4548	WREG32(IH_RB_CNTL, ih_rb_cntl);
		4549
		4550	/* set rptr, wptr to 0 */
		4551	WREG32(IH_RB_RPTR, 0);
		4552	WREG32(IH_RB_WPTR, 0);
		4553
		4554	/* Default settings for IH_CNTL (disabled at first) */
		4555	ih_cntl = MC_WRREQ_CREDIT(0x10) \| MC_WR_CLEAN_CNT(0x10) \| MC_VMID(0);
		4556	/* RPTR_REARM only works if msi's are enabled */
		4557	if (rdev->msi_enabled)
		4558	ih_cntl \|= RPTR_REARM;
		4559	WREG32(IH_CNTL, ih_cntl);
		4560
		4561	/* force the active interrupt state to all disabled */
		4562	si_disable_interrupt_state(rdev);
		4563
		4564	pci_set_master(rdev->pdev);
		4565
		4566	/* enable irqs */
		4567	si_enable_interrupts(rdev);
		4568
		4569	return ret;
		4570	}
		4571
		4572	int si_irq_set(struct radeon_device *rdev)
		4573	{
		4574	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
		4575	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
		4576	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
3764	Serge	4577	u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
2997	Serge	4578	u32 grbm_int_cntl = 0;
		4579	u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
3192	Serge	4580	u32 dma_cntl, dma_cntl1;
2997	Serge	4581
		4582	if (!rdev->irq.installed) {
		4583	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		4584	return -EINVAL;
		4585	}
		4586	/* don't enable anything if the ih is disabled */
		4587	if (!rdev->ih.enabled) {
		4588	si_disable_interrupts(rdev);
		4589	/* force the active interrupt state to all disabled */
		4590	si_disable_interrupt_state(rdev);
		4591	return 0;
		4592	}
		4593
3764	Serge	4594	if (!ASIC_IS_NODCE(rdev)) {
2997	Serge	4595	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		4596	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		4597	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		4598	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		4599	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		4600	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
3764	Serge	4601	}
2997	Serge	4602
3192	Serge	4603	dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
		4604	dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
		4605
2997	Serge	4606	/* enable CP interrupts on all rings */
		4607	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
		4608	DRM_DEBUG("si_irq_set: sw int gfx\n");
		4609	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		4610	}
		4611	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
		4612	DRM_DEBUG("si_irq_set: sw int cp1\n");
		4613	cp_int_cntl1 \|= TIME_STAMP_INT_ENABLE;
		4614	}
		4615	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
		4616	DRM_DEBUG("si_irq_set: sw int cp2\n");
		4617	cp_int_cntl2 \|= TIME_STAMP_INT_ENABLE;
		4618	}
3192	Serge	4619	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
		4620	DRM_DEBUG("si_irq_set: sw int dma\n");
		4621	dma_cntl \|= TRAP_ENABLE;
		4622	}
		4623
		4624	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
		4625	DRM_DEBUG("si_irq_set: sw int dma1\n");
		4626	dma_cntl1 \|= TRAP_ENABLE;
		4627	}
2997	Serge	4628	if (rdev->irq.crtc_vblank_int[0] \|\|
		4629	atomic_read(&rdev->irq.pflip[0])) {
		4630	DRM_DEBUG("si_irq_set: vblank 0\n");
		4631	crtc1 \|= VBLANK_INT_MASK;
		4632	}
		4633	if (rdev->irq.crtc_vblank_int[1] \|\|
		4634	atomic_read(&rdev->irq.pflip[1])) {
		4635	DRM_DEBUG("si_irq_set: vblank 1\n");
		4636	crtc2 \|= VBLANK_INT_MASK;
		4637	}
		4638	if (rdev->irq.crtc_vblank_int[2] \|\|
		4639	atomic_read(&rdev->irq.pflip[2])) {
		4640	DRM_DEBUG("si_irq_set: vblank 2\n");
		4641	crtc3 \|= VBLANK_INT_MASK;
		4642	}
		4643	if (rdev->irq.crtc_vblank_int[3] \|\|
		4644	atomic_read(&rdev->irq.pflip[3])) {
		4645	DRM_DEBUG("si_irq_set: vblank 3\n");
		4646	crtc4 \|= VBLANK_INT_MASK;
		4647	}
		4648	if (rdev->irq.crtc_vblank_int[4] \|\|
		4649	atomic_read(&rdev->irq.pflip[4])) {
		4650	DRM_DEBUG("si_irq_set: vblank 4\n");
		4651	crtc5 \|= VBLANK_INT_MASK;
		4652	}
		4653	if (rdev->irq.crtc_vblank_int[5] \|\|
		4654	atomic_read(&rdev->irq.pflip[5])) {
		4655	DRM_DEBUG("si_irq_set: vblank 5\n");
		4656	crtc6 \|= VBLANK_INT_MASK;
		4657	}
		4658	if (rdev->irq.hpd[0]) {
		4659	DRM_DEBUG("si_irq_set: hpd 1\n");
		4660	hpd1 \|= DC_HPDx_INT_EN;
		4661	}
		4662	if (rdev->irq.hpd[1]) {
		4663	DRM_DEBUG("si_irq_set: hpd 2\n");
		4664	hpd2 \|= DC_HPDx_INT_EN;
		4665	}
		4666	if (rdev->irq.hpd[2]) {
		4667	DRM_DEBUG("si_irq_set: hpd 3\n");
		4668	hpd3 \|= DC_HPDx_INT_EN;
		4669	}
		4670	if (rdev->irq.hpd[3]) {
		4671	DRM_DEBUG("si_irq_set: hpd 4\n");
		4672	hpd4 \|= DC_HPDx_INT_EN;
		4673	}
		4674	if (rdev->irq.hpd[4]) {
		4675	DRM_DEBUG("si_irq_set: hpd 5\n");
		4676	hpd5 \|= DC_HPDx_INT_EN;
		4677	}
		4678	if (rdev->irq.hpd[5]) {
		4679	DRM_DEBUG("si_irq_set: hpd 6\n");
		4680	hpd6 \|= DC_HPDx_INT_EN;
		4681	}
		4682
		4683	WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
		4684	WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
		4685	WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
		4686
3192	Serge	4687	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
		4688	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
		4689
2997	Serge	4690	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		4691
3764	Serge	4692	if (rdev->num_crtc >= 2) {
2997	Serge	4693	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
		4694	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
3764	Serge	4695	}
2997	Serge	4696	if (rdev->num_crtc >= 4) {
		4697	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
		4698	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
		4699	}
		4700	if (rdev->num_crtc >= 6) {
		4701	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
		4702	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
		4703	}
		4704
3764	Serge	4705	if (rdev->num_crtc >= 2) {
2997	Serge	4706	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
		4707	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
3764	Serge	4708	}
2997	Serge	4709	if (rdev->num_crtc >= 4) {
		4710	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
		4711	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
		4712	}
		4713	if (rdev->num_crtc >= 6) {
		4714	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
		4715	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
		4716	}
		4717
3764	Serge	4718	if (!ASIC_IS_NODCE(rdev)) {
2997	Serge	4719	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		4720	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		4721	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		4722	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		4723	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		4724	WREG32(DC_HPD6_INT_CONTROL, hpd6);
3764	Serge	4725	}
2997	Serge	4726
		4727	return 0;
		4728	}
		4729
		4730	static inline void si_irq_ack(struct radeon_device *rdev)
		4731	{
		4732	u32 tmp;
		4733
3764	Serge	4734	if (ASIC_IS_NODCE(rdev))
		4735	return;
		4736
2997	Serge	4737	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		4738	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		4739	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
		4740	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
		4741	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
		4742	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
		4743	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
		4744	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
		4745	if (rdev->num_crtc >= 4) {
		4746	rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
		4747	rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
		4748	}
		4749	if (rdev->num_crtc >= 6) {
		4750	rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
		4751	rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
		4752	}
		4753
		4754	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
		4755	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4756	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
		4757	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4758	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
		4759	WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
		4760	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
		4761	WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
		4762	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
		4763	WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
		4764	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
		4765	WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
		4766
		4767	if (rdev->num_crtc >= 4) {
		4768	if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
		4769	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4770	if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
		4771	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4772	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
		4773	WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
		4774	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
		4775	WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
		4776	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
		4777	WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
		4778	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
		4779	WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
		4780	}
		4781
		4782	if (rdev->num_crtc >= 6) {
		4783	if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
		4784	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4785	if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
		4786	WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
		4787	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
		4788	WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
		4789	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
		4790	WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
		4791	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
		4792	WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
		4793	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
		4794	WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
		4795	}
		4796
		4797	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		4798	tmp = RREG32(DC_HPD1_INT_CONTROL);
		4799	tmp \|= DC_HPDx_INT_ACK;
		4800	WREG32(DC_HPD1_INT_CONTROL, tmp);
		4801	}
		4802	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		4803	tmp = RREG32(DC_HPD2_INT_CONTROL);
		4804	tmp \|= DC_HPDx_INT_ACK;
		4805	WREG32(DC_HPD2_INT_CONTROL, tmp);
		4806	}
		4807	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		4808	tmp = RREG32(DC_HPD3_INT_CONTROL);
		4809	tmp \|= DC_HPDx_INT_ACK;
		4810	WREG32(DC_HPD3_INT_CONTROL, tmp);
		4811	}
		4812	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		4813	tmp = RREG32(DC_HPD4_INT_CONTROL);
		4814	tmp \|= DC_HPDx_INT_ACK;
		4815	WREG32(DC_HPD4_INT_CONTROL, tmp);
		4816	}
		4817	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		4818	tmp = RREG32(DC_HPD5_INT_CONTROL);
		4819	tmp \|= DC_HPDx_INT_ACK;
		4820	WREG32(DC_HPD5_INT_CONTROL, tmp);
		4821	}
		4822	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		4823	tmp = RREG32(DC_HPD5_INT_CONTROL);
		4824	tmp \|= DC_HPDx_INT_ACK;
		4825	WREG32(DC_HPD6_INT_CONTROL, tmp);
		4826	}
		4827	}
		4828
		4829	static void si_irq_disable(struct radeon_device *rdev)
		4830	{
		4831	si_disable_interrupts(rdev);
		4832	/* Wait and acknowledge irq */
		4833	mdelay(1);
		4834	si_irq_ack(rdev);
		4835	si_disable_interrupt_state(rdev);
		4836	}
		4837
		4838	static void si_irq_suspend(struct radeon_device *rdev)
		4839	{
		4840	si_irq_disable(rdev);
		4841	si_rlc_stop(rdev);
		4842	}
		4843
		4844	static void si_irq_fini(struct radeon_device *rdev)
		4845	{
		4846	si_irq_suspend(rdev);
		4847	r600_ih_ring_fini(rdev);
		4848	}
		4849
		4850	static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
		4851	{
		4852	u32 wptr, tmp;
		4853
		4854	if (rdev->wb.enabled)
		4855	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		4856	else
		4857	wptr = RREG32(IH_RB_WPTR);
		4858
		4859	if (wptr & RB_OVERFLOW) {
		4860	/* When a ring buffer overflow happen start parsing interrupt
		4861	* from the last not overwritten vector (wptr + 16). Hopefully
		4862	* this should allow us to catchup.
		4863	*/
		4864	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
		4865	wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
		4866	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		4867	tmp = RREG32(IH_RB_CNTL);
		4868	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		4869	WREG32(IH_RB_CNTL, tmp);
		4870	}
		4871	return (wptr & rdev->ih.ptr_mask);
		4872	}
		4873
		4874	/* SI IV Ring
		4875	* Each IV ring entry is 128 bits:
		4876	* [7:0] - interrupt source id
		4877	* [31:8] - reserved
		4878	* [59:32] - interrupt source data
		4879	* [63:60] - reserved
		4880	* [71:64] - RINGID
		4881	* [79:72] - VMID
		4882	* [127:80] - reserved
		4883	*/
		4884	int si_irq_process(struct radeon_device *rdev)
		4885	{
		4886	u32 wptr;
		4887	u32 rptr;
		4888	u32 src_id, src_data, ring_id;
		4889	u32 ring_index;
		4890	bool queue_hotplug = false;
		4891
		4892	if (!rdev->ih.enabled \|\| rdev->shutdown)
		4893	return IRQ_NONE;
		4894
		4895	wptr = si_get_ih_wptr(rdev);
		4896
		4897	restart_ih:
		4898	/* is somebody else already processing irqs? */
		4899	if (atomic_xchg(&rdev->ih.lock, 1))
		4900	return IRQ_NONE;
		4901
		4902	rptr = rdev->ih.rptr;
		4903	DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		4904
		4905	/* Order reading of wptr vs. reading of IH ring data */
		4906	rmb();
		4907
		4908	/* display interrupts */
		4909	si_irq_ack(rdev);
		4910
		4911	while (rptr != wptr) {
		4912	/* wptr/rptr are in bytes! */
		4913	ring_index = rptr / 4;
		4914	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		4915	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		4916	ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
		4917
		4918	switch (src_id) {
		4919	case 1: /* D1 vblank/vline */
		4920	switch (src_data) {
		4921	case 0: /* D1 vblank */
		4922	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
		4923	if (rdev->irq.crtc_vblank_int[0]) {
		4924	// drm_handle_vblank(rdev->ddev, 0);
		4925	rdev->pm.vblank_sync = true;
		4926	// wake_up(&rdev->irq.vblank_queue);
		4927	}
		4928	// if (atomic_read(&rdev->irq.pflip[0]))
		4929	// radeon_crtc_handle_flip(rdev, 0);
		4930	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		4931	DRM_DEBUG("IH: D1 vblank\n");
		4932	}
		4933	break;
		4934	case 1: /* D1 vline */
		4935	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
		4936	rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		4937	DRM_DEBUG("IH: D1 vline\n");
		4938	}
		4939	break;
		4940	default:
		4941	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4942	break;
		4943	}
		4944	break;
		4945	case 2: /* D2 vblank/vline */
		4946	switch (src_data) {
		4947	case 0: /* D2 vblank */
		4948	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
		4949	if (rdev->irq.crtc_vblank_int[1]) {
		4950	// drm_handle_vblank(rdev->ddev, 1);
		4951	rdev->pm.vblank_sync = true;
		4952	// wake_up(&rdev->irq.vblank_queue);
		4953	}
		4954	// if (atomic_read(&rdev->irq.pflip[1]))
		4955	// radeon_crtc_handle_flip(rdev, 1);
		4956	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
		4957	DRM_DEBUG("IH: D2 vblank\n");
		4958	}
		4959	break;
		4960	case 1: /* D2 vline */
		4961	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
		4962	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
		4963	DRM_DEBUG("IH: D2 vline\n");
		4964	}
		4965	break;
		4966	default:
		4967	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4968	break;
		4969	}
		4970	break;
		4971	case 3: /* D3 vblank/vline */
		4972	switch (src_data) {
		4973	case 0: /* D3 vblank */
		4974	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
		4975	if (rdev->irq.crtc_vblank_int[2]) {
		4976	// drm_handle_vblank(rdev->ddev, 2);
		4977	rdev->pm.vblank_sync = true;
		4978	// wake_up(&rdev->irq.vblank_queue);
		4979	}
		4980	// if (atomic_read(&rdev->irq.pflip[2]))
		4981	// radeon_crtc_handle_flip(rdev, 2);
		4982	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
		4983	DRM_DEBUG("IH: D3 vblank\n");
		4984	}
		4985	break;
		4986	case 1: /* D3 vline */
		4987	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
		4988	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
		4989	DRM_DEBUG("IH: D3 vline\n");
		4990	}
		4991	break;
		4992	default:
		4993	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4994	break;
		4995	}
		4996	break;
		4997	case 4: /* D4 vblank/vline */
		4998	switch (src_data) {
		4999	case 0: /* D4 vblank */
		5000	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
		5001	if (rdev->irq.crtc_vblank_int[3]) {
		5002	// drm_handle_vblank(rdev->ddev, 3);
		5003	rdev->pm.vblank_sync = true;
		5004	// wake_up(&rdev->irq.vblank_queue);
		5005	}
		5006	// if (atomic_read(&rdev->irq.pflip[3]))
		5007	// radeon_crtc_handle_flip(rdev, 3);
		5008	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
		5009	DRM_DEBUG("IH: D4 vblank\n");
		5010	}
		5011	break;
		5012	case 1: /* D4 vline */
		5013	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
		5014	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
		5015	DRM_DEBUG("IH: D4 vline\n");
		5016	}
		5017	break;
		5018	default:
		5019	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		5020	break;
		5021	}
		5022	break;
		5023	case 5: /* D5 vblank/vline */
		5024	switch (src_data) {
		5025	case 0: /* D5 vblank */
		5026	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
		5027	if (rdev->irq.crtc_vblank_int[4]) {
		5028	// drm_handle_vblank(rdev->ddev, 4);
		5029	rdev->pm.vblank_sync = true;
		5030	// wake_up(&rdev->irq.vblank_queue);
		5031	}
		5032	// if (atomic_read(&rdev->irq.pflip[4]))
		5033	// radeon_crtc_handle_flip(rdev, 4);
		5034	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
		5035	DRM_DEBUG("IH: D5 vblank\n");
		5036	}
		5037	break;
		5038	case 1: /* D5 vline */
		5039	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
		5040	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
		5041	DRM_DEBUG("IH: D5 vline\n");
		5042	}
		5043	break;
		5044	default:
		5045	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		5046	break;
		5047	}
		5048	break;
		5049	case 6: /* D6 vblank/vline */
		5050	switch (src_data) {
		5051	case 0: /* D6 vblank */
		5052	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
		5053	if (rdev->irq.crtc_vblank_int[5]) {
		5054	// drm_handle_vblank(rdev->ddev, 5);
		5055	rdev->pm.vblank_sync = true;
		5056	// wake_up(&rdev->irq.vblank_queue);
		5057	}
		5058	// if (atomic_read(&rdev->irq.pflip[5]))
		5059	// radeon_crtc_handle_flip(rdev, 5);
		5060	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
		5061	DRM_DEBUG("IH: D6 vblank\n");
		5062	}
		5063	break;
		5064	case 1: /* D6 vline */
		5065	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
		5066	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
		5067	DRM_DEBUG("IH: D6 vline\n");
		5068	}
		5069	break;
		5070	default:
		5071	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		5072	break;
		5073	}
		5074	break;
		5075	case 42: /* HPD hotplug */
		5076	switch (src_data) {
		5077	case 0:
		5078	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
		5079	rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
		5080	queue_hotplug = true;
		5081	DRM_DEBUG("IH: HPD1\n");
		5082	}
		5083	break;
		5084	case 1:
		5085	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
		5086	rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
		5087	queue_hotplug = true;
		5088	DRM_DEBUG("IH: HPD2\n");
		5089	}
		5090	break;
		5091	case 2:
		5092	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
		5093	rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
		5094	queue_hotplug = true;
		5095	DRM_DEBUG("IH: HPD3\n");
		5096	}
		5097	break;
		5098	case 3:
		5099	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
		5100	rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
		5101	queue_hotplug = true;
		5102	DRM_DEBUG("IH: HPD4\n");
		5103	}
		5104	break;
		5105	case 4:
		5106	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
		5107	rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
		5108	queue_hotplug = true;
		5109	DRM_DEBUG("IH: HPD5\n");
		5110	}
		5111	break;
		5112	case 5:
		5113	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
		5114	rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
		5115	queue_hotplug = true;
		5116	DRM_DEBUG("IH: HPD6\n");
		5117	}
		5118	break;
		5119	default:
		5120	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		5121	break;
		5122	}
		5123	break;
3192	Serge	5124	case 146:
		5125	case 147:
		5126	dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
		5127	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
		5128	RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
		5129	dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
		5130	RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
		5131	/* reset addr and status */
		5132	WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
		5133	break;
2997	Serge	5134	case 176: /* RINGID0 CP_INT */
		5135	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
		5136	break;
		5137	case 177: /* RINGID1 CP_INT */
		5138	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		5139	break;
		5140	case 178: /* RINGID2 CP_INT */
		5141	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		5142	break;
		5143	case 181: /* CP EOP event */
		5144	DRM_DEBUG("IH: CP EOP\n");
		5145	switch (ring_id) {
		5146	case 0:
		5147	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
		5148	break;
		5149	case 1:
		5150	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		5151	break;
		5152	case 2:
		5153	radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		5154	break;
		5155	}
		5156	break;
3192	Serge	5157	case 224: /* DMA trap event */
		5158	DRM_DEBUG("IH: DMA trap\n");
		5159	radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
		5160	break;
2997	Serge	5161	case 233: /* GUI IDLE */
		5162	DRM_DEBUG("IH: GUI idle\n");
		5163	break;
3192	Serge	5164	case 244: /* DMA trap event */
		5165	DRM_DEBUG("IH: DMA1 trap\n");
		5166	radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
		5167	break;
2997	Serge	5168	default:
		5169	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		5170	break;
		5171	}
		5172
		5173	/* wptr/rptr are in bytes! */
		5174	rptr += 16;
		5175	rptr &= rdev->ih.ptr_mask;
		5176	}
		5177	// if (queue_hotplug)
		5178	// schedule_work(&rdev->hotplug_work);
		5179	rdev->ih.rptr = rptr;
		5180	WREG32(IH_RB_RPTR, rdev->ih.rptr);
		5181	atomic_set(&rdev->ih.lock, 0);
		5182
		5183	/* make sure wptr hasn't changed while processing */
		5184	wptr = si_get_ih_wptr(rdev);
		5185	if (wptr != rptr)
		5186	goto restart_ih;
		5187
		5188	return IRQ_HANDLED;
		5189	}
		5190
3192	Serge	5191	/**
		5192	* si_copy_dma - copy pages using the DMA engine
		5193	*
		5194	* @rdev: radeon_device pointer
		5195	* @src_offset: src GPU address
		5196	* @dst_offset: dst GPU address
		5197	* @num_gpu_pages: number of GPU pages to xfer
		5198	* @fence: radeon fence object
		5199	*
		5200	* Copy GPU paging using the DMA engine (SI).
		5201	* Used by the radeon ttm implementation to move pages if
		5202	* registered as the asic copy callback.
		5203	*/
		5204	int si_copy_dma(struct radeon_device *rdev,
		5205	uint64_t src_offset, uint64_t dst_offset,
		5206	unsigned num_gpu_pages,
		5207	struct radeon_fence **fence)
		5208	{
		5209	struct radeon_semaphore *sem = NULL;
		5210	int ring_index = rdev->asic->copy.dma_ring_index;
		5211	struct radeon_ring *ring = &rdev->ring[ring_index];
		5212	u32 size_in_bytes, cur_size_in_bytes;
		5213	int i, num_loops;
		5214	int r = 0;
		5215
		5216	r = radeon_semaphore_create(rdev, &sem);
		5217	if (r) {
		5218	DRM_ERROR("radeon: moving bo (%d).\n", r);
		5219	return r;
		5220	}
		5221
		5222	size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
		5223	num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
		5224	r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
		5225	if (r) {
		5226	DRM_ERROR("radeon: moving bo (%d).\n", r);
		5227	radeon_semaphore_free(rdev, &sem, NULL);
		5228	return r;
		5229	}
		5230
		5231	if (radeon_fence_need_sync(*fence, ring->idx)) {
		5232	radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
		5233	ring->idx);
		5234	radeon_fence_note_sync(*fence, ring->idx);
		5235	} else {
		5236	radeon_semaphore_free(rdev, &sem, NULL);
		5237	}
		5238
		5239	for (i = 0; i < num_loops; i++) {
		5240	cur_size_in_bytes = size_in_bytes;
		5241	if (cur_size_in_bytes > 0xFFFFF)
		5242	cur_size_in_bytes = 0xFFFFF;
		5243	size_in_bytes -= cur_size_in_bytes;
		5244	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
		5245	radeon_ring_write(ring, dst_offset & 0xffffffff);
		5246	radeon_ring_write(ring, src_offset & 0xffffffff);
		5247	radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
		5248	radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
		5249	src_offset += cur_size_in_bytes;
		5250	dst_offset += cur_size_in_bytes;
		5251	}
		5252
		5253	r = radeon_fence_emit(rdev, fence, ring->idx);
		5254	if (r) {
		5255	radeon_ring_unlock_undo(rdev, ring);
		5256	return r;
		5257	}
		5258
		5259	radeon_ring_unlock_commit(rdev, ring);
		5260	radeon_semaphore_free(rdev, &sem, *fence);
		5261
		5262	return r;
		5263	}
		5264
2997	Serge	5265	/*
		5266	* startup/shutdown callbacks
		5267	*/
		5268	static int si_startup(struct radeon_device *rdev)
		5269	{
		5270	struct radeon_ring *ring;
		5271	int r;
		5272
		5273	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->ce_fw \|\|
		5274	!rdev->rlc_fw \|\| !rdev->mc_fw) {
		5275	r = si_init_microcode(rdev);
		5276	if (r) {
		5277	DRM_ERROR("Failed to load firmware!\n");
		5278	return r;
		5279	}
		5280	}
		5281
		5282	r = si_mc_load_microcode(rdev);
		5283	if (r) {
		5284	DRM_ERROR("Failed to load MC firmware!\n");
		5285	return r;
		5286	}
		5287
		5288	r = r600_vram_scratch_init(rdev);
		5289	if (r)
		5290	return r;
		5291
		5292	si_mc_program(rdev);
		5293	r = si_pcie_gart_enable(rdev);
		5294	if (r)
		5295	return r;
		5296	si_gpu_init(rdev);
		5297
		5298	/* allocate rlc buffers */
		5299	r = si_rlc_init(rdev);
		5300	if (r) {
		5301	DRM_ERROR("Failed to init rlc BOs!\n");
		5302	return r;
		5303	}
		5304
		5305	/* allocate wb buffer */
		5306	r = radeon_wb_init(rdev);
		5307	if (r)
		5308	return r;
		5309
		5310	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
		5311	if (r) {
		5312	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		5313	return r;
		5314	}
		5315
		5316	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		5317	if (r) {
		5318	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		5319	return r;
		5320	}
		5321
		5322	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		5323	if (r) {
		5324	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		5325	return r;
		5326	}
		5327
3192	Serge	5328	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
		5329	if (r) {
		5330	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		5331	return r;
		5332	}
		5333
		5334	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
		5335	if (r) {
		5336	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		5337	return r;
		5338	}
		5339
3764	Serge	5340
2997	Serge	5341	/* Enable IRQ */
3764	Serge	5342	if (!rdev->irq.installed) {
		5343	r = radeon_irq_kms_init(rdev);
		5344	if (r)
		5345	return r;
		5346	}
		5347
2997	Serge	5348	r = si_irq_init(rdev);
		5349	if (r) {
		5350	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		5351	radeon_irq_kms_fini(rdev);
		5352	return r;
		5353	}
		5354	si_irq_set(rdev);
		5355
		5356	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		5357	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
		5358	CP_RB0_RPTR, CP_RB0_WPTR,
		5359	0, 0xfffff, RADEON_CP_PACKET2);
		5360	if (r)
		5361	return r;
		5362
		5363	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
		5364	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
		5365	CP_RB1_RPTR, CP_RB1_WPTR,
		5366	0, 0xfffff, RADEON_CP_PACKET2);
		5367	if (r)
		5368	return r;
		5369
		5370	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
		5371	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
		5372	CP_RB2_RPTR, CP_RB2_WPTR,
		5373	0, 0xfffff, RADEON_CP_PACKET2);
		5374	if (r)
		5375	return r;
		5376
3192	Serge	5377	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		5378	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
		5379	DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
		5380	DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
		5381	2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
		5382	if (r)
		5383	return r;
		5384
		5385	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		5386	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
		5387	DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
		5388	DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
		5389	2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
		5390	if (r)
		5391	return r;
		5392
2997	Serge	5393	r = si_cp_load_microcode(rdev);
		5394	if (r)
		5395	return r;
		5396	r = si_cp_resume(rdev);
		5397	if (r)
		5398	return r;
		5399
3192	Serge	5400	r = cayman_dma_resume(rdev);
		5401	if (r)
		5402	return r;
		5403
3764	Serge	5404
3120	serge	5405	r = radeon_ib_pool_init(rdev);
		5406	if (r) {
		5407	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		5408	return r;
		5409	}
2997	Serge	5410
3764	Serge	5411	r = radeon_vm_manager_init(rdev);
		5412	if (r) {
		5413	dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
		5414	return r;
		5415	}
2997	Serge	5416
		5417	return 0;
		5418	}
		5419
		5420
3120	serge	5421
		5422
2997	Serge	5423	/* Plan is to move initialization in that function and use
		5424	* helper function so that radeon_device_init pretty much
		5425	* do nothing more than calling asic specific function. This
		5426	* should also allow to remove a bunch of callback function
		5427	* like vram_info.
		5428	*/
		5429	int si_init(struct radeon_device *rdev)
		5430	{
		5431	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		5432	int r;
		5433
		5434	ENTER();
		5435
		5436	/* Read BIOS */
		5437	if (!radeon_get_bios(rdev)) {
		5438	if (ASIC_IS_AVIVO(rdev))
		5439	return -EINVAL;
		5440	}
		5441	/* Must be an ATOMBIOS */
		5442	if (!rdev->is_atom_bios) {
		5443	dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
		5444	return -EINVAL;
		5445	}
		5446	r = radeon_atombios_init(rdev);
		5447	if (r)
		5448	return r;
		5449
		5450	/* Post card if necessary */
		5451	if (!radeon_card_posted(rdev)) {
		5452	if (!rdev->bios) {
		5453	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		5454	return -EINVAL;
		5455	}
		5456	DRM_INFO("GPU not posted. posting now...\n");
		5457	atom_asic_init(rdev->mode_info.atom_context);
		5458	}
3764	Serge	5459	/* init golden registers */
		5460	si_init_golden_registers(rdev);
2997	Serge	5461	/* Initialize scratch registers */
		5462	si_scratch_init(rdev);
		5463	/* Initialize surface registers */
		5464	radeon_surface_init(rdev);
		5465	/* Initialize clocks */
		5466	radeon_get_clock_info(rdev->ddev);
		5467
		5468	/* Fence driver */
		5469	r = radeon_fence_driver_init(rdev);
		5470	if (r)
		5471	return r;
		5472
		5473	/* initialize memory controller */
		5474	r = si_mc_init(rdev);
		5475	if (r)
		5476	return r;
		5477	/* Memory manager */
		5478	r = radeon_bo_init(rdev);
		5479	if (r)
		5480	return r;
		5481
		5482	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
		5483	ring->ring_obj = NULL;
		5484	r600_ring_init(rdev, ring, 1024 * 1024);
		5485
		5486	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
		5487	ring->ring_obj = NULL;
		5488	r600_ring_init(rdev, ring, 1024 * 1024);
		5489
		5490	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
		5491	ring->ring_obj = NULL;
		5492	r600_ring_init(rdev, ring, 1024 * 1024);
		5493
3192	Serge	5494	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		5495	ring->ring_obj = NULL;
		5496	r600_ring_init(rdev, ring, 64 * 1024);
		5497
		5498	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		5499	ring->ring_obj = NULL;
		5500	r600_ring_init(rdev, ring, 64 * 1024);
		5501
3764	Serge	5502	rdev->ih.ring_obj = NULL;
2997	Serge	5503	r600_ih_ring_init(rdev, 64 * 1024);
		5504
		5505	r = r600_pcie_gart_init(rdev);
		5506	if (r)
		5507	return r;
		5508
		5509	rdev->accel_working = true;
		5510	r = si_startup(rdev);
		5511	if (r) {
		5512	dev_err(rdev->dev, "disabling GPU acceleration\n");
		5513	// si_cp_fini(rdev);
		5514	// si_irq_fini(rdev);
		5515	// si_rlc_fini(rdev);
		5516	// radeon_wb_fini(rdev);
		5517	// radeon_ib_pool_fini(rdev);
		5518	// radeon_vm_manager_fini(rdev);
		5519	// radeon_irq_kms_fini(rdev);
		5520	// si_pcie_gart_fini(rdev);
		5521	rdev->accel_working = false;
		5522	}
		5523
		5524	/* Don't start up if the MC ucode is missing.
		5525	* The default clocks and voltages before the MC ucode
		5526	* is loaded are not suffient for advanced operations.
		5527	*/
		5528	if (!rdev->mc_fw) {
		5529	DRM_ERROR("radeon: MC ucode required for NI+.\n");
		5530	return -EINVAL;
		5531	}
		5532	LEAVE();
		5533
		5534	return 0;
		5535	}
		5536
		5537	/**
3764	Serge	5538	* si_get_gpu_clock_counter - return GPU clock counter snapshot
2997	Serge	5539	*
		5540	* @rdev: radeon_device pointer
		5541	*
		5542	* Fetches a GPU clock counter snapshot (SI).
		5543	* Returns the 64 bit clock counter snapshot.
		5544	*/
3764	Serge	5545	uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
2997	Serge	5546	{
		5547	uint64_t clock;
		5548
		5549	mutex_lock(&rdev->gpu_clock_mutex);
		5550	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
		5551	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) \|
		5552	((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
		5553	mutex_unlock(&rdev->gpu_clock_mutex);
		5554	return clock;
		5555	}
3764	Serge	5556
		5557	#if 0
		5558	int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
		5559	{
		5560	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
		5561	int r;
		5562
		5563	/* bypass vclk and dclk with bclk */
		5564	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		5565	VCLK_SRC_SEL(1) \| DCLK_SRC_SEL(1),
		5566	~(VCLK_SRC_SEL_MASK \| DCLK_SRC_SEL_MASK));
		5567
		5568	/* put PLL in bypass mode */
		5569	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
		5570
		5571	if (!vclk \|\| !dclk) {
		5572	/* keep the Bypass mode, put PLL to sleep */
		5573	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
		5574	return 0;
		5575	}
		5576
		5577	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
		5578	16384, 0x03FFFFFF, 0, 128, 5,
		5579	&fb_div, &vclk_div, &dclk_div);
		5580	if (r)
		5581	return r;
		5582
		5583	/* set RESET_ANTI_MUX to 0 */
		5584	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
		5585
		5586	/* set VCO_MODE to 1 */
		5587	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
		5588
		5589	/* toggle UPLL_SLEEP to 1 then back to 0 */
		5590	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
		5591	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
		5592
		5593	/* deassert UPLL_RESET */
		5594	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
		5595
		5596	mdelay(1);
		5597
		5598	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
		5599	if (r)
		5600	return r;
		5601
		5602	/* assert UPLL_RESET again */
		5603	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
		5604
		5605	/* disable spread spectrum. */
		5606	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
		5607
		5608	/* set feedback divider */
		5609	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
		5610
		5611	/* set ref divider to 0 */
		5612	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
		5613
		5614	if (fb_div < 307200)
		5615	WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
		5616	else
		5617	WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
		5618
		5619	/* set PDIV_A and PDIV_B */
		5620	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		5621	UPLL_PDIV_A(vclk_div) \| UPLL_PDIV_B(dclk_div),
		5622	~(UPLL_PDIV_A_MASK \| UPLL_PDIV_B_MASK));
		5623
		5624	/* give the PLL some time to settle */
		5625	mdelay(15);
		5626
		5627	/* deassert PLL_RESET */
		5628	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
		5629
		5630	mdelay(15);
		5631
		5632	/* switch from bypass mode to normal mode */
		5633	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
		5634
		5635	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
		5636	if (r)
		5637	return r;
		5638
		5639	/* switch VCLK and DCLK selection */
		5640	WREG32_P(CG_UPLL_FUNC_CNTL_2,
		5641	VCLK_SRC_SEL(2) \| DCLK_SRC_SEL(2),
		5642	~(VCLK_SRC_SEL_MASK \| DCLK_SRC_SEL_MASK));
		5643
		5644	mdelay(100);
		5645
		5646	return 0;
		5647	}
		5648	#endif

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/si.c – Rev 3764