WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/ni.c

Rev	Author	Line No.	Line
1990	serge	1	/*
		2	* Copyright 2010 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	#include
2997	Serge	26	#include
		27	#include
1990	serge	28	#include "radeon.h"
		29	#include "radeon_asic.h"
2997	Serge	30	#include
1990	serge	31	#include "nid.h"
		32	#include "atom.h"
		33	#include "ni_reg.h"
2004	serge	34	#include "cayman_blit_shaders.h"
5078	serge	35	#include "radeon_ucode.h"
		36	#include "clearstate_cayman.h"
1990	serge	37
5078	serge	38	static const u32 tn_rlc_save_restore_register_list[] =
		39	{
		40	0x98fc,
		41	0x98f0,
		42	0x9834,
		43	0x9838,
		44	0x9870,
		45	0x9874,
		46	0x8a14,
		47	0x8b24,
		48	0x8bcc,
		49	0x8b10,
		50	0x8c30,
		51	0x8d00,
		52	0x8d04,
		53	0x8c00,
		54	0x8c04,
		55	0x8c10,
		56	0x8c14,
		57	0x8d8c,
		58	0x8cf0,
		59	0x8e38,
		60	0x9508,
		61	0x9688,
		62	0x9608,
		63	0x960c,
		64	0x9610,
		65	0x9614,
		66	0x88c4,
		67	0x8978,
		68	0x88d4,
		69	0x900c,
		70	0x9100,
		71	0x913c,
		72	0x90e8,
		73	0x9354,
		74	0xa008,
		75	0x98f8,
		76	0x9148,
		77	0x914c,
		78	0x3f94,
		79	0x98f4,
		80	0x9b7c,
		81	0x3f8c,
		82	0x8950,
		83	0x8954,
		84	0x8a18,
		85	0x8b28,
		86	0x9144,
		87	0x3f90,
		88	0x915c,
		89	0x9160,
		90	0x9178,
		91	0x917c,
		92	0x9180,
		93	0x918c,
		94	0x9190,
		95	0x9194,
		96	0x9198,
		97	0x919c,
		98	0x91a8,
		99	0x91ac,
		100	0x91b0,
		101	0x91b4,
		102	0x91b8,
		103	0x91c4,
		104	0x91c8,
		105	0x91cc,
		106	0x91d0,
		107	0x91d4,
		108	0x91e0,
		109	0x91e4,
		110	0x91ec,
		111	0x91f0,
		112	0x91f4,
		113	0x9200,
		114	0x9204,
		115	0x929c,
		116	0x8030,
		117	0x9150,
		118	0x9a60,
		119	0x920c,
		120	0x9210,
		121	0x9228,
		122	0x922c,
		123	0x9244,
		124	0x9248,
		125	0x91e8,
		126	0x9294,
		127	0x9208,
		128	0x9224,
		129	0x9240,
		130	0x9220,
		131	0x923c,
		132	0x9258,
		133	0x9744,
		134	0xa200,
		135	0xa204,
		136	0xa208,
		137	0xa20c,
		138	0x8d58,
		139	0x9030,
		140	0x9034,
		141	0x9038,
		142	0x903c,
		143	0x9040,
		144	0x9654,
		145	0x897c,
		146	0xa210,
		147	0xa214,
		148	0x9868,
		149	0xa02c,
		150	0x9664,
		151	0x9698,
		152	0x949c,
		153	0x8e10,
		154	0x8e18,
		155	0x8c50,
		156	0x8c58,
		157	0x8c60,
		158	0x8c68,
		159	0x89b4,
		160	0x9830,
		161	0x802c,
		162	};
		163
3764	Serge	164	extern bool evergreen_is_display_hung(struct radeon_device *rdev);
		165	extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
1990	serge	166	extern void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save);
		167	extern void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save);
		168	extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
		169	extern void evergreen_mc_program(struct radeon_device *rdev);
		170	extern void evergreen_irq_suspend(struct radeon_device *rdev);
		171	extern int evergreen_mc_init(struct radeon_device *rdev);
2997	Serge	172	extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
		173	extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
5078	serge	174	extern void evergreen_program_aspm(struct radeon_device *rdev);
		175	extern void sumo_rlc_fini(struct radeon_device *rdev);
		176	extern int sumo_rlc_init(struct radeon_device *rdev);
		177	extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
1990	serge	178
		179	/* Firmware Names */
		180	MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
		181	MODULE_FIRMWARE("radeon/BARTS_me.bin");
		182	MODULE_FIRMWARE("radeon/BARTS_mc.bin");
5078	serge	183	MODULE_FIRMWARE("radeon/BARTS_smc.bin");
1990	serge	184	MODULE_FIRMWARE("radeon/BTC_rlc.bin");
		185	MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
		186	MODULE_FIRMWARE("radeon/TURKS_me.bin");
		187	MODULE_FIRMWARE("radeon/TURKS_mc.bin");
5078	serge	188	MODULE_FIRMWARE("radeon/TURKS_smc.bin");
1990	serge	189	MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
		190	MODULE_FIRMWARE("radeon/CAICOS_me.bin");
		191	MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
5078	serge	192	MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
1990	serge	193	MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
		194	MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
		195	MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
		196	MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
5078	serge	197	MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
2997	Serge	198	MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
		199	MODULE_FIRMWARE("radeon/ARUBA_me.bin");
		200	MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
1990	serge	201
3764	Serge	202
		203	static const u32 cayman_golden_registers2[] =
		204	{
		205	0x3e5c, 0xffffffff, 0x00000000,
		206	0x3e48, 0xffffffff, 0x00000000,
		207	0x3e4c, 0xffffffff, 0x00000000,
		208	0x3e64, 0xffffffff, 0x00000000,
		209	0x3e50, 0xffffffff, 0x00000000,
		210	0x3e60, 0xffffffff, 0x00000000
		211	};
		212
		213	static const u32 cayman_golden_registers[] =
		214	{
		215	0x5eb4, 0xffffffff, 0x00000002,
		216	0x5e78, 0x8f311ff1, 0x001000f0,
		217	0x3f90, 0xffff0000, 0xff000000,
		218	0x9148, 0xffff0000, 0xff000000,
		219	0x3f94, 0xffff0000, 0xff000000,
		220	0x914c, 0xffff0000, 0xff000000,
		221	0xc78, 0x00000080, 0x00000080,
		222	0xbd4, 0x70073777, 0x00011003,
		223	0xd02c, 0xbfffff1f, 0x08421000,
		224	0xd0b8, 0x73773777, 0x02011003,
		225	0x5bc0, 0x00200000, 0x50100000,
		226	0x98f8, 0x33773777, 0x02011003,
		227	0x98fc, 0xffffffff, 0x76541032,
		228	0x7030, 0x31000311, 0x00000011,
		229	0x2f48, 0x33773777, 0x42010001,
		230	0x6b28, 0x00000010, 0x00000012,
		231	0x7728, 0x00000010, 0x00000012,
		232	0x10328, 0x00000010, 0x00000012,
		233	0x10f28, 0x00000010, 0x00000012,
		234	0x11b28, 0x00000010, 0x00000012,
		235	0x12728, 0x00000010, 0x00000012,
		236	0x240c, 0x000007ff, 0x00000000,
		237	0x8a14, 0xf000001f, 0x00000007,
		238	0x8b24, 0x3fff3fff, 0x00ff0fff,
		239	0x8b10, 0x0000ff0f, 0x00000000,
		240	0x28a4c, 0x07ffffff, 0x06000000,
		241	0x10c, 0x00000001, 0x00010003,
		242	0xa02c, 0xffffffff, 0x0000009b,
		243	0x913c, 0x0000010f, 0x01000100,
		244	0x8c04, 0xf8ff00ff, 0x40600060,
		245	0x28350, 0x00000f01, 0x00000000,
		246	0x9508, 0x3700001f, 0x00000002,
		247	0x960c, 0xffffffff, 0x54763210,
		248	0x88c4, 0x001f3ae3, 0x00000082,
		249	0x88d0, 0xffffffff, 0x0f40df40,
		250	0x88d4, 0x0000001f, 0x00000010,
		251	0x8974, 0xffffffff, 0x00000000
		252	};
		253
		254	static const u32 dvst_golden_registers2[] =
		255	{
		256	0x8f8, 0xffffffff, 0,
		257	0x8fc, 0x00380000, 0,
		258	0x8f8, 0xffffffff, 1,
		259	0x8fc, 0x0e000000, 0
		260	};
		261
		262	static const u32 dvst_golden_registers[] =
		263	{
		264	0x690, 0x3fff3fff, 0x20c00033,
		265	0x918c, 0x0fff0fff, 0x00010006,
		266	0x91a8, 0x0fff0fff, 0x00010006,
		267	0x9150, 0xffffdfff, 0x6e944040,
		268	0x917c, 0x0fff0fff, 0x00030002,
		269	0x9198, 0x0fff0fff, 0x00030002,
		270	0x915c, 0x0fff0fff, 0x00010000,
		271	0x3f90, 0xffff0001, 0xff000000,
		272	0x9178, 0x0fff0fff, 0x00070000,
		273	0x9194, 0x0fff0fff, 0x00070000,
		274	0x9148, 0xffff0001, 0xff000000,
		275	0x9190, 0x0fff0fff, 0x00090008,
		276	0x91ac, 0x0fff0fff, 0x00090008,
		277	0x3f94, 0xffff0000, 0xff000000,
		278	0x914c, 0xffff0000, 0xff000000,
		279	0x929c, 0x00000fff, 0x00000001,
		280	0x55e4, 0xff607fff, 0xfc000100,
		281	0x8a18, 0xff000fff, 0x00000100,
		282	0x8b28, 0xff000fff, 0x00000100,
		283	0x9144, 0xfffc0fff, 0x00000100,
		284	0x6ed8, 0x00010101, 0x00010000,
		285	0x9830, 0xffffffff, 0x00000000,
		286	0x9834, 0xf00fffff, 0x00000400,
		287	0x9838, 0xfffffffe, 0x00000000,
		288	0xd0c0, 0xff000fff, 0x00000100,
		289	0xd02c, 0xbfffff1f, 0x08421000,
		290	0xd0b8, 0x73773777, 0x12010001,
		291	0x5bb0, 0x000000f0, 0x00000070,
		292	0x98f8, 0x73773777, 0x12010001,
		293	0x98fc, 0xffffffff, 0x00000010,
		294	0x9b7c, 0x00ff0000, 0x00fc0000,
		295	0x8030, 0x00001f0f, 0x0000100a,
		296	0x2f48, 0x73773777, 0x12010001,
		297	0x2408, 0x00030000, 0x000c007f,
		298	0x8a14, 0xf000003f, 0x00000007,
		299	0x8b24, 0x3fff3fff, 0x00ff0fff,
		300	0x8b10, 0x0000ff0f, 0x00000000,
		301	0x28a4c, 0x07ffffff, 0x06000000,
		302	0x4d8, 0x00000fff, 0x00000100,
		303	0xa008, 0xffffffff, 0x00010000,
		304	0x913c, 0xffff03ff, 0x01000100,
		305	0x8c00, 0x000000ff, 0x00000003,
		306	0x8c04, 0xf8ff00ff, 0x40600060,
		307	0x8cf0, 0x1fff1fff, 0x08e00410,
		308	0x28350, 0x00000f01, 0x00000000,
		309	0x9508, 0xf700071f, 0x00000002,
		310	0x960c, 0xffffffff, 0x54763210,
		311	0x20ef8, 0x01ff01ff, 0x00000002,
		312	0x20e98, 0xfffffbff, 0x00200000,
		313	0x2015c, 0xffffffff, 0x00000f40,
		314	0x88c4, 0x001f3ae3, 0x00000082,
		315	0x8978, 0x3fffffff, 0x04050140,
		316	0x88d4, 0x0000001f, 0x00000010,
		317	0x8974, 0xffffffff, 0x00000000
		318	};
		319
		320	static const u32 scrapper_golden_registers[] =
		321	{
		322	0x690, 0x3fff3fff, 0x20c00033,
		323	0x918c, 0x0fff0fff, 0x00010006,
		324	0x918c, 0x0fff0fff, 0x00010006,
		325	0x91a8, 0x0fff0fff, 0x00010006,
		326	0x91a8, 0x0fff0fff, 0x00010006,
		327	0x9150, 0xffffdfff, 0x6e944040,
		328	0x9150, 0xffffdfff, 0x6e944040,
		329	0x917c, 0x0fff0fff, 0x00030002,
		330	0x917c, 0x0fff0fff, 0x00030002,
		331	0x9198, 0x0fff0fff, 0x00030002,
		332	0x9198, 0x0fff0fff, 0x00030002,
		333	0x915c, 0x0fff0fff, 0x00010000,
		334	0x915c, 0x0fff0fff, 0x00010000,
		335	0x3f90, 0xffff0001, 0xff000000,
		336	0x3f90, 0xffff0001, 0xff000000,
		337	0x9178, 0x0fff0fff, 0x00070000,
		338	0x9178, 0x0fff0fff, 0x00070000,
		339	0x9194, 0x0fff0fff, 0x00070000,
		340	0x9194, 0x0fff0fff, 0x00070000,
		341	0x9148, 0xffff0001, 0xff000000,
		342	0x9148, 0xffff0001, 0xff000000,
		343	0x9190, 0x0fff0fff, 0x00090008,
		344	0x9190, 0x0fff0fff, 0x00090008,
		345	0x91ac, 0x0fff0fff, 0x00090008,
		346	0x91ac, 0x0fff0fff, 0x00090008,
		347	0x3f94, 0xffff0000, 0xff000000,
		348	0x3f94, 0xffff0000, 0xff000000,
		349	0x914c, 0xffff0000, 0xff000000,
		350	0x914c, 0xffff0000, 0xff000000,
		351	0x929c, 0x00000fff, 0x00000001,
		352	0x929c, 0x00000fff, 0x00000001,
		353	0x55e4, 0xff607fff, 0xfc000100,
		354	0x8a18, 0xff000fff, 0x00000100,
		355	0x8a18, 0xff000fff, 0x00000100,
		356	0x8b28, 0xff000fff, 0x00000100,
		357	0x8b28, 0xff000fff, 0x00000100,
		358	0x9144, 0xfffc0fff, 0x00000100,
		359	0x9144, 0xfffc0fff, 0x00000100,
		360	0x6ed8, 0x00010101, 0x00010000,
		361	0x9830, 0xffffffff, 0x00000000,
		362	0x9830, 0xffffffff, 0x00000000,
		363	0x9834, 0xf00fffff, 0x00000400,
		364	0x9834, 0xf00fffff, 0x00000400,
		365	0x9838, 0xfffffffe, 0x00000000,
		366	0x9838, 0xfffffffe, 0x00000000,
		367	0xd0c0, 0xff000fff, 0x00000100,
		368	0xd02c, 0xbfffff1f, 0x08421000,
		369	0xd02c, 0xbfffff1f, 0x08421000,
		370	0xd0b8, 0x73773777, 0x12010001,
		371	0xd0b8, 0x73773777, 0x12010001,
		372	0x5bb0, 0x000000f0, 0x00000070,
		373	0x98f8, 0x73773777, 0x12010001,
		374	0x98f8, 0x73773777, 0x12010001,
		375	0x98fc, 0xffffffff, 0x00000010,
		376	0x98fc, 0xffffffff, 0x00000010,
		377	0x9b7c, 0x00ff0000, 0x00fc0000,
		378	0x9b7c, 0x00ff0000, 0x00fc0000,
		379	0x8030, 0x00001f0f, 0x0000100a,
		380	0x8030, 0x00001f0f, 0x0000100a,
		381	0x2f48, 0x73773777, 0x12010001,
		382	0x2f48, 0x73773777, 0x12010001,
		383	0x2408, 0x00030000, 0x000c007f,
		384	0x8a14, 0xf000003f, 0x00000007,
		385	0x8a14, 0xf000003f, 0x00000007,
		386	0x8b24, 0x3fff3fff, 0x00ff0fff,
		387	0x8b24, 0x3fff3fff, 0x00ff0fff,
		388	0x8b10, 0x0000ff0f, 0x00000000,
		389	0x8b10, 0x0000ff0f, 0x00000000,
		390	0x28a4c, 0x07ffffff, 0x06000000,
		391	0x28a4c, 0x07ffffff, 0x06000000,
		392	0x4d8, 0x00000fff, 0x00000100,
		393	0x4d8, 0x00000fff, 0x00000100,
		394	0xa008, 0xffffffff, 0x00010000,
		395	0xa008, 0xffffffff, 0x00010000,
		396	0x913c, 0xffff03ff, 0x01000100,
		397	0x913c, 0xffff03ff, 0x01000100,
		398	0x90e8, 0x001fffff, 0x010400c0,
		399	0x8c00, 0x000000ff, 0x00000003,
		400	0x8c00, 0x000000ff, 0x00000003,
		401	0x8c04, 0xf8ff00ff, 0x40600060,
		402	0x8c04, 0xf8ff00ff, 0x40600060,
		403	0x8c30, 0x0000000f, 0x00040005,
		404	0x8cf0, 0x1fff1fff, 0x08e00410,
		405	0x8cf0, 0x1fff1fff, 0x08e00410,
		406	0x900c, 0x00ffffff, 0x0017071f,
		407	0x28350, 0x00000f01, 0x00000000,
		408	0x28350, 0x00000f01, 0x00000000,
		409	0x9508, 0xf700071f, 0x00000002,
		410	0x9508, 0xf700071f, 0x00000002,
		411	0x9688, 0x00300000, 0x0017000f,
		412	0x960c, 0xffffffff, 0x54763210,
		413	0x960c, 0xffffffff, 0x54763210,
		414	0x20ef8, 0x01ff01ff, 0x00000002,
		415	0x20e98, 0xfffffbff, 0x00200000,
		416	0x2015c, 0xffffffff, 0x00000f40,
		417	0x88c4, 0x001f3ae3, 0x00000082,
		418	0x88c4, 0x001f3ae3, 0x00000082,
		419	0x8978, 0x3fffffff, 0x04050140,
		420	0x8978, 0x3fffffff, 0x04050140,
		421	0x88d4, 0x0000001f, 0x00000010,
		422	0x88d4, 0x0000001f, 0x00000010,
		423	0x8974, 0xffffffff, 0x00000000,
		424	0x8974, 0xffffffff, 0x00000000
		425	};
		426
		427	static void ni_init_golden_registers(struct radeon_device *rdev)
		428	{
		429	switch (rdev->family) {
		430	case CHIP_CAYMAN:
		431	radeon_program_register_sequence(rdev,
		432	cayman_golden_registers,
		433	(const u32)ARRAY_SIZE(cayman_golden_registers));
		434	radeon_program_register_sequence(rdev,
		435	cayman_golden_registers2,
		436	(const u32)ARRAY_SIZE(cayman_golden_registers2));
		437	break;
		438	case CHIP_ARUBA:
		439	if ((rdev->pdev->device == 0x9900) \|\|
		440	(rdev->pdev->device == 0x9901) \|\|
		441	(rdev->pdev->device == 0x9903) \|\|
		442	(rdev->pdev->device == 0x9904) \|\|
		443	(rdev->pdev->device == 0x9905) \|\|
		444	(rdev->pdev->device == 0x9906) \|\|
		445	(rdev->pdev->device == 0x9907) \|\|
		446	(rdev->pdev->device == 0x9908) \|\|
		447	(rdev->pdev->device == 0x9909) \|\|
		448	(rdev->pdev->device == 0x990A) \|\|
		449	(rdev->pdev->device == 0x990B) \|\|
		450	(rdev->pdev->device == 0x990C) \|\|
		451	(rdev->pdev->device == 0x990D) \|\|
		452	(rdev->pdev->device == 0x990E) \|\|
		453	(rdev->pdev->device == 0x990F) \|\|
		454	(rdev->pdev->device == 0x9910) \|\|
		455	(rdev->pdev->device == 0x9913) \|\|
		456	(rdev->pdev->device == 0x9917) \|\|
		457	(rdev->pdev->device == 0x9918)) {
		458	radeon_program_register_sequence(rdev,
		459	dvst_golden_registers,
		460	(const u32)ARRAY_SIZE(dvst_golden_registers));
		461	radeon_program_register_sequence(rdev,
		462	dvst_golden_registers2,
		463	(const u32)ARRAY_SIZE(dvst_golden_registers2));
		464	} else {
		465	radeon_program_register_sequence(rdev,
		466	scrapper_golden_registers,
		467	(const u32)ARRAY_SIZE(scrapper_golden_registers));
		468	radeon_program_register_sequence(rdev,
		469	dvst_golden_registers2,
		470	(const u32)ARRAY_SIZE(dvst_golden_registers2));
		471	}
		472	break;
		473	default:
		474	break;
		475	}
		476	}
		477
1990	serge	478	#define BTC_IO_MC_REGS_SIZE 29
		479
		480	static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
		481	{0x00000077, 0xff010100},
		482	{0x00000078, 0x00000000},
		483	{0x00000079, 0x00001434},
		484	{0x0000007a, 0xcc08ec08},
		485	{0x0000007b, 0x00040000},
		486	{0x0000007c, 0x000080c0},
		487	{0x0000007d, 0x09000000},
		488	{0x0000007e, 0x00210404},
		489	{0x00000081, 0x08a8e800},
		490	{0x00000082, 0x00030444},
		491	{0x00000083, 0x00000000},
		492	{0x00000085, 0x00000001},
		493	{0x00000086, 0x00000002},
		494	{0x00000087, 0x48490000},
		495	{0x00000088, 0x20244647},
		496	{0x00000089, 0x00000005},
		497	{0x0000008b, 0x66030000},
		498	{0x0000008c, 0x00006603},
		499	{0x0000008d, 0x00000100},
		500	{0x0000008f, 0x00001c0a},
		501	{0x00000090, 0xff000001},
		502	{0x00000094, 0x00101101},
		503	{0x00000095, 0x00000fff},
		504	{0x00000096, 0x00116fff},
		505	{0x00000097, 0x60010000},
		506	{0x00000098, 0x10010000},
		507	{0x00000099, 0x00006000},
		508	{0x0000009a, 0x00001000},
		509	{0x0000009f, 0x00946a00}
		510	};
		511
		512	static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
		513	{0x00000077, 0xff010100},
		514	{0x00000078, 0x00000000},
		515	{0x00000079, 0x00001434},
		516	{0x0000007a, 0xcc08ec08},
		517	{0x0000007b, 0x00040000},
		518	{0x0000007c, 0x000080c0},
		519	{0x0000007d, 0x09000000},
		520	{0x0000007e, 0x00210404},
		521	{0x00000081, 0x08a8e800},
		522	{0x00000082, 0x00030444},
		523	{0x00000083, 0x00000000},
		524	{0x00000085, 0x00000001},
		525	{0x00000086, 0x00000002},
		526	{0x00000087, 0x48490000},
		527	{0x00000088, 0x20244647},
		528	{0x00000089, 0x00000005},
		529	{0x0000008b, 0x66030000},
		530	{0x0000008c, 0x00006603},
		531	{0x0000008d, 0x00000100},
		532	{0x0000008f, 0x00001c0a},
		533	{0x00000090, 0xff000001},
		534	{0x00000094, 0x00101101},
		535	{0x00000095, 0x00000fff},
		536	{0x00000096, 0x00116fff},
		537	{0x00000097, 0x60010000},
		538	{0x00000098, 0x10010000},
		539	{0x00000099, 0x00006000},
		540	{0x0000009a, 0x00001000},
		541	{0x0000009f, 0x00936a00}
		542	};
		543
		544	static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
		545	{0x00000077, 0xff010100},
		546	{0x00000078, 0x00000000},
		547	{0x00000079, 0x00001434},
		548	{0x0000007a, 0xcc08ec08},
		549	{0x0000007b, 0x00040000},
		550	{0x0000007c, 0x000080c0},
		551	{0x0000007d, 0x09000000},
		552	{0x0000007e, 0x00210404},
		553	{0x00000081, 0x08a8e800},
		554	{0x00000082, 0x00030444},
		555	{0x00000083, 0x00000000},
		556	{0x00000085, 0x00000001},
		557	{0x00000086, 0x00000002},
		558	{0x00000087, 0x48490000},
		559	{0x00000088, 0x20244647},
		560	{0x00000089, 0x00000005},
		561	{0x0000008b, 0x66030000},
		562	{0x0000008c, 0x00006603},
		563	{0x0000008d, 0x00000100},
		564	{0x0000008f, 0x00001c0a},
		565	{0x00000090, 0xff000001},
		566	{0x00000094, 0x00101101},
		567	{0x00000095, 0x00000fff},
		568	{0x00000096, 0x00116fff},
		569	{0x00000097, 0x60010000},
		570	{0x00000098, 0x10010000},
		571	{0x00000099, 0x00006000},
		572	{0x0000009a, 0x00001000},
		573	{0x0000009f, 0x00916a00}
		574	};
		575
		576	static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
		577	{0x00000077, 0xff010100},
		578	{0x00000078, 0x00000000},
		579	{0x00000079, 0x00001434},
		580	{0x0000007a, 0xcc08ec08},
		581	{0x0000007b, 0x00040000},
		582	{0x0000007c, 0x000080c0},
		583	{0x0000007d, 0x09000000},
		584	{0x0000007e, 0x00210404},
		585	{0x00000081, 0x08a8e800},
		586	{0x00000082, 0x00030444},
		587	{0x00000083, 0x00000000},
		588	{0x00000085, 0x00000001},
		589	{0x00000086, 0x00000002},
		590	{0x00000087, 0x48490000},
		591	{0x00000088, 0x20244647},
		592	{0x00000089, 0x00000005},
		593	{0x0000008b, 0x66030000},
		594	{0x0000008c, 0x00006603},
		595	{0x0000008d, 0x00000100},
		596	{0x0000008f, 0x00001c0a},
		597	{0x00000090, 0xff000001},
		598	{0x00000094, 0x00101101},
		599	{0x00000095, 0x00000fff},
		600	{0x00000096, 0x00116fff},
		601	{0x00000097, 0x60010000},
		602	{0x00000098, 0x10010000},
		603	{0x00000099, 0x00006000},
		604	{0x0000009a, 0x00001000},
		605	{0x0000009f, 0x00976b00}
		606	};
		607
		608	int ni_mc_load_microcode(struct radeon_device *rdev)
		609	{
		610	const __be32 *fw_data;
		611	u32 mem_type, running, blackout = 0;
		612	u32 *io_mc_regs;
		613	int i, ucode_size, regs_size;
		614
		615	if (!rdev->mc_fw)
		616	return -EINVAL;
		617
		618	switch (rdev->family) {
		619	case CHIP_BARTS:
		620	io_mc_regs = (u32 *)&barts_io_mc_regs;
		621	ucode_size = BTC_MC_UCODE_SIZE;
		622	regs_size = BTC_IO_MC_REGS_SIZE;
		623	break;
		624	case CHIP_TURKS:
		625	io_mc_regs = (u32 *)&turks_io_mc_regs;
		626	ucode_size = BTC_MC_UCODE_SIZE;
		627	regs_size = BTC_IO_MC_REGS_SIZE;
		628	break;
		629	case CHIP_CAICOS:
		630	default:
		631	io_mc_regs = (u32 *)&caicos_io_mc_regs;
		632	ucode_size = BTC_MC_UCODE_SIZE;
		633	regs_size = BTC_IO_MC_REGS_SIZE;
		634	break;
		635	case CHIP_CAYMAN:
		636	io_mc_regs = (u32 *)&cayman_io_mc_regs;
		637	ucode_size = CAYMAN_MC_UCODE_SIZE;
		638	regs_size = BTC_IO_MC_REGS_SIZE;
		639	break;
		640	}
		641
		642	mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
		643	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
		644
		645	if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
		646	if (running) {
		647	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
		648	WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
		649	}
		650
		651	/* reset the engine and set to writable */
		652	WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
		653	WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
		654
		655	/* load mc io regs */
		656	for (i = 0; i < regs_size; i++) {
		657	WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
		658	WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
		659	}
		660	/* load the MC ucode */
		661	fw_data = (const __be32 *)rdev->mc_fw->data;
		662	for (i = 0; i < ucode_size; i++)
		663	WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
		664
		665	/* put the engine back into the active state */
		666	WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
		667	WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
		668	WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
		669
		670	/* wait for training to complete */
2997	Serge	671	for (i = 0; i < rdev->usec_timeout; i++) {
		672	if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
		673	break;
		674	udelay(1);
		675	}
1990	serge	676
		677	if (running)
		678	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
		679	}
		680
		681	return 0;
		682	}
		683
		684	int ni_init_microcode(struct radeon_device *rdev)
		685	{
		686	const char *chip_name;
		687	const char *rlc_chip_name;
		688	size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
5078	serge	689	size_t smc_req_size = 0;
1990	serge	690	char fw_name[30];
		691	int err;
		692
		693	DRM_DEBUG("\n");
		694
		695	switch (rdev->family) {
		696	case CHIP_BARTS:
		697	chip_name = "BARTS";
		698	rlc_chip_name = "BTC";
		699	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		700	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		701	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		702	mc_req_size = BTC_MC_UCODE_SIZE * 4;
5078	serge	703	smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
1990	serge	704	break;
		705	case CHIP_TURKS:
		706	chip_name = "TURKS";
		707	rlc_chip_name = "BTC";
		708	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		709	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		710	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		711	mc_req_size = BTC_MC_UCODE_SIZE * 4;
5078	serge	712	smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
1990	serge	713	break;
		714	case CHIP_CAICOS:
		715	chip_name = "CAICOS";
		716	rlc_chip_name = "BTC";
		717	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		718	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		719	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		720	mc_req_size = BTC_MC_UCODE_SIZE * 4;
5078	serge	721	smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
1990	serge	722	break;
		723	case CHIP_CAYMAN:
		724	chip_name = "CAYMAN";
		725	rlc_chip_name = "CAYMAN";
		726	pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
		727	me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
		728	rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
		729	mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
5078	serge	730	smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
1990	serge	731	break;
2997	Serge	732	case CHIP_ARUBA:
		733	chip_name = "ARUBA";
		734	rlc_chip_name = "ARUBA";
		735	/* pfp/me same size as CAYMAN */
		736	pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
		737	me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
		738	rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
		739	mc_req_size = 0;
		740	break;
1990	serge	741	default: BUG();
		742	}
		743
		744	DRM_INFO("Loading %s Microcode\n", chip_name);
		745
		746	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
5078	serge	747	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1990	serge	748	if (err)
		749	goto out;
		750	if (rdev->pfp_fw->size != pfp_req_size) {
		751	printk(KERN_ERR
		752	"ni_cp: Bogus length %zu in firmware \"%s\"\n",
		753	rdev->pfp_fw->size, fw_name);
		754	err = -EINVAL;
		755	goto out;
		756	}
		757
		758	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
5078	serge	759	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1990	serge	760	if (err)
		761	goto out;
		762	if (rdev->me_fw->size != me_req_size) {
		763	printk(KERN_ERR
		764	"ni_cp: Bogus length %zu in firmware \"%s\"\n",
		765	rdev->me_fw->size, fw_name);
		766	err = -EINVAL;
		767	}
		768
		769	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
5078	serge	770	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1990	serge	771	if (err)
		772	goto out;
		773	if (rdev->rlc_fw->size != rlc_req_size) {
		774	printk(KERN_ERR
		775	"ni_rlc: Bogus length %zu in firmware \"%s\"\n",
		776	rdev->rlc_fw->size, fw_name);
		777	err = -EINVAL;
		778	}
		779
2997	Serge	780	/* no MC ucode on TN */
		781	if (!(rdev->flags & RADEON_IS_IGP)) {
1990	serge	782	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
5078	serge	783	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1990	serge	784	if (err)
		785	goto out;
		786	if (rdev->mc_fw->size != mc_req_size) {
		787	printk(KERN_ERR
		788	"ni_mc: Bogus length %zu in firmware \"%s\"\n",
		789	rdev->mc_fw->size, fw_name);
		790	err = -EINVAL;
		791	}
2997	Serge	792	}
5078	serge	793
		794	if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
		795	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
		796	err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
		797	if (err) {
		798	printk(KERN_ERR
		799	"smc: error loading firmware \"%s\"\n",
		800	fw_name);
		801	release_firmware(rdev->smc_fw);
		802	rdev->smc_fw = NULL;
		803	err = 0;
		804	} else if (rdev->smc_fw->size != smc_req_size) {
		805	printk(KERN_ERR
		806	"ni_mc: Bogus length %zu in firmware \"%s\"\n",
		807	rdev->mc_fw->size, fw_name);
		808	err = -EINVAL;
		809	}
		810	}
		811
1990	serge	812	out:
		813	if (err) {
		814	if (err != -EINVAL)
		815	printk(KERN_ERR
		816	"ni_cp: Failed to load firmware \"%s\"\n",
		817	fw_name);
		818	release_firmware(rdev->pfp_fw);
		819	rdev->pfp_fw = NULL;
		820	release_firmware(rdev->me_fw);
		821	rdev->me_fw = NULL;
		822	release_firmware(rdev->rlc_fw);
		823	rdev->rlc_fw = NULL;
		824	release_firmware(rdev->mc_fw);
		825	rdev->mc_fw = NULL;
		826	}
		827	return err;
		828	}
		829
5078	serge	830	int tn_get_temp(struct radeon_device *rdev)
		831	{
		832	u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
		833	int actual_temp = (temp / 8) - 49;
		834
		835	return actual_temp * 1000;
		836	}
		837
1990	serge	838	/*
		839	* Core functions
		840	*/
		841	static void cayman_gpu_init(struct radeon_device *rdev)
		842	{
		843	u32 gb_addr_config = 0;
		844	u32 mc_shared_chmap, mc_arb_ramcfg;
		845	u32 cgts_tcc_disable;
		846	u32 sx_debug_1;
		847	u32 smx_dc_ctl0;
		848	u32 cgts_sm_ctrl_reg;
		849	u32 hdp_host_path_cntl;
		850	u32 tmp;
2997	Serge	851	u32 disabled_rb_mask;
1990	serge	852	int i, j;
		853
		854	switch (rdev->family) {
		855	case CHIP_CAYMAN:
		856	rdev->config.cayman.max_shader_engines = 2;
		857	rdev->config.cayman.max_pipes_per_simd = 4;
		858	rdev->config.cayman.max_tile_pipes = 8;
		859	rdev->config.cayman.max_simds_per_se = 12;
		860	rdev->config.cayman.max_backends_per_se = 4;
		861	rdev->config.cayman.max_texture_channel_caches = 8;
		862	rdev->config.cayman.max_gprs = 256;
		863	rdev->config.cayman.max_threads = 256;
		864	rdev->config.cayman.max_gs_threads = 32;
		865	rdev->config.cayman.max_stack_entries = 512;
		866	rdev->config.cayman.sx_num_of_sets = 8;
		867	rdev->config.cayman.sx_max_export_size = 256;
		868	rdev->config.cayman.sx_max_export_pos_size = 64;
		869	rdev->config.cayman.sx_max_export_smx_size = 192;
		870	rdev->config.cayman.max_hw_contexts = 8;
		871	rdev->config.cayman.sq_num_cf_insts = 2;
		872
		873	rdev->config.cayman.sc_prim_fifo_size = 0x100;
		874	rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
		875	rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	876	gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
1990	serge	877	break;
2997	Serge	878	case CHIP_ARUBA:
		879	default:
		880	rdev->config.cayman.max_shader_engines = 1;
		881	rdev->config.cayman.max_pipes_per_simd = 4;
		882	rdev->config.cayman.max_tile_pipes = 2;
		883	if ((rdev->pdev->device == 0x9900) \|\|
		884	(rdev->pdev->device == 0x9901) \|\|
		885	(rdev->pdev->device == 0x9905) \|\|
		886	(rdev->pdev->device == 0x9906) \|\|
		887	(rdev->pdev->device == 0x9907) \|\|
		888	(rdev->pdev->device == 0x9908) \|\|
		889	(rdev->pdev->device == 0x9909) \|\|
3764	Serge	890	(rdev->pdev->device == 0x990B) \|\|
		891	(rdev->pdev->device == 0x990C) \|\|
		892	(rdev->pdev->device == 0x990F) \|\|
2997	Serge	893	(rdev->pdev->device == 0x9910) \|\|
3764	Serge	894	(rdev->pdev->device == 0x9917) \|\|
		895	(rdev->pdev->device == 0x9999) \|\|
		896	(rdev->pdev->device == 0x999C)) {
2997	Serge	897	rdev->config.cayman.max_simds_per_se = 6;
		898	rdev->config.cayman.max_backends_per_se = 2;
5078	serge	899	rdev->config.cayman.max_hw_contexts = 8;
		900	rdev->config.cayman.sx_max_export_size = 256;
		901	rdev->config.cayman.sx_max_export_pos_size = 64;
		902	rdev->config.cayman.sx_max_export_smx_size = 192;
2997	Serge	903	} else if ((rdev->pdev->device == 0x9903) \|\|
		904	(rdev->pdev->device == 0x9904) \|\|
		905	(rdev->pdev->device == 0x990A) \|\|
3764	Serge	906	(rdev->pdev->device == 0x990D) \|\|
		907	(rdev->pdev->device == 0x990E) \|\|
2997	Serge	908	(rdev->pdev->device == 0x9913) \|\|
3764	Serge	909	(rdev->pdev->device == 0x9918) \|\|
		910	(rdev->pdev->device == 0x999D)) {
2997	Serge	911	rdev->config.cayman.max_simds_per_se = 4;
		912	rdev->config.cayman.max_backends_per_se = 2;
5078	serge	913	rdev->config.cayman.max_hw_contexts = 8;
		914	rdev->config.cayman.sx_max_export_size = 256;
		915	rdev->config.cayman.sx_max_export_pos_size = 64;
		916	rdev->config.cayman.sx_max_export_smx_size = 192;
2997	Serge	917	} else if ((rdev->pdev->device == 0x9919) \|\|
		918	(rdev->pdev->device == 0x9990) \|\|
		919	(rdev->pdev->device == 0x9991) \|\|
		920	(rdev->pdev->device == 0x9994) \|\|
3764	Serge	921	(rdev->pdev->device == 0x9995) \|\|
		922	(rdev->pdev->device == 0x9996) \|\|
		923	(rdev->pdev->device == 0x999A) \|\|
2997	Serge	924	(rdev->pdev->device == 0x99A0)) {
		925	rdev->config.cayman.max_simds_per_se = 3;
		926	rdev->config.cayman.max_backends_per_se = 1;
5078	serge	927	rdev->config.cayman.max_hw_contexts = 4;
		928	rdev->config.cayman.sx_max_export_size = 128;
		929	rdev->config.cayman.sx_max_export_pos_size = 32;
		930	rdev->config.cayman.sx_max_export_smx_size = 96;
2997	Serge	931	} else {
		932	rdev->config.cayman.max_simds_per_se = 2;
		933	rdev->config.cayman.max_backends_per_se = 1;
5078	serge	934	rdev->config.cayman.max_hw_contexts = 4;
		935	rdev->config.cayman.sx_max_export_size = 128;
		936	rdev->config.cayman.sx_max_export_pos_size = 32;
		937	rdev->config.cayman.sx_max_export_smx_size = 96;
2997	Serge	938	}
		939	rdev->config.cayman.max_texture_channel_caches = 2;
		940	rdev->config.cayman.max_gprs = 256;
		941	rdev->config.cayman.max_threads = 256;
		942	rdev->config.cayman.max_gs_threads = 32;
		943	rdev->config.cayman.max_stack_entries = 512;
		944	rdev->config.cayman.sx_num_of_sets = 8;
		945	rdev->config.cayman.sq_num_cf_insts = 2;
		946
		947	rdev->config.cayman.sc_prim_fifo_size = 0x40;
		948	rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
		949	rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
		950	gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
		951	break;
1990	serge	952	}
		953
		954	/* Initialize HDP */
		955	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		956	WREG32((0x2c14 + j), 0x00000000);
		957	WREG32((0x2c18 + j), 0x00000000);
		958	WREG32((0x2c1c + j), 0x00000000);
		959	WREG32((0x2c20 + j), 0x00000000);
		960	WREG32((0x2c24 + j), 0x00000000);
		961	}
		962
		963	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		964
2997	Serge	965	evergreen_fix_pci_max_read_req_size(rdev);
		966
1990	serge	967	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
		968	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
		969
		970	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
		971	rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
		972	if (rdev->config.cayman.mem_row_size_in_kb > 4)
		973	rdev->config.cayman.mem_row_size_in_kb = 4;
		974	/* XXX use MC settings? */
		975	rdev->config.cayman.shader_engine_tile_size = 32;
		976	rdev->config.cayman.num_gpus = 1;
		977	rdev->config.cayman.multi_gpu_tile_size = 64;
		978
		979	tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
		980	rdev->config.cayman.num_tile_pipes = (1 << tmp);
		981	tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
		982	rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
		983	tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
		984	rdev->config.cayman.num_shader_engines = tmp + 1;
		985	tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
		986	rdev->config.cayman.num_gpus = tmp + 1;
		987	tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
		988	rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
		989	tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
		990	rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
		991
2997	Serge	992
1990	serge	993	/* setup tiling info dword. gb_addr_config is not adequate since it does
		994	* not have bank info, so create a custom tiling dword.
		995	* bits 3:0 num_pipes
		996	* bits 7:4 num_banks
		997	* bits 11:8 group_size
		998	* bits 15:12 row_size
		999	*/
		1000	rdev->config.cayman.tile_config = 0;
		1001	switch (rdev->config.cayman.num_tile_pipes) {
		1002	case 1:
		1003	default:
		1004	rdev->config.cayman.tile_config \|= (0 << 0);
		1005	break;
		1006	case 2:
		1007	rdev->config.cayman.tile_config \|= (1 << 0);
		1008	break;
		1009	case 4:
		1010	rdev->config.cayman.tile_config \|= (2 << 0);
		1011	break;
		1012	case 8:
		1013	rdev->config.cayman.tile_config \|= (3 << 0);
		1014	break;
		1015	}
2997	Serge	1016
		1017	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
		1018	if (rdev->flags & RADEON_IS_IGP)
		1019	rdev->config.cayman.tile_config \|= 1 << 4;
		1020	else {
		1021	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
		1022	case 0: /* four banks */
		1023	rdev->config.cayman.tile_config \|= 0 << 4;
		1024	break;
		1025	case 1: /* eight banks */
		1026	rdev->config.cayman.tile_config \|= 1 << 4;
		1027	break;
		1028	case 2: /* sixteen banks */
		1029	default:
		1030	rdev->config.cayman.tile_config \|= 2 << 4;
		1031	break;
		1032	}
		1033	}
1990	serge	1034	rdev->config.cayman.tile_config \|=
		1035	((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
		1036	rdev->config.cayman.tile_config \|=
		1037	((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
		1038
2997	Serge	1039	tmp = 0;
		1040	for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
		1041	u32 rb_disable_bitmap;
		1042
		1043	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		1044	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		1045	rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
		1046	tmp <<= 4;
		1047	tmp \|= rb_disable_bitmap;
		1048	}
		1049	/* enabled rb are just the one not disabled :) */
		1050	disabled_rb_mask = tmp;
3764	Serge	1051	tmp = 0;
		1052	for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
		1053	tmp \|= (1 << i);
		1054	/* if all the backends are disabled, fix it up here */
		1055	if ((disabled_rb_mask & tmp) == tmp) {
		1056	for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
		1057	disabled_rb_mask &= ~(1 << i);
		1058	}
2997	Serge	1059
5078	serge	1060	for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
		1061	u32 simd_disable_bitmap;
		1062
		1063	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		1064	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		1065	simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
		1066	simd_disable_bitmap \|= 0xffffffff << rdev->config.cayman.max_simds_per_se;
		1067	tmp <<= 16;
		1068	tmp \|= simd_disable_bitmap;
		1069	}
		1070	rdev->config.cayman.active_simds = hweight32(~tmp);
		1071
2997	Serge	1072	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
		1073	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
		1074
1990	serge	1075	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		1076	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3764	Serge	1077	if (ASIC_IS_DCE6(rdev))
		1078	WREG32(DMIF_ADDR_CALC, gb_addr_config);
1990	serge	1079	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3192	Serge	1080	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
		1081	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3764	Serge	1082	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
		1083	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
		1084	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1990	serge	1085
3764	Serge	1086	if ((rdev->config.cayman.max_backends_per_se == 1) &&
		1087	(rdev->flags & RADEON_IS_IGP)) {
		1088	if ((disabled_rb_mask & 3) == 1) {
		1089	/* RB0 disabled, RB1 enabled */
		1090	tmp = 0x11111111;
		1091	} else {
		1092	/* RB1 disabled, RB0 enabled */
		1093	tmp = 0x00000000;
		1094	}
		1095	} else {
2997	Serge	1096	tmp = gb_addr_config & NUM_PIPES_MASK;
		1097	tmp = r6xx_remap_render_backend(rdev, tmp,
		1098	rdev->config.cayman.max_backends_per_se *
		1099	rdev->config.cayman.max_shader_engines,
		1100	CAYMAN_MAX_BACKENDS, disabled_rb_mask);
3764	Serge	1101	}
2997	Serge	1102	WREG32(GB_BACKEND_MAP, tmp);
1990	serge	1103
2997	Serge	1104	cgts_tcc_disable = 0xffff0000;
		1105	for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
		1106	cgts_tcc_disable &= ~(1 << (16 + i));
1990	serge	1107	WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
		1108	WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
		1109	WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
		1110	WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
		1111
		1112	/* reprogram the shader complex */
		1113	cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
		1114	for (i = 0; i < 16; i++)
		1115	WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
		1116	WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
		1117
		1118	/* set HW defaults for 3D engine */
		1119	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) \| MEQ2_START(0x60));
		1120
		1121	sx_debug_1 = RREG32(SX_DEBUG_1);
		1122	sx_debug_1 \|= ENABLE_NEW_SMX_ADDRESS;
		1123	WREG32(SX_DEBUG_1, sx_debug_1);
		1124
		1125	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
		1126	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
		1127	smx_dc_ctl0 \|= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
		1128	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
		1129
		1130	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) \| CRC_SIMD_ID_WADDR_DISABLE);
		1131
		1132	/* need to be explicitly zero-ed */
		1133	WREG32(VGT_OFFCHIP_LDS_BASE, 0);
		1134	WREG32(SQ_LSTMP_RING_BASE, 0);
		1135	WREG32(SQ_HSTMP_RING_BASE, 0);
		1136	WREG32(SQ_ESTMP_RING_BASE, 0);
		1137	WREG32(SQ_GSTMP_RING_BASE, 0);
		1138	WREG32(SQ_VSTMP_RING_BASE, 0);
		1139	WREG32(SQ_PSTMP_RING_BASE, 0);
		1140
		1141	WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
		1142
		1143	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) \|
		1144	POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) \|
		1145	SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
		1146
		1147	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) \|
		1148	SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) \|
		1149	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
		1150
		1151
		1152	WREG32(VGT_NUM_INSTANCES, 1);
		1153
		1154	WREG32(CP_PERFMON_CNTL, 0);
		1155
		1156	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) \|
		1157	FETCH_FIFO_HIWATER(0x4) \|
		1158	DONE_FIFO_HIWATER(0xe0) \|
		1159	ALU_UPDATE_FIFO_HIWATER(0x8)));
		1160
		1161	WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
		1162	WREG32(SQ_CONFIG, (VC_ENABLE \|
		1163	EXPORT_SRC_C \|
		1164	GFX_PRIO(0) \|
		1165	CS1_PRIO(0) \|
		1166	CS2_PRIO(1)));
		1167	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
		1168
		1169	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		1170	FORCE_EOV_MAX_REZ_CNT(255)));
		1171
		1172	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) \|
		1173	AUTO_INVLD_EN(ES_AND_GS_AUTO));
		1174
		1175	WREG32(VGT_GS_VERTEX_REUSE, 16);
		1176	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		1177
		1178	WREG32(CB_PERF_CTR0_SEL_0, 0);
		1179	WREG32(CB_PERF_CTR0_SEL_1, 0);
		1180	WREG32(CB_PERF_CTR1_SEL_0, 0);
		1181	WREG32(CB_PERF_CTR1_SEL_1, 0);
		1182	WREG32(CB_PERF_CTR2_SEL_0, 0);
		1183	WREG32(CB_PERF_CTR2_SEL_1, 0);
		1184	WREG32(CB_PERF_CTR3_SEL_0, 0);
		1185	WREG32(CB_PERF_CTR3_SEL_1, 0);
		1186
		1187	tmp = RREG32(HDP_MISC_CNTL);
		1188	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		1189	WREG32(HDP_MISC_CNTL, tmp);
		1190
		1191	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		1192	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		1193
		1194	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		1195
		1196	udelay(50);
5078	serge	1197
		1198	/* set clockgating golden values on TN */
		1199	if (rdev->family == CHIP_ARUBA) {
		1200	tmp = RREG32_CG(CG_CGTT_LOCAL_0);
		1201	tmp &= ~0x00380000;
		1202	WREG32_CG(CG_CGTT_LOCAL_0, tmp);
		1203	tmp = RREG32_CG(CG_CGTT_LOCAL_1);
		1204	tmp &= ~0x0e000000;
		1205	WREG32_CG(CG_CGTT_LOCAL_1, tmp);
		1206	}
1990	serge	1207	}
		1208
		1209	/*
		1210	* GART
		1211	*/
		1212	void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
		1213	{
		1214	/* flush hdp cache */
		1215	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		1216
		1217	/* bits 0-7 are the VM contexts0-7 */
		1218	WREG32(VM_INVALIDATE_REQUEST, 1);
		1219	}
		1220
2997	Serge	1221	static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1990	serge	1222	{
2997	Serge	1223	int i, r;
1990	serge	1224
2997	Serge	1225	if (rdev->gart.robj == NULL) {
1990	serge	1226	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		1227	return -EINVAL;
		1228	}
		1229	r = radeon_gart_table_vram_pin(rdev);
		1230	if (r)
		1231	return r;
		1232	/* Setup TLB control */
2997	Serge	1233	WREG32(MC_VM_MX_L1_TLB_CNTL,
		1234	(0xA << 7) \|
		1235	ENABLE_L1_TLB \|
1990	serge	1236	ENABLE_L1_FRAGMENT_PROCESSING \|
		1237	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
2997	Serge	1238	ENABLE_ADVANCED_DRIVER_MODEL \|
1990	serge	1239	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
		1240	/* Setup L2 cache */
		1241	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \|
5078	serge	1242	ENABLE_L2_FRAGMENT_PROCESSING \|
1990	serge	1243	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1244	ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE \|
		1245	EFFECTIVE_L2_QUEUE_SIZE(7) \|
		1246	CONTEXT1_IDENTITY_ACCESS_MODE(1));
		1247	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS \| INVALIDATE_L2_CACHE);
		1248	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY \|
5078	serge	1249	BANK_SELECT(6) \|
1990	serge	1250	L2_CACHE_BIGK_FRAGMENT_SIZE(6));
		1251	/* setup context0 */
		1252	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		1253	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		1254	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		1255	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		1256	(u32)(rdev->dummy_page.addr >> 12));
		1257	WREG32(VM_CONTEXT0_CNTL2, 0);
		1258	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		1259	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2997	Serge	1260
		1261	WREG32(0x15D4, 0);
		1262	WREG32(0x15D8, 0);
		1263	WREG32(0x15DC, 0);
		1264
		1265	/* empty context1-7 */
		1266	/* Assign the pt base to something valid for now; the pts used for
		1267	* the VMs are determined by the application and setup and assigned
		1268	* on the fly in the vm part of radeon_gart.c
		1269	*/
		1270	for (i = 1; i < 8; i++) {
		1271	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
		1272	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
		1273	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5078	serge	1274	rdev->vm_manager.saved_table_addr[i]);
2997	Serge	1275	}
		1276
		1277	/* enable context1-7 */
		1278	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
		1279	(u32)(rdev->dummy_page.addr >> 12));
3192	Serge	1280	WREG32(VM_CONTEXT1_CNTL2, 4);
2997	Serge	1281	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(1) \|
5078	serge	1282	PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) \|
3192	Serge	1283	RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1284	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT \|
		1285	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1286	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT \|
		1287	PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1288	PDE0_PROTECTION_FAULT_ENABLE_DEFAULT \|
		1289	VALID_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1290	VALID_PROTECTION_FAULT_ENABLE_DEFAULT \|
		1291	READ_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1292	READ_PROTECTION_FAULT_ENABLE_DEFAULT \|
		1293	WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1294	WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1990	serge	1295
		1296	cayman_pcie_gart_tlb_flush(rdev);
2997	Serge	1297	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		1298	(unsigned)(rdev->mc.gtt_size >> 20),
		1299	(unsigned long long)rdev->gart.table_addr);
1990	serge	1300	rdev->gart.ready = true;
		1301	return 0;
		1302	}
		1303
2997	Serge	1304	static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1990	serge	1305	{
5078	serge	1306	unsigned i;
		1307
		1308	for (i = 1; i < 8; ++i) {
		1309	rdev->vm_manager.saved_table_addr[i] = RREG32(
		1310	VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
		1311	}
		1312
1990	serge	1313	/* Disable all tables */
		1314	WREG32(VM_CONTEXT0_CNTL, 0);
		1315	WREG32(VM_CONTEXT1_CNTL, 0);
		1316	/* Setup TLB control */
		1317	WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING \|
		1318	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		1319	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
		1320	/* Setup L2 cache */
		1321	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1322	ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE \|
		1323	EFFECTIVE_L2_QUEUE_SIZE(7) \|
		1324	CONTEXT1_IDENTITY_ACCESS_MODE(1));
		1325	WREG32(VM_L2_CNTL2, 0);
		1326	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY \|
		1327	L2_CACHE_BIGK_FRAGMENT_SIZE(6));
2997	Serge	1328	radeon_gart_table_vram_unpin(rdev);
1990	serge	1329	}
		1330
2997	Serge	1331	void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
		1332	int ring, u32 cp_int_cntl)
		1333	{
		1334	u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1990	serge	1335
2997	Serge	1336	WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl \| (ring & 3));
		1337	WREG32(CP_INT_CNTL, cp_int_cntl);
		1338	}
		1339
1990	serge	1340	/*
		1341	* CP.
		1342	*/
2997	Serge	1343	void cayman_fence_ring_emit(struct radeon_device *rdev,
		1344	struct radeon_fence *fence)
		1345	{
		1346	struct radeon_ring *ring = &rdev->ring[fence->ring];
		1347	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
5078	serge	1348	u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA \| PACKET3_TC_ACTION_ENA \|
		1349	PACKET3_SH_ACTION_ENA;
2997	Serge	1350
		1351	/* flush read cache over gart for this vmid */
		1352	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
5078	serge	1353	radeon_ring_write(ring, PACKET3_ENGINE_ME \| cp_coher_cntl);
2997	Serge	1354	radeon_ring_write(ring, 0xFFFFFFFF);
		1355	radeon_ring_write(ring, 0);
		1356	radeon_ring_write(ring, 10); /* poll interval */
		1357	/* EVENT_WRITE_EOP - flush caches, send int */
		1358	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		1359	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) \| EVENT_INDEX(5));
5078	serge	1360	radeon_ring_write(ring, lower_32_bits(addr));
2997	Serge	1361	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		1362	radeon_ring_write(ring, fence->seq);
		1363	radeon_ring_write(ring, 0);
		1364	}
		1365
		1366	void cayman_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		1367	{
		1368	struct radeon_ring *ring = &rdev->ring[ib->ring];
5271	serge	1369	unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
5078	serge	1370	u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA \| PACKET3_TC_ACTION_ENA \|
		1371	PACKET3_SH_ACTION_ENA;
2997	Serge	1372
		1373	/* set to DX10/11 mode */
		1374	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
		1375	radeon_ring_write(ring, 1);
		1376
		1377	if (ring->rptr_save_reg) {
		1378	uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
		1379	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3764	Serge	1380	radeon_ring_write(ring, ((ring->rptr_save_reg -
2997	Serge	1381	PACKET3_SET_CONFIG_REG_START) >> 2));
		1382	radeon_ring_write(ring, next_rptr);
		1383	}
		1384
		1385	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		1386	radeon_ring_write(ring,
		1387	#ifdef __BIG_ENDIAN
		1388	(2 << 0) \|
		1389	#endif
		1390	(ib->gpu_addr & 0xFFFFFFFC));
		1391	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
5271	serge	1392	radeon_ring_write(ring, ib->length_dw \| (vm_id << 24));
2997	Serge	1393
		1394	/* flush read cache over gart for this vmid */
		1395	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
5078	serge	1396	radeon_ring_write(ring, PACKET3_ENGINE_ME \| cp_coher_cntl);
2997	Serge	1397	radeon_ring_write(ring, 0xFFFFFFFF);
		1398	radeon_ring_write(ring, 0);
5271	serge	1399	radeon_ring_write(ring, (vm_id << 24) \| 10); /* poll interval */
2997	Serge	1400	}
		1401
1990	serge	1402	static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
		1403	{
		1404	if (enable)
		1405	WREG32(CP_ME_CNTL, 0);
		1406	else {
5078	serge	1407	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3192	Serge	1408	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1990	serge	1409	WREG32(CP_ME_CNTL, (CP_ME_HALT \| CP_PFP_HALT));
		1410	WREG32(SCRATCH_UMSK, 0);
3192	Serge	1411	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1990	serge	1412	}
		1413	}
		1414
5078	serge	1415	u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
		1416	struct radeon_ring *ring)
		1417	{
		1418	u32 rptr;
		1419
		1420	if (rdev->wb.enabled)
		1421	rptr = rdev->wb.wb[ring->rptr_offs/4];
		1422	else {
		1423	if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
		1424	rptr = RREG32(CP_RB0_RPTR);
		1425	else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
		1426	rptr = RREG32(CP_RB1_RPTR);
		1427	else
		1428	rptr = RREG32(CP_RB2_RPTR);
		1429	}
		1430
		1431	return rptr;
		1432	}
		1433
		1434	u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
		1435	struct radeon_ring *ring)
		1436	{
		1437	u32 wptr;
		1438
		1439	if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
		1440	wptr = RREG32(CP_RB0_WPTR);
		1441	else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
		1442	wptr = RREG32(CP_RB1_WPTR);
		1443	else
		1444	wptr = RREG32(CP_RB2_WPTR);
		1445
		1446	return wptr;
		1447	}
		1448
		1449	void cayman_gfx_set_wptr(struct radeon_device *rdev,
		1450	struct radeon_ring *ring)
		1451	{
		1452	if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
		1453	WREG32(CP_RB0_WPTR, ring->wptr);
		1454	(void)RREG32(CP_RB0_WPTR);
		1455	} else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
		1456	WREG32(CP_RB1_WPTR, ring->wptr);
		1457	(void)RREG32(CP_RB1_WPTR);
		1458	} else {
		1459	WREG32(CP_RB2_WPTR, ring->wptr);
		1460	(void)RREG32(CP_RB2_WPTR);
		1461	}
		1462	}
		1463
1990	serge	1464	static int cayman_cp_load_microcode(struct radeon_device *rdev)
		1465	{
		1466	const __be32 *fw_data;
		1467	int i;
		1468
		1469	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1470	return -EINVAL;
		1471
		1472	cayman_cp_enable(rdev, false);
		1473
		1474	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1475	WREG32(CP_PFP_UCODE_ADDR, 0);
		1476	for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
		1477	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		1478	WREG32(CP_PFP_UCODE_ADDR, 0);
		1479
		1480	fw_data = (const __be32 *)rdev->me_fw->data;
		1481	WREG32(CP_ME_RAM_WADDR, 0);
		1482	for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
		1483	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		1484
		1485	WREG32(CP_PFP_UCODE_ADDR, 0);
		1486	WREG32(CP_ME_RAM_WADDR, 0);
		1487	WREG32(CP_ME_RAM_RADDR, 0);
		1488	return 0;
		1489	}
		1490
		1491	static int cayman_cp_start(struct radeon_device *rdev)
		1492	{
2997	Serge	1493	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1990	serge	1494	int r, i;
		1495
2997	Serge	1496	r = radeon_ring_lock(rdev, ring, 7);
1990	serge	1497	if (r) {
		1498	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1499	return r;
		1500	}
2997	Serge	1501	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1502	radeon_ring_write(ring, 0x1);
		1503	radeon_ring_write(ring, 0x0);
		1504	radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
		1505	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1506	radeon_ring_write(ring, 0);
		1507	radeon_ring_write(ring, 0);
5078	serge	1508	radeon_ring_unlock_commit(rdev, ring, false);
1990	serge	1509
		1510	cayman_cp_enable(rdev, true);
		1511
2997	Serge	1512	r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1990	serge	1513	if (r) {
		1514	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1515	return r;
		1516	}
		1517
		1518	/* setup clear context state */
2997	Serge	1519	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1520	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1990	serge	1521
		1522	for (i = 0; i < cayman_default_size; i++)
2997	Serge	1523	radeon_ring_write(ring, cayman_default_state[i]);
1990	serge	1524
2997	Serge	1525	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1526	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1990	serge	1527
		1528	/* set clear context state */
2997	Serge	1529	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
		1530	radeon_ring_write(ring, 0);
1990	serge	1531
		1532	/* SQ_VTX_BASE_VTX_LOC */
2997	Serge	1533	radeon_ring_write(ring, 0xc0026f00);
		1534	radeon_ring_write(ring, 0x00000000);
		1535	radeon_ring_write(ring, 0x00000000);
		1536	radeon_ring_write(ring, 0x00000000);
1990	serge	1537
		1538	/* Clear consts */
2997	Serge	1539	radeon_ring_write(ring, 0xc0036f00);
		1540	radeon_ring_write(ring, 0x00000bc4);
		1541	radeon_ring_write(ring, 0xffffffff);
		1542	radeon_ring_write(ring, 0xffffffff);
		1543	radeon_ring_write(ring, 0xffffffff);
1990	serge	1544
2997	Serge	1545	radeon_ring_write(ring, 0xc0026900);
		1546	radeon_ring_write(ring, 0x00000316);
		1547	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		1548	radeon_ring_write(ring, 0x00000010); /* */
1990	serge	1549
5078	serge	1550	radeon_ring_unlock_commit(rdev, ring, false);
1990	serge	1551
		1552	/* XXX init other rings */
		1553
		1554	return 0;
		1555	}
		1556
		1557
2997	Serge	1558	static int cayman_cp_resume(struct radeon_device *rdev)
1990	serge	1559	{
2997	Serge	1560	static const int ridx[] = {
		1561	RADEON_RING_TYPE_GFX_INDEX,
		1562	CAYMAN_RING_TYPE_CP1_INDEX,
		1563	CAYMAN_RING_TYPE_CP2_INDEX
		1564	};
		1565	static const unsigned cp_rb_cntl[] = {
		1566	CP_RB0_CNTL,
		1567	CP_RB1_CNTL,
		1568	CP_RB2_CNTL,
		1569	};
		1570	static const unsigned cp_rb_rptr_addr[] = {
		1571	CP_RB0_RPTR_ADDR,
		1572	CP_RB1_RPTR_ADDR,
		1573	CP_RB2_RPTR_ADDR
		1574	};
		1575	static const unsigned cp_rb_rptr_addr_hi[] = {
		1576	CP_RB0_RPTR_ADDR_HI,
		1577	CP_RB1_RPTR_ADDR_HI,
		1578	CP_RB2_RPTR_ADDR_HI
		1579	};
		1580	static const unsigned cp_rb_base[] = {
		1581	CP_RB0_BASE,
		1582	CP_RB1_BASE,
		1583	CP_RB2_BASE
		1584	};
5078	serge	1585	static const unsigned cp_rb_rptr[] = {
		1586	CP_RB0_RPTR,
		1587	CP_RB1_RPTR,
		1588	CP_RB2_RPTR
		1589	};
		1590	static const unsigned cp_rb_wptr[] = {
		1591	CP_RB0_WPTR,
		1592	CP_RB1_WPTR,
		1593	CP_RB2_WPTR
		1594	};
2997	Serge	1595	struct radeon_ring *ring;
		1596	int i, r;
1990	serge	1597
		1598	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
		1599	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP \|
		1600	SOFT_RESET_PA \|
		1601	SOFT_RESET_SH \|
		1602	SOFT_RESET_VGT \|
2160	serge	1603	SOFT_RESET_SPI \|
1990	serge	1604	SOFT_RESET_SX));
		1605	RREG32(GRBM_SOFT_RESET);
		1606	mdelay(15);
		1607	WREG32(GRBM_SOFT_RESET, 0);
		1608	RREG32(GRBM_SOFT_RESET);
		1609
2997	Serge	1610	WREG32(CP_SEM_WAIT_TIMER, 0x0);
		1611	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1990	serge	1612
		1613	/* Set the write pointer delay */
		1614	WREG32(CP_RB_WPTR_DELAY, 0);
		1615
		1616	WREG32(CP_DEBUG, (1 << 27));
		1617
3120	serge	1618	/* set the wb address whether it's enabled or not */
1990	serge	1619	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1620	WREG32(SCRATCH_UMSK, 0xff);
		1621
2997	Serge	1622	for (i = 0; i < 3; ++i) {
		1623	uint32_t rb_cntl;
		1624	uint64_t addr;
1990	serge	1625
		1626	/* Set ring buffer size */
2997	Serge	1627	ring = &rdev->ring[ridx[i]];
5078	serge	1628	rb_cntl = order_base_2(ring->ring_size / 8);
		1629	rb_cntl \|= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1990	serge	1630	#ifdef __BIG_ENDIAN
2997	Serge	1631	rb_cntl \|= BUF_SWAP_32BIT;
1990	serge	1632	#endif
2997	Serge	1633	WREG32(cp_rb_cntl[i], rb_cntl);
1990	serge	1634
3120	serge	1635	/* set the wb address whether it's enabled or not */
2997	Serge	1636	addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
		1637	WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
		1638	WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
		1639	}
1990	serge	1640
2997	Serge	1641	/* set the rb base addr, this causes an internal reset of ALL rings */
		1642	for (i = 0; i < 3; ++i) {
		1643	ring = &rdev->ring[ridx[i]];
		1644	WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
		1645	}
1990	serge	1646
2997	Serge	1647	for (i = 0; i < 3; ++i) {
1990	serge	1648	/* Initialize the ring buffer's read and write pointers */
2997	Serge	1649	ring = &rdev->ring[ridx[i]];
		1650	WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1990	serge	1651
5078	serge	1652	ring->wptr = 0;
		1653	WREG32(cp_rb_rptr[i], 0);
		1654	WREG32(cp_rb_wptr[i], ring->wptr);
1990	serge	1655
		1656	mdelay(1);
2997	Serge	1657	WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
		1658	}
1990	serge	1659
		1660	/* start the rings */
		1661	cayman_cp_start(rdev);
2997	Serge	1662	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
		1663	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		1664	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1990	serge	1665	/* this only test cp0 */
2997	Serge	1666	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1990	serge	1667	if (r) {
2997	Serge	1668	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
		1669	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		1670	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1990	serge	1671	return r;
		1672	}
		1673
5078	serge	1674	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
5271	serge	1675	radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3192	Serge	1676
		1677	return 0;
		1678	}
		1679
5078	serge	1680	u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
3192	Serge	1681	{
3764	Serge	1682	u32 reset_mask = 0;
		1683	u32 tmp;
1990	serge	1684
3764	Serge	1685	/* GRBM_STATUS */
		1686	tmp = RREG32(GRBM_STATUS);
		1687	if (tmp & (PA_BUSY \| SC_BUSY \|
		1688	SH_BUSY \| SX_BUSY \|
		1689	TA_BUSY \| VGT_BUSY \|
		1690	DB_BUSY \| CB_BUSY \|
		1691	GDS_BUSY \| SPI_BUSY \|
		1692	IA_BUSY \| IA_BUSY_NO_DMA))
		1693	reset_mask \|= RADEON_RESET_GFX;
1990	serge	1694
3764	Serge	1695	if (tmp & (CF_RQ_PENDING \| PF_RQ_PENDING \|
		1696	CP_BUSY \| CP_COHERENCY_BUSY))
		1697	reset_mask \|= RADEON_RESET_CP;
2997	Serge	1698
3764	Serge	1699	if (tmp & GRBM_EE_BUSY)
		1700	reset_mask \|= RADEON_RESET_GRBM \| RADEON_RESET_GFX \| RADEON_RESET_CP;
1990	serge	1701
3764	Serge	1702	/* DMA_STATUS_REG 0 */
		1703	tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
		1704	if (!(tmp & DMA_IDLE))
		1705	reset_mask \|= RADEON_RESET_DMA;
1990	serge	1706
3764	Serge	1707	/* DMA_STATUS_REG 1 */
		1708	tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
		1709	if (!(tmp & DMA_IDLE))
		1710	reset_mask \|= RADEON_RESET_DMA1;
2997	Serge	1711
3764	Serge	1712	/* SRBM_STATUS2 */
		1713	tmp = RREG32(SRBM_STATUS2);
		1714	if (tmp & DMA_BUSY)
		1715	reset_mask \|= RADEON_RESET_DMA;
3192	Serge	1716
3764	Serge	1717	if (tmp & DMA1_BUSY)
		1718	reset_mask \|= RADEON_RESET_DMA1;
3192	Serge	1719
3764	Serge	1720	/* SRBM_STATUS */
		1721	tmp = RREG32(SRBM_STATUS);
		1722	if (tmp & (RLC_RQ_PENDING \| RLC_BUSY))
		1723	reset_mask \|= RADEON_RESET_RLC;
3192	Serge	1724
3764	Serge	1725	if (tmp & IH_BUSY)
		1726	reset_mask \|= RADEON_RESET_IH;
3192	Serge	1727
3764	Serge	1728	if (tmp & SEM_BUSY)
		1729	reset_mask \|= RADEON_RESET_SEM;
3192	Serge	1730
3764	Serge	1731	if (tmp & GRBM_RQ_PENDING)
		1732	reset_mask \|= RADEON_RESET_GRBM;
3192	Serge	1733
3764	Serge	1734	if (tmp & VMC_BUSY)
		1735	reset_mask \|= RADEON_RESET_VMC;
3192	Serge	1736
3764	Serge	1737	if (tmp & (MCB_BUSY \| MCB_NON_DISPLAY_BUSY \|
		1738	MCC_BUSY \| MCD_BUSY))
		1739	reset_mask \|= RADEON_RESET_MC;
3192	Serge	1740
3764	Serge	1741	if (evergreen_is_display_hung(rdev))
		1742	reset_mask \|= RADEON_RESET_DISPLAY;
3192	Serge	1743
3764	Serge	1744	/* VM_L2_STATUS */
		1745	tmp = RREG32(VM_L2_STATUS);
		1746	if (tmp & L2_BUSY)
		1747	reset_mask \|= RADEON_RESET_VMC;
		1748
		1749	/* Skip MC reset as it's mostly likely not hung, just busy */
		1750	if (reset_mask & RADEON_RESET_MC) {
		1751	DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
		1752	reset_mask &= ~RADEON_RESET_MC;
		1753	}
		1754
		1755	return reset_mask;
3192	Serge	1756	}
		1757
3764	Serge	1758	static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3192	Serge	1759	{
		1760	struct evergreen_mc_save save;
3764	Serge	1761	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
		1762	u32 tmp;
3192	Serge	1763
		1764	if (reset_mask == 0)
3764	Serge	1765	return;
3192	Serge	1766
		1767	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
		1768
3764	Serge	1769	evergreen_print_gpu_status_regs(rdev);
3192	Serge	1770	dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_ADDR 0x%08X\n",
		1771	RREG32(0x14F8));
		1772	dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
		1773	RREG32(0x14D8));
		1774	dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
		1775	RREG32(0x14FC));
		1776	dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
		1777	RREG32(0x14DC));
		1778
3764	Serge	1779	/* Disable CP parsing/prefetching */
		1780	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT);
		1781
		1782	if (reset_mask & RADEON_RESET_DMA) {
		1783	/* dma0 */
		1784	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
		1785	tmp &= ~DMA_RB_ENABLE;
		1786	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
		1787	}
		1788
		1789	if (reset_mask & RADEON_RESET_DMA1) {
		1790	/* dma1 */
		1791	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
		1792	tmp &= ~DMA_RB_ENABLE;
		1793	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
		1794	}
		1795
		1796	udelay(50);
		1797
3192	Serge	1798	evergreen_mc_stop(rdev, &save);
		1799	if (evergreen_mc_wait_for_idle(rdev)) {
		1800	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1801	}
		1802
3764	Serge	1803	if (reset_mask & (RADEON_RESET_GFX \| RADEON_RESET_COMPUTE)) {
		1804	grbm_soft_reset = SOFT_RESET_CB \|
		1805	SOFT_RESET_DB \|
		1806	SOFT_RESET_GDS \|
		1807	SOFT_RESET_PA \|
		1808	SOFT_RESET_SC \|
		1809	SOFT_RESET_SPI \|
		1810	SOFT_RESET_SH \|
		1811	SOFT_RESET_SX \|
		1812	SOFT_RESET_TC \|
		1813	SOFT_RESET_TA \|
		1814	SOFT_RESET_VGT \|
		1815	SOFT_RESET_IA;
		1816	}
3192	Serge	1817
3764	Serge	1818	if (reset_mask & RADEON_RESET_CP) {
		1819	grbm_soft_reset \|= SOFT_RESET_CP \| SOFT_RESET_VGT;
		1820
		1821	srbm_soft_reset \|= SOFT_RESET_GRBM;
		1822	}
		1823
3192	Serge	1824	if (reset_mask & RADEON_RESET_DMA)
3764	Serge	1825	srbm_soft_reset \|= SOFT_RESET_DMA;
3192	Serge	1826
3764	Serge	1827	if (reset_mask & RADEON_RESET_DMA1)
		1828	srbm_soft_reset \|= SOFT_RESET_DMA1;
		1829
		1830	if (reset_mask & RADEON_RESET_DISPLAY)
		1831	srbm_soft_reset \|= SOFT_RESET_DC;
		1832
		1833	if (reset_mask & RADEON_RESET_RLC)
		1834	srbm_soft_reset \|= SOFT_RESET_RLC;
		1835
		1836	if (reset_mask & RADEON_RESET_SEM)
		1837	srbm_soft_reset \|= SOFT_RESET_SEM;
		1838
		1839	if (reset_mask & RADEON_RESET_IH)
		1840	srbm_soft_reset \|= SOFT_RESET_IH;
		1841
		1842	if (reset_mask & RADEON_RESET_GRBM)
		1843	srbm_soft_reset \|= SOFT_RESET_GRBM;
		1844
		1845	if (reset_mask & RADEON_RESET_VMC)
		1846	srbm_soft_reset \|= SOFT_RESET_VMC;
		1847
		1848	if (!(rdev->flags & RADEON_IS_IGP)) {
		1849	if (reset_mask & RADEON_RESET_MC)
		1850	srbm_soft_reset \|= SOFT_RESET_MC;
		1851	}
		1852
		1853	if (grbm_soft_reset) {
		1854	tmp = RREG32(GRBM_SOFT_RESET);
		1855	tmp \|= grbm_soft_reset;
		1856	dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
		1857	WREG32(GRBM_SOFT_RESET, tmp);
		1858	tmp = RREG32(GRBM_SOFT_RESET);
		1859
		1860	udelay(50);
		1861
		1862	tmp &= ~grbm_soft_reset;
		1863	WREG32(GRBM_SOFT_RESET, tmp);
		1864	tmp = RREG32(GRBM_SOFT_RESET);
		1865	}
		1866
		1867	if (srbm_soft_reset) {
		1868	tmp = RREG32(SRBM_SOFT_RESET);
		1869	tmp \|= srbm_soft_reset;
		1870	dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
		1871	WREG32(SRBM_SOFT_RESET, tmp);
		1872	tmp = RREG32(SRBM_SOFT_RESET);
		1873
		1874	udelay(50);
		1875
		1876	tmp &= ~srbm_soft_reset;
		1877	WREG32(SRBM_SOFT_RESET, tmp);
		1878	tmp = RREG32(SRBM_SOFT_RESET);
		1879	}
		1880
3192	Serge	1881	/* Wait a little for things to settle down */
		1882	udelay(50);
		1883
1990	serge	1884	evergreen_mc_resume(rdev, &save);
3764	Serge	1885	udelay(50);
		1886
		1887	evergreen_print_gpu_status_regs(rdev);
1990	serge	1888	}
		1889
		1890	int cayman_asic_reset(struct radeon_device *rdev)
		1891	{
3764	Serge	1892	u32 reset_mask;
		1893
		1894	reset_mask = cayman_gpu_check_soft_reset(rdev);
		1895
		1896	if (reset_mask)
		1897	r600_set_bios_scratch_engine_hung(rdev, true);
		1898
		1899	cayman_gpu_soft_reset(rdev, reset_mask);
		1900
		1901	reset_mask = cayman_gpu_check_soft_reset(rdev);
		1902
5078	serge	1903	if (reset_mask)
		1904	evergreen_gpu_pci_config_reset(rdev);
		1905
5271	serge	1906	r600_set_bios_scratch_engine_hung(rdev, false);
3764	Serge	1907
		1908	return 0;
		1909	}
		1910
		1911	/**
		1912	* cayman_gfx_is_lockup - Check if the GFX engine is locked up
		1913	*
		1914	* @rdev: radeon_device pointer
		1915	* @ring: radeon_ring structure holding ring information
		1916	*
		1917	* Check if the GFX engine is locked up.
		1918	* Returns true if the engine appears to be locked up, false if not.
		1919	*/
		1920	bool cayman_gfx_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
		1921	{
		1922	u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
		1923
		1924	if (!(reset_mask & (RADEON_RESET_GFX \|
3192	Serge	1925	RADEON_RESET_COMPUTE \|
3764	Serge	1926	RADEON_RESET_CP))) {
5078	serge	1927	radeon_ring_lockup_update(rdev, ring);
3764	Serge	1928	return false;
		1929	}
		1930	return radeon_ring_test_lockup(rdev, ring);
1990	serge	1931	}
		1932
		1933	static int cayman_startup(struct radeon_device *rdev)
		1934	{
2997	Serge	1935	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1990	serge	1936	int r;
		1937
2997	Serge	1938	/* enable pcie gen2 link */
		1939	evergreen_pcie_gen2_enable(rdev);
5078	serge	1940	/* enable aspm */
		1941	evergreen_program_aspm(rdev);
2997	Serge	1942
5078	serge	1943	/* scratch needs to be initialized before MC */
		1944	r = r600_vram_scratch_init(rdev);
		1945	if (r)
5271	serge	1946	return r;
2997	Serge	1947
5078	serge	1948	evergreen_mc_program(rdev);
		1949
		1950	if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
5271	serge	1951	r = ni_mc_load_microcode(rdev);
		1952	if (r) {
		1953	DRM_ERROR("Failed to load MC firmware!\n");
		1954	return r;
		1955	}
1990	serge	1956	}
		1957
		1958	r = cayman_pcie_gart_enable(rdev);
		1959	if (r)
		1960	return r;
		1961	cayman_gpu_init(rdev);
		1962
2997	Serge	1963	/* allocate rlc buffers */
		1964	if (rdev->flags & RADEON_IS_IGP) {
5078	serge	1965	rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
		1966	rdev->rlc.reg_list_size =
		1967	(u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
		1968	rdev->rlc.cs_data = cayman_cs_data;
		1969	r = sumo_rlc_init(rdev);
2997	Serge	1970	if (r) {
		1971	DRM_ERROR("Failed to init rlc BOs!\n");
		1972	return r;
		1973	}
		1974	}
		1975
1990	serge	1976	/* allocate wb buffer */
2005	serge	1977	r = radeon_wb_init(rdev);
		1978	if (r)
		1979	return r;
1990	serge	1980
3192	Serge	1981	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
		1982	if (r) {
		1983	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		1984	return r;
		1985	}
		1986
3764	Serge	1987	// r = rv770_uvd_resume(rdev);
		1988	// if (!r) {
		1989	// r = radeon_fence_driver_start_ring(rdev,
		1990	// R600_RING_TYPE_UVD_INDEX);
		1991	// if (r)
		1992	// dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
		1993	// }
		1994	// if (r)
		1995	// rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
		1996
3192	Serge	1997	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		1998	if (r) {
		1999	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		2000	return r;
		2001	}
		2002
		2003	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		2004	if (r) {
		2005	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		2006	return r;
		2007	}
		2008
		2009	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
		2010	if (r) {
		2011	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		2012	return r;
		2013	}
		2014
		2015	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
		2016	if (r) {
		2017	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		2018	return r;
		2019	}
		2020
1990	serge	2021	/* Enable IRQ */
3764	Serge	2022	if (!rdev->irq.installed) {
		2023	r = radeon_irq_kms_init(rdev);
		2024	if (r)
		2025	return r;
		2026	}
		2027
2005	serge	2028	r = r600_irq_init(rdev);
		2029	if (r) {
		2030	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		2031	// radeon_irq_kms_fini(rdev);
		2032	return r;
		2033	}
		2034	evergreen_irq_set(rdev);
1990	serge	2035
2997	Serge	2036	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5078	serge	2037	RADEON_CP_PACKET2);
1990	serge	2038	if (r)
		2039	return r;
3192	Serge	2040
		2041	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		2042	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5078	serge	2043	DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3192	Serge	2044	if (r)
		2045	return r;
		2046
		2047	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		2048	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5078	serge	2049	DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3192	Serge	2050	if (r)
		2051	return r;
		2052
1990	serge	2053	r = cayman_cp_load_microcode(rdev);
		2054	if (r)
		2055	return r;
		2056	r = cayman_cp_resume(rdev);
		2057	if (r)
		2058	return r;
		2059
3192	Serge	2060	r = cayman_dma_resume(rdev);
		2061	if (r)
		2062	return r;
		2063
3764	Serge	2064
3192	Serge	2065	r = radeon_ib_pool_init(rdev);
		2066	if (r) {
		2067	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		2068	return r;
		2069	}
3764	Serge	2070
		2071	r = radeon_vm_manager_init(rdev);
		2072	if (r) {
		2073	dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
		2074	return r;
		2075	}
		2076
		2077
1990	serge	2078	return 0;
		2079	}
		2080
		2081
		2082
		2083
		2084
		2085	/* Plan is to move initialization in that function and use
		2086	* helper function so that radeon_device_init pretty much
		2087	* do nothing more than calling asic specific function. This
		2088	* should also allow to remove a bunch of callback function
		2089	* like vram_info.
		2090	*/
		2091	int cayman_init(struct radeon_device *rdev)
		2092	{
2997	Serge	2093	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1990	serge	2094	int r;
		2095
		2096	/* Read BIOS */
		2097	if (!radeon_get_bios(rdev)) {
		2098	if (ASIC_IS_AVIVO(rdev))
		2099	return -EINVAL;
		2100	}
		2101	/* Must be an ATOMBIOS */
		2102	if (!rdev->is_atom_bios) {
		2103	dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
		2104	return -EINVAL;
		2105	}
		2106	r = radeon_atombios_init(rdev);
		2107	if (r)
		2108	return r;
		2109
		2110	/* Post card if necessary */
		2111	if (!radeon_card_posted(rdev)) {
		2112	if (!rdev->bios) {
		2113	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		2114	return -EINVAL;
		2115	}
		2116	DRM_INFO("GPU not posted. posting now...\n");
		2117	atom_asic_init(rdev->mode_info.atom_context);
		2118	}
3764	Serge	2119	/* init golden registers */
		2120	ni_init_golden_registers(rdev);
1990	serge	2121	/* Initialize scratch registers */
		2122	r600_scratch_init(rdev);
		2123	/* Initialize surface registers */
		2124	radeon_surface_init(rdev);
		2125	/* Initialize clocks */
		2126	radeon_get_clock_info(rdev->ddev);
		2127	/* Fence driver */
2005	serge	2128	r = radeon_fence_driver_init(rdev);
		2129	if (r)
		2130	return r;
1990	serge	2131	/* initialize memory controller */
		2132	r = evergreen_mc_init(rdev);
		2133	if (r)
		2134	return r;
		2135	/* Memory manager */
		2136	r = radeon_bo_init(rdev);
		2137	if (r)
		2138	return r;
		2139
5078	serge	2140	if (rdev->flags & RADEON_IS_IGP) {
		2141	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		2142	r = ni_init_microcode(rdev);
		2143	if (r) {
		2144	DRM_ERROR("Failed to load firmware!\n");
		2145	return r;
		2146	}
		2147	}
		2148	} else {
		2149	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw \|\| !rdev->mc_fw) {
		2150	r = ni_init_microcode(rdev);
		2151	if (r) {
		2152	DRM_ERROR("Failed to load firmware!\n");
		2153	return r;
		2154	}
		2155	}
		2156	}
		2157
		2158	/* Initialize power management */
		2159	radeon_pm_init(rdev);
		2160
2997	Serge	2161	ring->ring_obj = NULL;
		2162	r600_ring_init(rdev, ring, 1024 * 1024);
1990	serge	2163
3192	Serge	2164	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		2165	ring->ring_obj = NULL;
		2166	r600_ring_init(rdev, ring, 64 * 1024);
		2167
		2168	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		2169	ring->ring_obj = NULL;
		2170	r600_ring_init(rdev, ring, 64 * 1024);
		2171
3764	Serge	2172	// r = radeon_uvd_init(rdev);
		2173	// if (!r) {
		2174	// ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
		2175	// ring->ring_obj = NULL;
		2176	// r600_ring_init(rdev, ring, 4096);
		2177	// }
		2178
2005	serge	2179	rdev->ih.ring_obj = NULL;
		2180	r600_ih_ring_init(rdev, 64 * 1024);
1990	serge	2181
		2182	r = r600_pcie_gart_init(rdev);
		2183	if (r)
		2184	return r;
		2185
		2186	rdev->accel_working = true;
		2187	r = cayman_startup(rdev);
		2188	if (r) {
		2189	dev_err(rdev->dev, "disabling GPU acceleration\n");
		2190	rdev->accel_working = false;
		2191	}
		2192
		2193	/* Don't start up if the MC ucode is missing.
		2194	* The default clocks and voltages before the MC ucode
		2195	* is loaded are not suffient for advanced operations.
2997	Serge	2196	*
		2197	* We can skip this check for TN, because there is no MC
		2198	* ucode.
1990	serge	2199	*/
2997	Serge	2200	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
1990	serge	2201	DRM_ERROR("radeon: MC ucode required for NI+.\n");
		2202	return -EINVAL;
		2203	}
		2204
		2205	return 0;
		2206	}
		2207
2997	Serge	2208	/*
		2209	* vm
		2210	*/
		2211	int cayman_vm_init(struct radeon_device *rdev)
		2212	{
		2213	/* number of VMs */
		2214	rdev->vm_manager.nvm = 8;
		2215	/* base offset of vram pages */
		2216	if (rdev->flags & RADEON_IS_IGP) {
		2217	u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
		2218	tmp <<= 22;
		2219	rdev->vm_manager.vram_base_offset = tmp;
		2220	} else
		2221	rdev->vm_manager.vram_base_offset = 0;
		2222	return 0;
		2223	}
		2224
		2225	void cayman_vm_fini(struct radeon_device *rdev)
		2226	{
		2227	}
		2228
		2229	/**
5078	serge	2230	* cayman_vm_decode_fault - print human readable fault info
2997	Serge	2231	*
		2232	* @rdev: radeon_device pointer
5078	serge	2233	* @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
		2234	* @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2997	Serge	2235	*
5078	serge	2236	* Print human readable fault information (cayman/TN).
2997	Serge	2237	*/
5078	serge	2238	void cayman_vm_decode_fault(struct radeon_device *rdev,
		2239	u32 status, u32 addr)
2997	Serge	2240	{
5078	serge	2241	u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
		2242	u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
		2243	u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
		2244	char *block;
2997	Serge	2245
5078	serge	2246	switch (mc_id) {
		2247	case 32:
		2248	case 16:
		2249	case 96:
		2250	case 80:
		2251	case 160:
		2252	case 144:
		2253	case 224:
		2254	case 208:
		2255	block = "CB";
		2256	break;
		2257	case 33:
		2258	case 17:
		2259	case 97:
		2260	case 81:
		2261	case 161:
		2262	case 145:
		2263	case 225:
		2264	case 209:
		2265	block = "CB_FMASK";
		2266	break;
		2267	case 34:
		2268	case 18:
		2269	case 98:
		2270	case 82:
		2271	case 162:
		2272	case 146:
		2273	case 226:
		2274	case 210:
		2275	block = "CB_CMASK";
		2276	break;
		2277	case 35:
		2278	case 19:
		2279	case 99:
		2280	case 83:
		2281	case 163:
		2282	case 147:
		2283	case 227:
		2284	case 211:
		2285	block = "CB_IMMED";
		2286	break;
		2287	case 36:
		2288	case 20:
		2289	case 100:
		2290	case 84:
		2291	case 164:
		2292	case 148:
		2293	case 228:
		2294	case 212:
		2295	block = "DB";
		2296	break;
		2297	case 37:
		2298	case 21:
		2299	case 101:
		2300	case 85:
		2301	case 165:
		2302	case 149:
		2303	case 229:
		2304	case 213:
		2305	block = "DB_HTILE";
		2306	break;
		2307	case 38:
		2308	case 22:
		2309	case 102:
		2310	case 86:
		2311	case 166:
		2312	case 150:
		2313	case 230:
		2314	case 214:
		2315	block = "SX";
		2316	break;
		2317	case 39:
		2318	case 23:
		2319	case 103:
		2320	case 87:
		2321	case 167:
		2322	case 151:
		2323	case 231:
		2324	case 215:
		2325	block = "DB_STEN";
		2326	break;
		2327	case 40:
		2328	case 24:
		2329	case 104:
		2330	case 88:
		2331	case 232:
		2332	case 216:
		2333	case 168:
		2334	case 152:
		2335	block = "TC_TFETCH";
		2336	break;
		2337	case 41:
		2338	case 25:
		2339	case 105:
		2340	case 89:
		2341	case 233:
		2342	case 217:
		2343	case 169:
		2344	case 153:
		2345	block = "TC_VFETCH";
		2346	break;
		2347	case 42:
		2348	case 26:
		2349	case 106:
		2350	case 90:
		2351	case 234:
		2352	case 218:
		2353	case 170:
		2354	case 154:
		2355	block = "VC";
		2356	break;
		2357	case 112:
		2358	block = "CP";
		2359	break;
		2360	case 113:
		2361	case 114:
		2362	block = "SH";
		2363	break;
		2364	case 115:
		2365	block = "VGT";
		2366	break;
		2367	case 178:
		2368	block = "IH";
		2369	break;
		2370	case 51:
		2371	block = "RLC";
		2372	break;
		2373	case 55:
		2374	block = "DMA";
		2375	break;
		2376	case 56:
		2377	block = "HDP";
		2378	break;
		2379	default:
		2380	block = "unknown";
		2381	break;
5271	serge	2382	}
2997	Serge	2383
5078	serge	2384	printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
		2385	protections, vmid, addr,
		2386	(status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
		2387	block, mc_id);
2997	Serge	2388	}
		2389
		2390	/**
		2391	* cayman_vm_flush - vm flush using the CP
		2392	*
		2393	* @rdev: radeon_device pointer
		2394	*
		2395	* Update the page table base and flush the VM TLB
		2396	* using the CP (cayman-si).
		2397	*/
5271	serge	2398	void cayman_vm_flush(struct radeon_device rdev, struct radeon_ring ring,
		2399	unsigned vm_id, uint64_t pd_addr)
2997	Serge	2400	{
5271	serge	2401	radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
		2402	radeon_ring_write(ring, pd_addr >> 12);
2997	Serge	2403
		2404	/* flush hdp cache */
		2405	radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
		2406	radeon_ring_write(ring, 0x1);
		2407
		2408	/* bits 0-7 are the VM contexts0-7 */
		2409	radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
5271	serge	2410	radeon_ring_write(ring, 1 << vm_id);
2997	Serge	2411
		2412	/* sync PFP to ME, otherwise we might get invalid PFP reads */
		2413	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
		2414	radeon_ring_write(ring, 0x0);
		2415	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/ni.c – Rev 5271