WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/ni.c

Rev	Author	Line No.	Line
1990	serge	1	/*
		2	* Copyright 2010 Advanced Micro Devices, Inc.
		3	*
		4	* Permission is hereby granted, free of charge, to any person obtaining a
		5	* copy of this software and associated documentation files (the "Software"),
		6	* to deal in the Software without restriction, including without limitation
		7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		8	* and/or sell copies of the Software, and to permit persons to whom the
		9	* Software is furnished to do so, subject to the following conditions:
		10	*
		11	* The above copyright notice and this permission notice shall be included in
		12	* all copies or substantial portions of the Software.
		13	*
		14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		20	* OTHER DEALINGS IN THE SOFTWARE.
		21	*
		22	* Authors: Alex Deucher
		23	*/
		24	#include
		25	//#include
		26	#include
2997	Serge	27	#include
		28	#include
1990	serge	29	#include "radeon.h"
		30	#include "radeon_asic.h"
2997	Serge	31	#include
1990	serge	32	#include "nid.h"
		33	#include "atom.h"
		34	#include "ni_reg.h"
2004	serge	35	#include "cayman_blit_shaders.h"
1990	serge	36
3764	Serge	37	extern bool evergreen_is_display_hung(struct radeon_device *rdev);
		38	extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
1990	serge	39	extern void evergreen_mc_stop(struct radeon_device rdev, struct evergreen_mc_save save);
		40	extern void evergreen_mc_resume(struct radeon_device rdev, struct evergreen_mc_save save);
		41	extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
		42	extern void evergreen_mc_program(struct radeon_device *rdev);
		43	extern void evergreen_irq_suspend(struct radeon_device *rdev);
		44	extern int evergreen_mc_init(struct radeon_device *rdev);
2997	Serge	45	extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
		46	extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
		47	extern void si_rlc_fini(struct radeon_device *rdev);
		48	extern int si_rlc_init(struct radeon_device *rdev);
1990	serge	49
		50	#define EVERGREEN_PFP_UCODE_SIZE 1120
		51	#define EVERGREEN_PM4_UCODE_SIZE 1376
		52	#define EVERGREEN_RLC_UCODE_SIZE 768
		53	#define BTC_MC_UCODE_SIZE 6024
		54
		55	#define CAYMAN_PFP_UCODE_SIZE 2176
		56	#define CAYMAN_PM4_UCODE_SIZE 2176
		57	#define CAYMAN_RLC_UCODE_SIZE 1024
		58	#define CAYMAN_MC_UCODE_SIZE 6037
		59
2997	Serge	60	#define ARUBA_RLC_UCODE_SIZE 1536
		61
1990	serge	62	/* Firmware Names */
		63	MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
		64	MODULE_FIRMWARE("radeon/BARTS_me.bin");
		65	MODULE_FIRMWARE("radeon/BARTS_mc.bin");
		66	MODULE_FIRMWARE("radeon/BTC_rlc.bin");
		67	MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
		68	MODULE_FIRMWARE("radeon/TURKS_me.bin");
		69	MODULE_FIRMWARE("radeon/TURKS_mc.bin");
		70	MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
		71	MODULE_FIRMWARE("radeon/CAICOS_me.bin");
		72	MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
		73	MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
		74	MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
		75	MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
		76	MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
2997	Serge	77	MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
		78	MODULE_FIRMWARE("radeon/ARUBA_me.bin");
		79	MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
1990	serge	80
3764	Serge	81
		82	static const u32 cayman_golden_registers2[] =
		83	{
		84	0x3e5c, 0xffffffff, 0x00000000,
		85	0x3e48, 0xffffffff, 0x00000000,
		86	0x3e4c, 0xffffffff, 0x00000000,
		87	0x3e64, 0xffffffff, 0x00000000,
		88	0x3e50, 0xffffffff, 0x00000000,
		89	0x3e60, 0xffffffff, 0x00000000
		90	};
		91
		92	static const u32 cayman_golden_registers[] =
		93	{
		94	0x5eb4, 0xffffffff, 0x00000002,
		95	0x5e78, 0x8f311ff1, 0x001000f0,
		96	0x3f90, 0xffff0000, 0xff000000,
		97	0x9148, 0xffff0000, 0xff000000,
		98	0x3f94, 0xffff0000, 0xff000000,
		99	0x914c, 0xffff0000, 0xff000000,
		100	0xc78, 0x00000080, 0x00000080,
		101	0xbd4, 0x70073777, 0x00011003,
		102	0xd02c, 0xbfffff1f, 0x08421000,
		103	0xd0b8, 0x73773777, 0x02011003,
		104	0x5bc0, 0x00200000, 0x50100000,
		105	0x98f8, 0x33773777, 0x02011003,
		106	0x98fc, 0xffffffff, 0x76541032,
		107	0x7030, 0x31000311, 0x00000011,
		108	0x2f48, 0x33773777, 0x42010001,
		109	0x6b28, 0x00000010, 0x00000012,
		110	0x7728, 0x00000010, 0x00000012,
		111	0x10328, 0x00000010, 0x00000012,
		112	0x10f28, 0x00000010, 0x00000012,
		113	0x11b28, 0x00000010, 0x00000012,
		114	0x12728, 0x00000010, 0x00000012,
		115	0x240c, 0x000007ff, 0x00000000,
		116	0x8a14, 0xf000001f, 0x00000007,
		117	0x8b24, 0x3fff3fff, 0x00ff0fff,
		118	0x8b10, 0x0000ff0f, 0x00000000,
		119	0x28a4c, 0x07ffffff, 0x06000000,
		120	0x10c, 0x00000001, 0x00010003,
		121	0xa02c, 0xffffffff, 0x0000009b,
		122	0x913c, 0x0000010f, 0x01000100,
		123	0x8c04, 0xf8ff00ff, 0x40600060,
		124	0x28350, 0x00000f01, 0x00000000,
		125	0x9508, 0x3700001f, 0x00000002,
		126	0x960c, 0xffffffff, 0x54763210,
		127	0x88c4, 0x001f3ae3, 0x00000082,
		128	0x88d0, 0xffffffff, 0x0f40df40,
		129	0x88d4, 0x0000001f, 0x00000010,
		130	0x8974, 0xffffffff, 0x00000000
		131	};
		132
		133	static const u32 dvst_golden_registers2[] =
		134	{
		135	0x8f8, 0xffffffff, 0,
		136	0x8fc, 0x00380000, 0,
		137	0x8f8, 0xffffffff, 1,
		138	0x8fc, 0x0e000000, 0
		139	};
		140
		141	static const u32 dvst_golden_registers[] =
		142	{
		143	0x690, 0x3fff3fff, 0x20c00033,
		144	0x918c, 0x0fff0fff, 0x00010006,
		145	0x91a8, 0x0fff0fff, 0x00010006,
		146	0x9150, 0xffffdfff, 0x6e944040,
		147	0x917c, 0x0fff0fff, 0x00030002,
		148	0x9198, 0x0fff0fff, 0x00030002,
		149	0x915c, 0x0fff0fff, 0x00010000,
		150	0x3f90, 0xffff0001, 0xff000000,
		151	0x9178, 0x0fff0fff, 0x00070000,
		152	0x9194, 0x0fff0fff, 0x00070000,
		153	0x9148, 0xffff0001, 0xff000000,
		154	0x9190, 0x0fff0fff, 0x00090008,
		155	0x91ac, 0x0fff0fff, 0x00090008,
		156	0x3f94, 0xffff0000, 0xff000000,
		157	0x914c, 0xffff0000, 0xff000000,
		158	0x929c, 0x00000fff, 0x00000001,
		159	0x55e4, 0xff607fff, 0xfc000100,
		160	0x8a18, 0xff000fff, 0x00000100,
		161	0x8b28, 0xff000fff, 0x00000100,
		162	0x9144, 0xfffc0fff, 0x00000100,
		163	0x6ed8, 0x00010101, 0x00010000,
		164	0x9830, 0xffffffff, 0x00000000,
		165	0x9834, 0xf00fffff, 0x00000400,
		166	0x9838, 0xfffffffe, 0x00000000,
		167	0xd0c0, 0xff000fff, 0x00000100,
		168	0xd02c, 0xbfffff1f, 0x08421000,
		169	0xd0b8, 0x73773777, 0x12010001,
		170	0x5bb0, 0x000000f0, 0x00000070,
		171	0x98f8, 0x73773777, 0x12010001,
		172	0x98fc, 0xffffffff, 0x00000010,
		173	0x9b7c, 0x00ff0000, 0x00fc0000,
		174	0x8030, 0x00001f0f, 0x0000100a,
		175	0x2f48, 0x73773777, 0x12010001,
		176	0x2408, 0x00030000, 0x000c007f,
		177	0x8a14, 0xf000003f, 0x00000007,
		178	0x8b24, 0x3fff3fff, 0x00ff0fff,
		179	0x8b10, 0x0000ff0f, 0x00000000,
		180	0x28a4c, 0x07ffffff, 0x06000000,
		181	0x4d8, 0x00000fff, 0x00000100,
		182	0xa008, 0xffffffff, 0x00010000,
		183	0x913c, 0xffff03ff, 0x01000100,
		184	0x8c00, 0x000000ff, 0x00000003,
		185	0x8c04, 0xf8ff00ff, 0x40600060,
		186	0x8cf0, 0x1fff1fff, 0x08e00410,
		187	0x28350, 0x00000f01, 0x00000000,
		188	0x9508, 0xf700071f, 0x00000002,
		189	0x960c, 0xffffffff, 0x54763210,
		190	0x20ef8, 0x01ff01ff, 0x00000002,
		191	0x20e98, 0xfffffbff, 0x00200000,
		192	0x2015c, 0xffffffff, 0x00000f40,
		193	0x88c4, 0x001f3ae3, 0x00000082,
		194	0x8978, 0x3fffffff, 0x04050140,
		195	0x88d4, 0x0000001f, 0x00000010,
		196	0x8974, 0xffffffff, 0x00000000
		197	};
		198
		199	static const u32 scrapper_golden_registers[] =
		200	{
		201	0x690, 0x3fff3fff, 0x20c00033,
		202	0x918c, 0x0fff0fff, 0x00010006,
		203	0x918c, 0x0fff0fff, 0x00010006,
		204	0x91a8, 0x0fff0fff, 0x00010006,
		205	0x91a8, 0x0fff0fff, 0x00010006,
		206	0x9150, 0xffffdfff, 0x6e944040,
		207	0x9150, 0xffffdfff, 0x6e944040,
		208	0x917c, 0x0fff0fff, 0x00030002,
		209	0x917c, 0x0fff0fff, 0x00030002,
		210	0x9198, 0x0fff0fff, 0x00030002,
		211	0x9198, 0x0fff0fff, 0x00030002,
		212	0x915c, 0x0fff0fff, 0x00010000,
		213	0x915c, 0x0fff0fff, 0x00010000,
		214	0x3f90, 0xffff0001, 0xff000000,
		215	0x3f90, 0xffff0001, 0xff000000,
		216	0x9178, 0x0fff0fff, 0x00070000,
		217	0x9178, 0x0fff0fff, 0x00070000,
		218	0x9194, 0x0fff0fff, 0x00070000,
		219	0x9194, 0x0fff0fff, 0x00070000,
		220	0x9148, 0xffff0001, 0xff000000,
		221	0x9148, 0xffff0001, 0xff000000,
		222	0x9190, 0x0fff0fff, 0x00090008,
		223	0x9190, 0x0fff0fff, 0x00090008,
		224	0x91ac, 0x0fff0fff, 0x00090008,
		225	0x91ac, 0x0fff0fff, 0x00090008,
		226	0x3f94, 0xffff0000, 0xff000000,
		227	0x3f94, 0xffff0000, 0xff000000,
		228	0x914c, 0xffff0000, 0xff000000,
		229	0x914c, 0xffff0000, 0xff000000,
		230	0x929c, 0x00000fff, 0x00000001,
		231	0x929c, 0x00000fff, 0x00000001,
		232	0x55e4, 0xff607fff, 0xfc000100,
		233	0x8a18, 0xff000fff, 0x00000100,
		234	0x8a18, 0xff000fff, 0x00000100,
		235	0x8b28, 0xff000fff, 0x00000100,
		236	0x8b28, 0xff000fff, 0x00000100,
		237	0x9144, 0xfffc0fff, 0x00000100,
		238	0x9144, 0xfffc0fff, 0x00000100,
		239	0x6ed8, 0x00010101, 0x00010000,
		240	0x9830, 0xffffffff, 0x00000000,
		241	0x9830, 0xffffffff, 0x00000000,
		242	0x9834, 0xf00fffff, 0x00000400,
		243	0x9834, 0xf00fffff, 0x00000400,
		244	0x9838, 0xfffffffe, 0x00000000,
		245	0x9838, 0xfffffffe, 0x00000000,
		246	0xd0c0, 0xff000fff, 0x00000100,
		247	0xd02c, 0xbfffff1f, 0x08421000,
		248	0xd02c, 0xbfffff1f, 0x08421000,
		249	0xd0b8, 0x73773777, 0x12010001,
		250	0xd0b8, 0x73773777, 0x12010001,
		251	0x5bb0, 0x000000f0, 0x00000070,
		252	0x98f8, 0x73773777, 0x12010001,
		253	0x98f8, 0x73773777, 0x12010001,
		254	0x98fc, 0xffffffff, 0x00000010,
		255	0x98fc, 0xffffffff, 0x00000010,
		256	0x9b7c, 0x00ff0000, 0x00fc0000,
		257	0x9b7c, 0x00ff0000, 0x00fc0000,
		258	0x8030, 0x00001f0f, 0x0000100a,
		259	0x8030, 0x00001f0f, 0x0000100a,
		260	0x2f48, 0x73773777, 0x12010001,
		261	0x2f48, 0x73773777, 0x12010001,
		262	0x2408, 0x00030000, 0x000c007f,
		263	0x8a14, 0xf000003f, 0x00000007,
		264	0x8a14, 0xf000003f, 0x00000007,
		265	0x8b24, 0x3fff3fff, 0x00ff0fff,
		266	0x8b24, 0x3fff3fff, 0x00ff0fff,
		267	0x8b10, 0x0000ff0f, 0x00000000,
		268	0x8b10, 0x0000ff0f, 0x00000000,
		269	0x28a4c, 0x07ffffff, 0x06000000,
		270	0x28a4c, 0x07ffffff, 0x06000000,
		271	0x4d8, 0x00000fff, 0x00000100,
		272	0x4d8, 0x00000fff, 0x00000100,
		273	0xa008, 0xffffffff, 0x00010000,
		274	0xa008, 0xffffffff, 0x00010000,
		275	0x913c, 0xffff03ff, 0x01000100,
		276	0x913c, 0xffff03ff, 0x01000100,
		277	0x90e8, 0x001fffff, 0x010400c0,
		278	0x8c00, 0x000000ff, 0x00000003,
		279	0x8c00, 0x000000ff, 0x00000003,
		280	0x8c04, 0xf8ff00ff, 0x40600060,
		281	0x8c04, 0xf8ff00ff, 0x40600060,
		282	0x8c30, 0x0000000f, 0x00040005,
		283	0x8cf0, 0x1fff1fff, 0x08e00410,
		284	0x8cf0, 0x1fff1fff, 0x08e00410,
		285	0x900c, 0x00ffffff, 0x0017071f,
		286	0x28350, 0x00000f01, 0x00000000,
		287	0x28350, 0x00000f01, 0x00000000,
		288	0x9508, 0xf700071f, 0x00000002,
		289	0x9508, 0xf700071f, 0x00000002,
		290	0x9688, 0x00300000, 0x0017000f,
		291	0x960c, 0xffffffff, 0x54763210,
		292	0x960c, 0xffffffff, 0x54763210,
		293	0x20ef8, 0x01ff01ff, 0x00000002,
		294	0x20e98, 0xfffffbff, 0x00200000,
		295	0x2015c, 0xffffffff, 0x00000f40,
		296	0x88c4, 0x001f3ae3, 0x00000082,
		297	0x88c4, 0x001f3ae3, 0x00000082,
		298	0x8978, 0x3fffffff, 0x04050140,
		299	0x8978, 0x3fffffff, 0x04050140,
		300	0x88d4, 0x0000001f, 0x00000010,
		301	0x88d4, 0x0000001f, 0x00000010,
		302	0x8974, 0xffffffff, 0x00000000,
		303	0x8974, 0xffffffff, 0x00000000
		304	};
		305
		306	static void ni_init_golden_registers(struct radeon_device *rdev)
		307	{
		308	switch (rdev->family) {
		309	case CHIP_CAYMAN:
		310	radeon_program_register_sequence(rdev,
		311	cayman_golden_registers,
		312	(const u32)ARRAY_SIZE(cayman_golden_registers));
		313	radeon_program_register_sequence(rdev,
		314	cayman_golden_registers2,
		315	(const u32)ARRAY_SIZE(cayman_golden_registers2));
		316	break;
		317	case CHIP_ARUBA:
		318	if ((rdev->pdev->device == 0x9900) \|\|
		319	(rdev->pdev->device == 0x9901) \|\|
		320	(rdev->pdev->device == 0x9903) \|\|
		321	(rdev->pdev->device == 0x9904) \|\|
		322	(rdev->pdev->device == 0x9905) \|\|
		323	(rdev->pdev->device == 0x9906) \|\|
		324	(rdev->pdev->device == 0x9907) \|\|
		325	(rdev->pdev->device == 0x9908) \|\|
		326	(rdev->pdev->device == 0x9909) \|\|
		327	(rdev->pdev->device == 0x990A) \|\|
		328	(rdev->pdev->device == 0x990B) \|\|
		329	(rdev->pdev->device == 0x990C) \|\|
		330	(rdev->pdev->device == 0x990D) \|\|
		331	(rdev->pdev->device == 0x990E) \|\|
		332	(rdev->pdev->device == 0x990F) \|\|
		333	(rdev->pdev->device == 0x9910) \|\|
		334	(rdev->pdev->device == 0x9913) \|\|
		335	(rdev->pdev->device == 0x9917) \|\|
		336	(rdev->pdev->device == 0x9918)) {
		337	radeon_program_register_sequence(rdev,
		338	dvst_golden_registers,
		339	(const u32)ARRAY_SIZE(dvst_golden_registers));
		340	radeon_program_register_sequence(rdev,
		341	dvst_golden_registers2,
		342	(const u32)ARRAY_SIZE(dvst_golden_registers2));
		343	} else {
		344	radeon_program_register_sequence(rdev,
		345	scrapper_golden_registers,
		346	(const u32)ARRAY_SIZE(scrapper_golden_registers));
		347	radeon_program_register_sequence(rdev,
		348	dvst_golden_registers2,
		349	(const u32)ARRAY_SIZE(dvst_golden_registers2));
		350	}
		351	break;
		352	default:
		353	break;
		354	}
		355	}
		356
1990	serge	357	#define BTC_IO_MC_REGS_SIZE 29
		358
		359	static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
		360	{0x00000077, 0xff010100},
		361	{0x00000078, 0x00000000},
		362	{0x00000079, 0x00001434},
		363	{0x0000007a, 0xcc08ec08},
		364	{0x0000007b, 0x00040000},
		365	{0x0000007c, 0x000080c0},
		366	{0x0000007d, 0x09000000},
		367	{0x0000007e, 0x00210404},
		368	{0x00000081, 0x08a8e800},
		369	{0x00000082, 0x00030444},
		370	{0x00000083, 0x00000000},
		371	{0x00000085, 0x00000001},
		372	{0x00000086, 0x00000002},
		373	{0x00000087, 0x48490000},
		374	{0x00000088, 0x20244647},
		375	{0x00000089, 0x00000005},
		376	{0x0000008b, 0x66030000},
		377	{0x0000008c, 0x00006603},
		378	{0x0000008d, 0x00000100},
		379	{0x0000008f, 0x00001c0a},
		380	{0x00000090, 0xff000001},
		381	{0x00000094, 0x00101101},
		382	{0x00000095, 0x00000fff},
		383	{0x00000096, 0x00116fff},
		384	{0x00000097, 0x60010000},
		385	{0x00000098, 0x10010000},
		386	{0x00000099, 0x00006000},
		387	{0x0000009a, 0x00001000},
		388	{0x0000009f, 0x00946a00}
		389	};
		390
		391	static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
		392	{0x00000077, 0xff010100},
		393	{0x00000078, 0x00000000},
		394	{0x00000079, 0x00001434},
		395	{0x0000007a, 0xcc08ec08},
		396	{0x0000007b, 0x00040000},
		397	{0x0000007c, 0x000080c0},
		398	{0x0000007d, 0x09000000},
		399	{0x0000007e, 0x00210404},
		400	{0x00000081, 0x08a8e800},
		401	{0x00000082, 0x00030444},
		402	{0x00000083, 0x00000000},
		403	{0x00000085, 0x00000001},
		404	{0x00000086, 0x00000002},
		405	{0x00000087, 0x48490000},
		406	{0x00000088, 0x20244647},
		407	{0x00000089, 0x00000005},
		408	{0x0000008b, 0x66030000},
		409	{0x0000008c, 0x00006603},
		410	{0x0000008d, 0x00000100},
		411	{0x0000008f, 0x00001c0a},
		412	{0x00000090, 0xff000001},
		413	{0x00000094, 0x00101101},
		414	{0x00000095, 0x00000fff},
		415	{0x00000096, 0x00116fff},
		416	{0x00000097, 0x60010000},
		417	{0x00000098, 0x10010000},
		418	{0x00000099, 0x00006000},
		419	{0x0000009a, 0x00001000},
		420	{0x0000009f, 0x00936a00}
		421	};
		422
		423	static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
		424	{0x00000077, 0xff010100},
		425	{0x00000078, 0x00000000},
		426	{0x00000079, 0x00001434},
		427	{0x0000007a, 0xcc08ec08},
		428	{0x0000007b, 0x00040000},
		429	{0x0000007c, 0x000080c0},
		430	{0x0000007d, 0x09000000},
		431	{0x0000007e, 0x00210404},
		432	{0x00000081, 0x08a8e800},
		433	{0x00000082, 0x00030444},
		434	{0x00000083, 0x00000000},
		435	{0x00000085, 0x00000001},
		436	{0x00000086, 0x00000002},
		437	{0x00000087, 0x48490000},
		438	{0x00000088, 0x20244647},
		439	{0x00000089, 0x00000005},
		440	{0x0000008b, 0x66030000},
		441	{0x0000008c, 0x00006603},
		442	{0x0000008d, 0x00000100},
		443	{0x0000008f, 0x00001c0a},
		444	{0x00000090, 0xff000001},
		445	{0x00000094, 0x00101101},
		446	{0x00000095, 0x00000fff},
		447	{0x00000096, 0x00116fff},
		448	{0x00000097, 0x60010000},
		449	{0x00000098, 0x10010000},
		450	{0x00000099, 0x00006000},
		451	{0x0000009a, 0x00001000},
		452	{0x0000009f, 0x00916a00}
		453	};
		454
		455	static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
		456	{0x00000077, 0xff010100},
		457	{0x00000078, 0x00000000},
		458	{0x00000079, 0x00001434},
		459	{0x0000007a, 0xcc08ec08},
		460	{0x0000007b, 0x00040000},
		461	{0x0000007c, 0x000080c0},
		462	{0x0000007d, 0x09000000},
		463	{0x0000007e, 0x00210404},
		464	{0x00000081, 0x08a8e800},
		465	{0x00000082, 0x00030444},
		466	{0x00000083, 0x00000000},
		467	{0x00000085, 0x00000001},
		468	{0x00000086, 0x00000002},
		469	{0x00000087, 0x48490000},
		470	{0x00000088, 0x20244647},
		471	{0x00000089, 0x00000005},
		472	{0x0000008b, 0x66030000},
		473	{0x0000008c, 0x00006603},
		474	{0x0000008d, 0x00000100},
		475	{0x0000008f, 0x00001c0a},
		476	{0x00000090, 0xff000001},
		477	{0x00000094, 0x00101101},
		478	{0x00000095, 0x00000fff},
		479	{0x00000096, 0x00116fff},
		480	{0x00000097, 0x60010000},
		481	{0x00000098, 0x10010000},
		482	{0x00000099, 0x00006000},
		483	{0x0000009a, 0x00001000},
		484	{0x0000009f, 0x00976b00}
		485	};
		486
		487	int ni_mc_load_microcode(struct radeon_device *rdev)
		488	{
		489	const __be32 *fw_data;
		490	u32 mem_type, running, blackout = 0;
		491	u32 *io_mc_regs;
		492	int i, ucode_size, regs_size;
		493
		494	if (!rdev->mc_fw)
		495	return -EINVAL;
		496
		497	switch (rdev->family) {
		498	case CHIP_BARTS:
		499	io_mc_regs = (u32 *)&barts_io_mc_regs;
		500	ucode_size = BTC_MC_UCODE_SIZE;
		501	regs_size = BTC_IO_MC_REGS_SIZE;
		502	break;
		503	case CHIP_TURKS:
		504	io_mc_regs = (u32 *)&turks_io_mc_regs;
		505	ucode_size = BTC_MC_UCODE_SIZE;
		506	regs_size = BTC_IO_MC_REGS_SIZE;
		507	break;
		508	case CHIP_CAICOS:
		509	default:
		510	io_mc_regs = (u32 *)&caicos_io_mc_regs;
		511	ucode_size = BTC_MC_UCODE_SIZE;
		512	regs_size = BTC_IO_MC_REGS_SIZE;
		513	break;
		514	case CHIP_CAYMAN:
		515	io_mc_regs = (u32 *)&cayman_io_mc_regs;
		516	ucode_size = CAYMAN_MC_UCODE_SIZE;
		517	regs_size = BTC_IO_MC_REGS_SIZE;
		518	break;
		519	}
		520
		521	mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
		522	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
		523
		524	if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
		525	if (running) {
		526	blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
		527	WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
		528	}
		529
		530	/* reset the engine and set to writable */
		531	WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
		532	WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
		533
		534	/* load mc io regs */
		535	for (i = 0; i < regs_size; i++) {
		536	WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
		537	WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
		538	}
		539	/* load the MC ucode */
		540	fw_data = (const __be32 *)rdev->mc_fw->data;
		541	for (i = 0; i < ucode_size; i++)
		542	WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
		543
		544	/* put the engine back into the active state */
		545	WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
		546	WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
		547	WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
		548
		549	/* wait for training to complete */
2997	Serge	550	for (i = 0; i < rdev->usec_timeout; i++) {
		551	if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
		552	break;
		553	udelay(1);
		554	}
1990	serge	555
		556	if (running)
		557	WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
		558	}
		559
		560	return 0;
		561	}
		562
		563	int ni_init_microcode(struct radeon_device *rdev)
		564	{
		565	struct platform_device *pdev;
		566	const char *chip_name;
		567	const char *rlc_chip_name;
		568	size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
		569	char fw_name[30];
		570	int err;
		571
		572	DRM_DEBUG("\n");
		573
		574	pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
		575	err = IS_ERR(pdev);
		576	if (err) {
		577	printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
		578	return -EINVAL;
		579	}
		580
		581	switch (rdev->family) {
		582	case CHIP_BARTS:
		583	chip_name = "BARTS";
		584	rlc_chip_name = "BTC";
		585	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		586	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		587	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		588	mc_req_size = BTC_MC_UCODE_SIZE * 4;
		589	break;
		590	case CHIP_TURKS:
		591	chip_name = "TURKS";
		592	rlc_chip_name = "BTC";
		593	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		594	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		595	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		596	mc_req_size = BTC_MC_UCODE_SIZE * 4;
		597	break;
		598	case CHIP_CAICOS:
		599	chip_name = "CAICOS";
		600	rlc_chip_name = "BTC";
		601	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		602	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		603	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		604	mc_req_size = BTC_MC_UCODE_SIZE * 4;
		605	break;
		606	case CHIP_CAYMAN:
		607	chip_name = "CAYMAN";
		608	rlc_chip_name = "CAYMAN";
		609	pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
		610	me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
		611	rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
		612	mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
		613	break;
2997	Serge	614	case CHIP_ARUBA:
		615	chip_name = "ARUBA";
		616	rlc_chip_name = "ARUBA";
		617	/* pfp/me same size as CAYMAN */
		618	pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
		619	me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
		620	rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
		621	mc_req_size = 0;
		622	break;
1990	serge	623	default: BUG();
		624	}
		625
		626	DRM_INFO("Loading %s Microcode\n", chip_name);
		627
		628	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
		629	err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
		630	if (err)
		631	goto out;
		632	if (rdev->pfp_fw->size != pfp_req_size) {
		633	printk(KERN_ERR
		634	"ni_cp: Bogus length %zu in firmware \"%s\"\n",
		635	rdev->pfp_fw->size, fw_name);
		636	err = -EINVAL;
		637	goto out;
		638	}
		639
		640	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
		641	err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
		642	if (err)
		643	goto out;
		644	if (rdev->me_fw->size != me_req_size) {
		645	printk(KERN_ERR
		646	"ni_cp: Bogus length %zu in firmware \"%s\"\n",
		647	rdev->me_fw->size, fw_name);
		648	err = -EINVAL;
		649	}
		650
		651	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
		652	err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
		653	if (err)
		654	goto out;
		655	if (rdev->rlc_fw->size != rlc_req_size) {
		656	printk(KERN_ERR
		657	"ni_rlc: Bogus length %zu in firmware \"%s\"\n",
		658	rdev->rlc_fw->size, fw_name);
		659	err = -EINVAL;
		660	}
		661
2997	Serge	662	/* no MC ucode on TN */
		663	if (!(rdev->flags & RADEON_IS_IGP)) {
1990	serge	664	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
		665	err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
		666	if (err)
		667	goto out;
		668	if (rdev->mc_fw->size != mc_req_size) {
		669	printk(KERN_ERR
		670	"ni_mc: Bogus length %zu in firmware \"%s\"\n",
		671	rdev->mc_fw->size, fw_name);
		672	err = -EINVAL;
		673	}
2997	Serge	674	}
1990	serge	675	out:
		676	platform_device_unregister(pdev);
		677
		678	if (err) {
		679	if (err != -EINVAL)
		680	printk(KERN_ERR
		681	"ni_cp: Failed to load firmware \"%s\"\n",
		682	fw_name);
		683	release_firmware(rdev->pfp_fw);
		684	rdev->pfp_fw = NULL;
		685	release_firmware(rdev->me_fw);
		686	rdev->me_fw = NULL;
		687	release_firmware(rdev->rlc_fw);
		688	rdev->rlc_fw = NULL;
		689	release_firmware(rdev->mc_fw);
		690	rdev->mc_fw = NULL;
		691	}
		692	return err;
		693	}
		694
		695	/*
		696	* Core functions
		697	*/
		698	static void cayman_gpu_init(struct radeon_device *rdev)
		699	{
		700	u32 gb_addr_config = 0;
		701	u32 mc_shared_chmap, mc_arb_ramcfg;
		702	u32 cgts_tcc_disable;
		703	u32 sx_debug_1;
		704	u32 smx_dc_ctl0;
		705	u32 cgts_sm_ctrl_reg;
		706	u32 hdp_host_path_cntl;
		707	u32 tmp;
2997	Serge	708	u32 disabled_rb_mask;
1990	serge	709	int i, j;
		710
		711	switch (rdev->family) {
		712	case CHIP_CAYMAN:
		713	rdev->config.cayman.max_shader_engines = 2;
		714	rdev->config.cayman.max_pipes_per_simd = 4;
		715	rdev->config.cayman.max_tile_pipes = 8;
		716	rdev->config.cayman.max_simds_per_se = 12;
		717	rdev->config.cayman.max_backends_per_se = 4;
		718	rdev->config.cayman.max_texture_channel_caches = 8;
		719	rdev->config.cayman.max_gprs = 256;
		720	rdev->config.cayman.max_threads = 256;
		721	rdev->config.cayman.max_gs_threads = 32;
		722	rdev->config.cayman.max_stack_entries = 512;
		723	rdev->config.cayman.sx_num_of_sets = 8;
		724	rdev->config.cayman.sx_max_export_size = 256;
		725	rdev->config.cayman.sx_max_export_pos_size = 64;
		726	rdev->config.cayman.sx_max_export_smx_size = 192;
		727	rdev->config.cayman.max_hw_contexts = 8;
		728	rdev->config.cayman.sq_num_cf_insts = 2;
		729
		730	rdev->config.cayman.sc_prim_fifo_size = 0x100;
		731	rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
		732	rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
2997	Serge	733	gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
1990	serge	734	break;
2997	Serge	735	case CHIP_ARUBA:
		736	default:
		737	rdev->config.cayman.max_shader_engines = 1;
		738	rdev->config.cayman.max_pipes_per_simd = 4;
		739	rdev->config.cayman.max_tile_pipes = 2;
		740	if ((rdev->pdev->device == 0x9900) \|\|
		741	(rdev->pdev->device == 0x9901) \|\|
		742	(rdev->pdev->device == 0x9905) \|\|
		743	(rdev->pdev->device == 0x9906) \|\|
		744	(rdev->pdev->device == 0x9907) \|\|
		745	(rdev->pdev->device == 0x9908) \|\|
		746	(rdev->pdev->device == 0x9909) \|\|
3764	Serge	747	(rdev->pdev->device == 0x990B) \|\|
		748	(rdev->pdev->device == 0x990C) \|\|
		749	(rdev->pdev->device == 0x990F) \|\|
2997	Serge	750	(rdev->pdev->device == 0x9910) \|\|
3764	Serge	751	(rdev->pdev->device == 0x9917) \|\|
		752	(rdev->pdev->device == 0x9999) \|\|
		753	(rdev->pdev->device == 0x999C)) {
2997	Serge	754	rdev->config.cayman.max_simds_per_se = 6;
		755	rdev->config.cayman.max_backends_per_se = 2;
		756	} else if ((rdev->pdev->device == 0x9903) \|\|
		757	(rdev->pdev->device == 0x9904) \|\|
		758	(rdev->pdev->device == 0x990A) \|\|
3764	Serge	759	(rdev->pdev->device == 0x990D) \|\|
		760	(rdev->pdev->device == 0x990E) \|\|
2997	Serge	761	(rdev->pdev->device == 0x9913) \|\|
3764	Serge	762	(rdev->pdev->device == 0x9918) \|\|
		763	(rdev->pdev->device == 0x999D)) {
2997	Serge	764	rdev->config.cayman.max_simds_per_se = 4;
		765	rdev->config.cayman.max_backends_per_se = 2;
		766	} else if ((rdev->pdev->device == 0x9919) \|\|
		767	(rdev->pdev->device == 0x9990) \|\|
		768	(rdev->pdev->device == 0x9991) \|\|
		769	(rdev->pdev->device == 0x9994) \|\|
3764	Serge	770	(rdev->pdev->device == 0x9995) \|\|
		771	(rdev->pdev->device == 0x9996) \|\|
		772	(rdev->pdev->device == 0x999A) \|\|
2997	Serge	773	(rdev->pdev->device == 0x99A0)) {
		774	rdev->config.cayman.max_simds_per_se = 3;
		775	rdev->config.cayman.max_backends_per_se = 1;
		776	} else {
		777	rdev->config.cayman.max_simds_per_se = 2;
		778	rdev->config.cayman.max_backends_per_se = 1;
		779	}
		780	rdev->config.cayman.max_texture_channel_caches = 2;
		781	rdev->config.cayman.max_gprs = 256;
		782	rdev->config.cayman.max_threads = 256;
		783	rdev->config.cayman.max_gs_threads = 32;
		784	rdev->config.cayman.max_stack_entries = 512;
		785	rdev->config.cayman.sx_num_of_sets = 8;
		786	rdev->config.cayman.sx_max_export_size = 256;
		787	rdev->config.cayman.sx_max_export_pos_size = 64;
		788	rdev->config.cayman.sx_max_export_smx_size = 192;
		789	rdev->config.cayman.max_hw_contexts = 8;
		790	rdev->config.cayman.sq_num_cf_insts = 2;
		791
		792	rdev->config.cayman.sc_prim_fifo_size = 0x40;
		793	rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
		794	rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
		795	gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
		796	break;
1990	serge	797	}
		798
		799	/* Initialize HDP */
		800	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		801	WREG32((0x2c14 + j), 0x00000000);
		802	WREG32((0x2c18 + j), 0x00000000);
		803	WREG32((0x2c1c + j), 0x00000000);
		804	WREG32((0x2c20 + j), 0x00000000);
		805	WREG32((0x2c24 + j), 0x00000000);
		806	}
		807
		808	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		809
2997	Serge	810	evergreen_fix_pci_max_read_req_size(rdev);
		811
1990	serge	812	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
		813	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
		814
		815	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
		816	rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
		817	if (rdev->config.cayman.mem_row_size_in_kb > 4)
		818	rdev->config.cayman.mem_row_size_in_kb = 4;
		819	/* XXX use MC settings? */
		820	rdev->config.cayman.shader_engine_tile_size = 32;
		821	rdev->config.cayman.num_gpus = 1;
		822	rdev->config.cayman.multi_gpu_tile_size = 64;
		823
		824	tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
		825	rdev->config.cayman.num_tile_pipes = (1 << tmp);
		826	tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
		827	rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
		828	tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
		829	rdev->config.cayman.num_shader_engines = tmp + 1;
		830	tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
		831	rdev->config.cayman.num_gpus = tmp + 1;
		832	tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
		833	rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
		834	tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
		835	rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
		836
2997	Serge	837
1990	serge	838	/* setup tiling info dword. gb_addr_config is not adequate since it does
		839	* not have bank info, so create a custom tiling dword.
		840	* bits 3:0 num_pipes
		841	* bits 7:4 num_banks
		842	* bits 11:8 group_size
		843	* bits 15:12 row_size
		844	*/
		845	rdev->config.cayman.tile_config = 0;
		846	switch (rdev->config.cayman.num_tile_pipes) {
		847	case 1:
		848	default:
		849	rdev->config.cayman.tile_config \|= (0 << 0);
		850	break;
		851	case 2:
		852	rdev->config.cayman.tile_config \|= (1 << 0);
		853	break;
		854	case 4:
		855	rdev->config.cayman.tile_config \|= (2 << 0);
		856	break;
		857	case 8:
		858	rdev->config.cayman.tile_config \|= (3 << 0);
		859	break;
		860	}
2997	Serge	861
		862	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
		863	if (rdev->flags & RADEON_IS_IGP)
		864	rdev->config.cayman.tile_config \|= 1 << 4;
		865	else {
		866	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
		867	case 0: /* four banks */
		868	rdev->config.cayman.tile_config \|= 0 << 4;
		869	break;
		870	case 1: /* eight banks */
		871	rdev->config.cayman.tile_config \|= 1 << 4;
		872	break;
		873	case 2: /* sixteen banks */
		874	default:
		875	rdev->config.cayman.tile_config \|= 2 << 4;
		876	break;
		877	}
		878	}
1990	serge	879	rdev->config.cayman.tile_config \|=
		880	((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
		881	rdev->config.cayman.tile_config \|=
		882	((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
		883
2997	Serge	884	tmp = 0;
		885	for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
		886	u32 rb_disable_bitmap;
		887
		888	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		889	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_INDEX(i));
		890	rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
		891	tmp <<= 4;
		892	tmp \|= rb_disable_bitmap;
		893	}
		894	/* enabled rb are just the one not disabled :) */
		895	disabled_rb_mask = tmp;
3764	Serge	896	tmp = 0;
		897	for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
		898	tmp \|= (1 << i);
		899	/* if all the backends are disabled, fix it up here */
		900	if ((disabled_rb_mask & tmp) == tmp) {
		901	for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
		902	disabled_rb_mask &= ~(1 << i);
		903	}
2997	Serge	904
		905	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
		906	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES \| SE_BROADCAST_WRITES);
		907
1990	serge	908	WREG32(GB_ADDR_CONFIG, gb_addr_config);
		909	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3764	Serge	910	if (ASIC_IS_DCE6(rdev))
		911	WREG32(DMIF_ADDR_CALC, gb_addr_config);
1990	serge	912	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3192	Serge	913	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
		914	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3764	Serge	915	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
		916	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
		917	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1990	serge	918
3764	Serge	919	if ((rdev->config.cayman.max_backends_per_se == 1) &&
		920	(rdev->flags & RADEON_IS_IGP)) {
		921	if ((disabled_rb_mask & 3) == 1) {
		922	/* RB0 disabled, RB1 enabled */
		923	tmp = 0x11111111;
		924	} else {
		925	/* RB1 disabled, RB0 enabled */
		926	tmp = 0x00000000;
		927	}
		928	} else {
2997	Serge	929	tmp = gb_addr_config & NUM_PIPES_MASK;
		930	tmp = r6xx_remap_render_backend(rdev, tmp,
		931	rdev->config.cayman.max_backends_per_se *
		932	rdev->config.cayman.max_shader_engines,
		933	CAYMAN_MAX_BACKENDS, disabled_rb_mask);
3764	Serge	934	}
2997	Serge	935	WREG32(GB_BACKEND_MAP, tmp);
1990	serge	936
2997	Serge	937	cgts_tcc_disable = 0xffff0000;
		938	for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
		939	cgts_tcc_disable &= ~(1 << (16 + i));
1990	serge	940	WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
		941	WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
		942	WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
		943	WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
		944
		945	/* reprogram the shader complex */
		946	cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
		947	for (i = 0; i < 16; i++)
		948	WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
		949	WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
		950
		951	/* set HW defaults for 3D engine */
		952	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) \| MEQ2_START(0x60));
		953
		954	sx_debug_1 = RREG32(SX_DEBUG_1);
		955	sx_debug_1 \|= ENABLE_NEW_SMX_ADDRESS;
		956	WREG32(SX_DEBUG_1, sx_debug_1);
		957
		958	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
		959	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
		960	smx_dc_ctl0 \|= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
		961	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
		962
		963	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) \| CRC_SIMD_ID_WADDR_DISABLE);
		964
		965	/* need to be explicitly zero-ed */
		966	WREG32(VGT_OFFCHIP_LDS_BASE, 0);
		967	WREG32(SQ_LSTMP_RING_BASE, 0);
		968	WREG32(SQ_HSTMP_RING_BASE, 0);
		969	WREG32(SQ_ESTMP_RING_BASE, 0);
		970	WREG32(SQ_GSTMP_RING_BASE, 0);
		971	WREG32(SQ_VSTMP_RING_BASE, 0);
		972	WREG32(SQ_PSTMP_RING_BASE, 0);
		973
		974	WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
		975
		976	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) \|
		977	POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) \|
		978	SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
		979
		980	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) \|
		981	SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) \|
		982	SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
		983
		984
		985	WREG32(VGT_NUM_INSTANCES, 1);
		986
		987	WREG32(CP_PERFMON_CNTL, 0);
		988
		989	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) \|
		990	FETCH_FIFO_HIWATER(0x4) \|
		991	DONE_FIFO_HIWATER(0xe0) \|
		992	ALU_UPDATE_FIFO_HIWATER(0x8)));
		993
		994	WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
		995	WREG32(SQ_CONFIG, (VC_ENABLE \|
		996	EXPORT_SRC_C \|
		997	GFX_PRIO(0) \|
		998	CS1_PRIO(0) \|
		999	CS2_PRIO(1)));
		1000	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
		1001
		1002	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) \|
		1003	FORCE_EOV_MAX_REZ_CNT(255)));
		1004
		1005	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) \|
		1006	AUTO_INVLD_EN(ES_AND_GS_AUTO));
		1007
		1008	WREG32(VGT_GS_VERTEX_REUSE, 16);
		1009	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		1010
		1011	WREG32(CB_PERF_CTR0_SEL_0, 0);
		1012	WREG32(CB_PERF_CTR0_SEL_1, 0);
		1013	WREG32(CB_PERF_CTR1_SEL_0, 0);
		1014	WREG32(CB_PERF_CTR1_SEL_1, 0);
		1015	WREG32(CB_PERF_CTR2_SEL_0, 0);
		1016	WREG32(CB_PERF_CTR2_SEL_1, 0);
		1017	WREG32(CB_PERF_CTR3_SEL_0, 0);
		1018	WREG32(CB_PERF_CTR3_SEL_1, 0);
		1019
		1020	tmp = RREG32(HDP_MISC_CNTL);
		1021	tmp \|= HDP_FLUSH_INVALIDATE_CACHE;
		1022	WREG32(HDP_MISC_CNTL, tmp);
		1023
		1024	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
		1025	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
		1026
		1027	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA \| NUM_CLIP_SEQ(3));
		1028
		1029	udelay(50);
		1030	}
		1031
		1032	/*
		1033	* GART
		1034	*/
		1035	void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
		1036	{
		1037	/* flush hdp cache */
		1038	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		1039
		1040	/* bits 0-7 are the VM contexts0-7 */
		1041	WREG32(VM_INVALIDATE_REQUEST, 1);
		1042	}
		1043
2997	Serge	1044	static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1990	serge	1045	{
2997	Serge	1046	int i, r;
1990	serge	1047
2997	Serge	1048	if (rdev->gart.robj == NULL) {
1990	serge	1049	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		1050	return -EINVAL;
		1051	}
		1052	r = radeon_gart_table_vram_pin(rdev);
		1053	if (r)
		1054	return r;
		1055	radeon_gart_restore(rdev);
		1056	/* Setup TLB control */
2997	Serge	1057	WREG32(MC_VM_MX_L1_TLB_CNTL,
		1058	(0xA << 7) \|
		1059	ENABLE_L1_TLB \|
1990	serge	1060	ENABLE_L1_FRAGMENT_PROCESSING \|
		1061	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
2997	Serge	1062	ENABLE_ADVANCED_DRIVER_MODEL \|
1990	serge	1063	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
		1064	/* Setup L2 cache */
		1065	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \|
		1066	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1067	ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE \|
		1068	EFFECTIVE_L2_QUEUE_SIZE(7) \|
		1069	CONTEXT1_IDENTITY_ACCESS_MODE(1));
		1070	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS \| INVALIDATE_L2_CACHE);
		1071	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY \|
		1072	L2_CACHE_BIGK_FRAGMENT_SIZE(6));
		1073	/* setup context0 */
		1074	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		1075	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		1076	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		1077	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		1078	(u32)(rdev->dummy_page.addr >> 12));
		1079	WREG32(VM_CONTEXT0_CNTL2, 0);
		1080	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		1081	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2997	Serge	1082
		1083	WREG32(0x15D4, 0);
		1084	WREG32(0x15D8, 0);
		1085	WREG32(0x15DC, 0);
		1086
		1087	/* empty context1-7 */
		1088	/* Assign the pt base to something valid for now; the pts used for
		1089	* the VMs are determined by the application and setup and assigned
		1090	* on the fly in the vm part of radeon_gart.c
		1091	*/
		1092	for (i = 1; i < 8; i++) {
		1093	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
		1094	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
		1095	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
		1096	rdev->gart.table_addr >> 12);
		1097	}
		1098
		1099	/* enable context1-7 */
		1100	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
		1101	(u32)(rdev->dummy_page.addr >> 12));
3192	Serge	1102	WREG32(VM_CONTEXT1_CNTL2, 4);
2997	Serge	1103	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(1) \|
3192	Serge	1104	RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1105	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT \|
		1106	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1107	DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT \|
		1108	PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1109	PDE0_PROTECTION_FAULT_ENABLE_DEFAULT \|
		1110	VALID_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1111	VALID_PROTECTION_FAULT_ENABLE_DEFAULT \|
		1112	READ_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1113	READ_PROTECTION_FAULT_ENABLE_DEFAULT \|
		1114	WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT \|
		1115	WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1990	serge	1116
		1117	cayman_pcie_gart_tlb_flush(rdev);
2997	Serge	1118	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		1119	(unsigned)(rdev->mc.gtt_size >> 20),
		1120	(unsigned long long)rdev->gart.table_addr);
1990	serge	1121	rdev->gart.ready = true;
		1122	return 0;
		1123	}
		1124
2997	Serge	1125	static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1990	serge	1126	{
		1127	/* Disable all tables */
		1128	WREG32(VM_CONTEXT0_CNTL, 0);
		1129	WREG32(VM_CONTEXT1_CNTL, 0);
		1130	/* Setup TLB control */
		1131	WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING \|
		1132	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		1133	SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
		1134	/* Setup L2 cache */
		1135	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		1136	ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE \|
		1137	EFFECTIVE_L2_QUEUE_SIZE(7) \|
		1138	CONTEXT1_IDENTITY_ACCESS_MODE(1));
		1139	WREG32(VM_L2_CNTL2, 0);
		1140	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY \|
		1141	L2_CACHE_BIGK_FRAGMENT_SIZE(6));
2997	Serge	1142	radeon_gart_table_vram_unpin(rdev);
1990	serge	1143	}
		1144
2997	Serge	1145	void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
		1146	int ring, u32 cp_int_cntl)
		1147	{
		1148	u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1990	serge	1149
2997	Serge	1150	WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl \| (ring & 3));
		1151	WREG32(CP_INT_CNTL, cp_int_cntl);
		1152	}
		1153
1990	serge	1154	/*
		1155	* CP.
		1156	*/
2997	Serge	1157	void cayman_fence_ring_emit(struct radeon_device *rdev,
		1158	struct radeon_fence *fence)
		1159	{
		1160	struct radeon_ring *ring = &rdev->ring[fence->ring];
		1161	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		1162
		1163	/* flush read cache over gart for this vmid */
		1164	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1165	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
		1166	radeon_ring_write(ring, 0);
		1167	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		1168	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA \| PACKET3_SH_ACTION_ENA);
		1169	radeon_ring_write(ring, 0xFFFFFFFF);
		1170	radeon_ring_write(ring, 0);
		1171	radeon_ring_write(ring, 10); /* poll interval */
		1172	/* EVENT_WRITE_EOP - flush caches, send int */
		1173	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		1174	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) \| EVENT_INDEX(5));
		1175	radeon_ring_write(ring, addr & 0xffffffff);
		1176	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		1177	radeon_ring_write(ring, fence->seq);
		1178	radeon_ring_write(ring, 0);
		1179	}
		1180
		1181	void cayman_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		1182	{
		1183	struct radeon_ring *ring = &rdev->ring[ib->ring];
		1184
		1185	/* set to DX10/11 mode */
		1186	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
		1187	radeon_ring_write(ring, 1);
		1188
		1189	if (ring->rptr_save_reg) {
		1190	uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
		1191	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3764	Serge	1192	radeon_ring_write(ring, ((ring->rptr_save_reg -
2997	Serge	1193	PACKET3_SET_CONFIG_REG_START) >> 2));
		1194	radeon_ring_write(ring, next_rptr);
		1195	}
		1196
		1197	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		1198	radeon_ring_write(ring,
		1199	#ifdef __BIG_ENDIAN
		1200	(2 << 0) \|
		1201	#endif
		1202	(ib->gpu_addr & 0xFFFFFFFC));
		1203	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3764	Serge	1204	radeon_ring_write(ring, ib->length_dw \|
2997	Serge	1205	(ib->vm ? (ib->vm->id << 24) : 0));
		1206
		1207	/* flush read cache over gart for this vmid */
		1208	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1209	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
		1210	radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
		1211	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		1212	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA \| PACKET3_SH_ACTION_ENA);
		1213	radeon_ring_write(ring, 0xFFFFFFFF);
		1214	radeon_ring_write(ring, 0);
		1215	radeon_ring_write(ring, 10); /* poll interval */
		1216	}
		1217
3764	Serge	1218	void cayman_uvd_semaphore_emit(struct radeon_device *rdev,
		1219	struct radeon_ring *ring,
		1220	struct radeon_semaphore *semaphore,
		1221	bool emit_wait)
		1222	{
		1223	uint64_t addr = semaphore->gpu_addr;
		1224
		1225	radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0));
		1226	radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF);
		1227
		1228	radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0));
		1229	radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF);
		1230
		1231	radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0));
		1232	radeon_ring_write(ring, 0x80 \| (emit_wait ? 1 : 0));
		1233	}
		1234
1990	serge	1235	static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
		1236	{
		1237	if (enable)
		1238	WREG32(CP_ME_CNTL, 0);
		1239	else {
3192	Serge	1240	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1990	serge	1241	WREG32(CP_ME_CNTL, (CP_ME_HALT \| CP_PFP_HALT));
		1242	WREG32(SCRATCH_UMSK, 0);
3192	Serge	1243	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1990	serge	1244	}
		1245	}
		1246
		1247	static int cayman_cp_load_microcode(struct radeon_device *rdev)
		1248	{
		1249	const __be32 *fw_data;
		1250	int i;
		1251
		1252	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1253	return -EINVAL;
		1254
		1255	cayman_cp_enable(rdev, false);
		1256
		1257	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1258	WREG32(CP_PFP_UCODE_ADDR, 0);
		1259	for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
		1260	WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
		1261	WREG32(CP_PFP_UCODE_ADDR, 0);
		1262
		1263	fw_data = (const __be32 *)rdev->me_fw->data;
		1264	WREG32(CP_ME_RAM_WADDR, 0);
		1265	for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
		1266	WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
		1267
		1268	WREG32(CP_PFP_UCODE_ADDR, 0);
		1269	WREG32(CP_ME_RAM_WADDR, 0);
		1270	WREG32(CP_ME_RAM_RADDR, 0);
		1271	return 0;
		1272	}
		1273
		1274	static int cayman_cp_start(struct radeon_device *rdev)
		1275	{
2997	Serge	1276	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1990	serge	1277	int r, i;
		1278
2997	Serge	1279	r = radeon_ring_lock(rdev, ring, 7);
1990	serge	1280	if (r) {
		1281	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1282	return r;
		1283	}
2997	Serge	1284	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1285	radeon_ring_write(ring, 0x1);
		1286	radeon_ring_write(ring, 0x0);
		1287	radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
		1288	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1289	radeon_ring_write(ring, 0);
		1290	radeon_ring_write(ring, 0);
		1291	radeon_ring_unlock_commit(rdev, ring);
1990	serge	1292
		1293	cayman_cp_enable(rdev, true);
		1294
2997	Serge	1295	r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1990	serge	1296	if (r) {
		1297	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1298	return r;
		1299	}
		1300
		1301	/* setup clear context state */
2997	Serge	1302	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1303	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1990	serge	1304
		1305	for (i = 0; i < cayman_default_size; i++)
2997	Serge	1306	radeon_ring_write(ring, cayman_default_state[i]);
1990	serge	1307
2997	Serge	1308	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
		1309	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1990	serge	1310
		1311	/* set clear context state */
2997	Serge	1312	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
		1313	radeon_ring_write(ring, 0);
1990	serge	1314
		1315	/* SQ_VTX_BASE_VTX_LOC */
2997	Serge	1316	radeon_ring_write(ring, 0xc0026f00);
		1317	radeon_ring_write(ring, 0x00000000);
		1318	radeon_ring_write(ring, 0x00000000);
		1319	radeon_ring_write(ring, 0x00000000);
1990	serge	1320
		1321	/* Clear consts */
2997	Serge	1322	radeon_ring_write(ring, 0xc0036f00);
		1323	radeon_ring_write(ring, 0x00000bc4);
		1324	radeon_ring_write(ring, 0xffffffff);
		1325	radeon_ring_write(ring, 0xffffffff);
		1326	radeon_ring_write(ring, 0xffffffff);
1990	serge	1327
2997	Serge	1328	radeon_ring_write(ring, 0xc0026900);
		1329	radeon_ring_write(ring, 0x00000316);
		1330	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
		1331	radeon_ring_write(ring, 0x00000010); /* */
1990	serge	1332
2997	Serge	1333	radeon_ring_unlock_commit(rdev, ring);
1990	serge	1334
		1335	/* XXX init other rings */
		1336
		1337	return 0;
		1338	}
		1339
		1340
2997	Serge	1341	static int cayman_cp_resume(struct radeon_device *rdev)
1990	serge	1342	{
2997	Serge	1343	static const int ridx[] = {
		1344	RADEON_RING_TYPE_GFX_INDEX,
		1345	CAYMAN_RING_TYPE_CP1_INDEX,
		1346	CAYMAN_RING_TYPE_CP2_INDEX
		1347	};
		1348	static const unsigned cp_rb_cntl[] = {
		1349	CP_RB0_CNTL,
		1350	CP_RB1_CNTL,
		1351	CP_RB2_CNTL,
		1352	};
		1353	static const unsigned cp_rb_rptr_addr[] = {
		1354	CP_RB0_RPTR_ADDR,
		1355	CP_RB1_RPTR_ADDR,
		1356	CP_RB2_RPTR_ADDR
		1357	};
		1358	static const unsigned cp_rb_rptr_addr_hi[] = {
		1359	CP_RB0_RPTR_ADDR_HI,
		1360	CP_RB1_RPTR_ADDR_HI,
		1361	CP_RB2_RPTR_ADDR_HI
		1362	};
		1363	static const unsigned cp_rb_base[] = {
		1364	CP_RB0_BASE,
		1365	CP_RB1_BASE,
		1366	CP_RB2_BASE
		1367	};
		1368	struct radeon_ring *ring;
		1369	int i, r;
1990	serge	1370
		1371	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
		1372	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP \|
		1373	SOFT_RESET_PA \|
		1374	SOFT_RESET_SH \|
		1375	SOFT_RESET_VGT \|
2160	serge	1376	SOFT_RESET_SPI \|
1990	serge	1377	SOFT_RESET_SX));
		1378	RREG32(GRBM_SOFT_RESET);
		1379	mdelay(15);
		1380	WREG32(GRBM_SOFT_RESET, 0);
		1381	RREG32(GRBM_SOFT_RESET);
		1382
2997	Serge	1383	WREG32(CP_SEM_WAIT_TIMER, 0x0);
		1384	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1990	serge	1385
		1386	/* Set the write pointer delay */
		1387	WREG32(CP_RB_WPTR_DELAY, 0);
		1388
		1389	WREG32(CP_DEBUG, (1 << 27));
		1390
3120	serge	1391	/* set the wb address whether it's enabled or not */
1990	serge	1392	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1393	WREG32(SCRATCH_UMSK, 0xff);
		1394
2997	Serge	1395	for (i = 0; i < 3; ++i) {
		1396	uint32_t rb_cntl;
		1397	uint64_t addr;
1990	serge	1398
		1399	/* Set ring buffer size */
2997	Serge	1400	ring = &rdev->ring[ridx[i]];
		1401	rb_cntl = drm_order(ring->ring_size / 8);
		1402	rb_cntl \|= drm_order(RADEON_GPU_PAGE_SIZE/8) << 8;
1990	serge	1403	#ifdef __BIG_ENDIAN
2997	Serge	1404	rb_cntl \|= BUF_SWAP_32BIT;
1990	serge	1405	#endif
2997	Serge	1406	WREG32(cp_rb_cntl[i], rb_cntl);
1990	serge	1407
3120	serge	1408	/* set the wb address whether it's enabled or not */
2997	Serge	1409	addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
		1410	WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
		1411	WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
		1412	}
1990	serge	1413
2997	Serge	1414	/* set the rb base addr, this causes an internal reset of ALL rings */
		1415	for (i = 0; i < 3; ++i) {
		1416	ring = &rdev->ring[ridx[i]];
		1417	WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
		1418	}
1990	serge	1419
2997	Serge	1420	for (i = 0; i < 3; ++i) {
1990	serge	1421	/* Initialize the ring buffer's read and write pointers */
2997	Serge	1422	ring = &rdev->ring[ridx[i]];
		1423	WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1990	serge	1424
2997	Serge	1425	ring->rptr = ring->wptr = 0;
		1426	WREG32(ring->rptr_reg, ring->rptr);
		1427	WREG32(ring->wptr_reg, ring->wptr);
1990	serge	1428
		1429	mdelay(1);
2997	Serge	1430	WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
		1431	}
1990	serge	1432
		1433	/* start the rings */
		1434	cayman_cp_start(rdev);
2997	Serge	1435	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
		1436	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		1437	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1990	serge	1438	/* this only test cp0 */
2997	Serge	1439	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1990	serge	1440	if (r) {
2997	Serge	1441	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
		1442	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
		1443	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1990	serge	1444	return r;
		1445	}
		1446
		1447	return 0;
		1448	}
		1449
3192	Serge	1450	/*
		1451	* DMA
		1452	* Starting with R600, the GPU has an asynchronous
		1453	* DMA engine. The programming model is very similar
		1454	* to the 3D engine (ring buffer, IBs, etc.), but the
		1455	* DMA controller has it's own packet format that is
		1456	* different form the PM4 format used by the 3D engine.
		1457	* It supports copying data, writing embedded data,
		1458	* solid fills, and a number of other things. It also
		1459	* has support for tiling/detiling of buffers.
		1460	* Cayman and newer support two asynchronous DMA engines.
		1461	*/
		1462	/**
		1463	* cayman_dma_ring_ib_execute - Schedule an IB on the DMA engine
		1464	*
		1465	* @rdev: radeon_device pointer
		1466	* @ib: IB object to schedule
		1467	*
		1468	* Schedule an IB in the DMA ring (cayman-SI).
		1469	*/
		1470	void cayman_dma_ring_ib_execute(struct radeon_device *rdev,
		1471	struct radeon_ib *ib)
1990	serge	1472	{
3192	Serge	1473	struct radeon_ring *ring = &rdev->ring[ib->ring];
		1474
		1475	if (rdev->wb.enabled) {
		1476	u32 next_rptr = ring->wptr + 4;
		1477	while ((next_rptr & 7) != 5)
		1478	next_rptr++;
		1479	next_rptr += 3;
		1480	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
		1481	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		1482	radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
		1483	radeon_ring_write(ring, next_rptr);
		1484	}
		1485
		1486	/* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
		1487	* Pad as necessary with NOPs.
		1488	*/
		1489	while ((ring->wptr & 7) != 5)
		1490	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
		1491	radeon_ring_write(ring, DMA_IB_PACKET(DMA_PACKET_INDIRECT_BUFFER, ib->vm ? ib->vm->id : 0, 0));
		1492	radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
		1493	radeon_ring_write(ring, (ib->length_dw << 12) \| (upper_32_bits(ib->gpu_addr) & 0xFF));
		1494
		1495	}
		1496
		1497	/**
		1498	* cayman_dma_stop - stop the async dma engines
		1499	*
		1500	* @rdev: radeon_device pointer
		1501	*
		1502	* Stop the async dma engines (cayman-SI).
		1503	*/
		1504	void cayman_dma_stop(struct radeon_device *rdev)
		1505	{
		1506	u32 rb_cntl;
		1507
		1508	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
		1509
		1510	/* dma0 */
		1511	rb_cntl = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
		1512	rb_cntl &= ~DMA_RB_ENABLE;
		1513	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, rb_cntl);
		1514
		1515	/* dma1 */
		1516	rb_cntl = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
		1517	rb_cntl &= ~DMA_RB_ENABLE;
		1518	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, rb_cntl);
		1519
		1520	rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
		1521	rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX].ready = false;
		1522	}
		1523
		1524	/**
		1525	* cayman_dma_resume - setup and start the async dma engines
		1526	*
		1527	* @rdev: radeon_device pointer
		1528	*
		1529	* Set up the DMA ring buffers and enable them. (cayman-SI).
		1530	* Returns 0 for success, error for failure.
		1531	*/
		1532	int cayman_dma_resume(struct radeon_device *rdev)
		1533	{
		1534	struct radeon_ring *ring;
3764	Serge	1535	u32 rb_cntl, dma_cntl, ib_cntl;
3192	Serge	1536	u32 rb_bufsz;
		1537	u32 reg_offset, wb_offset;
		1538	int i, r;
		1539
		1540	/* Reset dma */
		1541	WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA \| SOFT_RESET_DMA1);
		1542	RREG32(SRBM_SOFT_RESET);
		1543	udelay(50);
		1544	WREG32(SRBM_SOFT_RESET, 0);
		1545
		1546	for (i = 0; i < 2; i++) {
		1547	if (i == 0) {
		1548	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		1549	reg_offset = DMA0_REGISTER_OFFSET;
		1550	wb_offset = R600_WB_DMA_RPTR_OFFSET;
		1551	} else {
		1552	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		1553	reg_offset = DMA1_REGISTER_OFFSET;
		1554	wb_offset = CAYMAN_WB_DMA1_RPTR_OFFSET;
		1555	}
		1556
		1557	WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL + reg_offset, 0);
		1558	WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL + reg_offset, 0);
		1559
		1560	/* Set ring buffer size in dwords */
		1561	rb_bufsz = drm_order(ring->ring_size / 4);
		1562	rb_cntl = rb_bufsz << 1;
		1563	#ifdef __BIG_ENDIAN
		1564	rb_cntl \|= DMA_RB_SWAP_ENABLE \| DMA_RPTR_WRITEBACK_SWAP_ENABLE;
		1565	#endif
		1566	WREG32(DMA_RB_CNTL + reg_offset, rb_cntl);
		1567
		1568	/* Initialize the ring buffer's read and write pointers */
		1569	WREG32(DMA_RB_RPTR + reg_offset, 0);
		1570	WREG32(DMA_RB_WPTR + reg_offset, 0);
		1571
		1572	/* set the wb address whether it's enabled or not */
		1573	WREG32(DMA_RB_RPTR_ADDR_HI + reg_offset,
		1574	upper_32_bits(rdev->wb.gpu_addr + wb_offset) & 0xFF);
		1575	WREG32(DMA_RB_RPTR_ADDR_LO + reg_offset,
		1576	((rdev->wb.gpu_addr + wb_offset) & 0xFFFFFFFC));
		1577
		1578	if (rdev->wb.enabled)
		1579	rb_cntl \|= DMA_RPTR_WRITEBACK_ENABLE;
		1580
		1581	WREG32(DMA_RB_BASE + reg_offset, ring->gpu_addr >> 8);
		1582
		1583	/* enable DMA IBs */
3764	Serge	1584	ib_cntl = DMA_IB_ENABLE \| CMD_VMID_FORCE;
		1585	#ifdef __BIG_ENDIAN
		1586	ib_cntl \|= DMA_IB_SWAP_ENABLE;
		1587	#endif
		1588	WREG32(DMA_IB_CNTL + reg_offset, ib_cntl);
3192	Serge	1589
		1590	dma_cntl = RREG32(DMA_CNTL + reg_offset);
		1591	dma_cntl &= ~CTXEMPTY_INT_ENABLE;
		1592	WREG32(DMA_CNTL + reg_offset, dma_cntl);
		1593
		1594	ring->wptr = 0;
		1595	WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2);
		1596
		1597	ring->rptr = RREG32(DMA_RB_RPTR + reg_offset) >> 2;
		1598
		1599	WREG32(DMA_RB_CNTL + reg_offset, rb_cntl \| DMA_RB_ENABLE);
		1600
		1601	ring->ready = true;
		1602
		1603	r = radeon_ring_test(rdev, ring->idx, ring);
		1604	if (r) {
		1605	ring->ready = false;
		1606	return r;
		1607	}
		1608	}
		1609
		1610	radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
		1611
		1612	return 0;
		1613	}
		1614
		1615	/**
		1616	* cayman_dma_fini - tear down the async dma engines
		1617	*
		1618	* @rdev: radeon_device pointer
		1619	*
		1620	* Stop the async dma engines and free the rings (cayman-SI).
		1621	*/
		1622	void cayman_dma_fini(struct radeon_device *rdev)
		1623	{
		1624	cayman_dma_stop(rdev);
		1625	radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
		1626	radeon_ring_fini(rdev, &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]);
		1627	}
		1628
3764	Serge	1629	static u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
3192	Serge	1630	{
3764	Serge	1631	u32 reset_mask = 0;
		1632	u32 tmp;
1990	serge	1633
3764	Serge	1634	/* GRBM_STATUS */
		1635	tmp = RREG32(GRBM_STATUS);
		1636	if (tmp & (PA_BUSY \| SC_BUSY \|
		1637	SH_BUSY \| SX_BUSY \|
		1638	TA_BUSY \| VGT_BUSY \|
		1639	DB_BUSY \| CB_BUSY \|
		1640	GDS_BUSY \| SPI_BUSY \|
		1641	IA_BUSY \| IA_BUSY_NO_DMA))
		1642	reset_mask \|= RADEON_RESET_GFX;
1990	serge	1643
3764	Serge	1644	if (tmp & (CF_RQ_PENDING \| PF_RQ_PENDING \|
		1645	CP_BUSY \| CP_COHERENCY_BUSY))
		1646	reset_mask \|= RADEON_RESET_CP;
2997	Serge	1647
3764	Serge	1648	if (tmp & GRBM_EE_BUSY)
		1649	reset_mask \|= RADEON_RESET_GRBM \| RADEON_RESET_GFX \| RADEON_RESET_CP;
1990	serge	1650
3764	Serge	1651	/* DMA_STATUS_REG 0 */
		1652	tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
		1653	if (!(tmp & DMA_IDLE))
		1654	reset_mask \|= RADEON_RESET_DMA;
1990	serge	1655
3764	Serge	1656	/* DMA_STATUS_REG 1 */
		1657	tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
		1658	if (!(tmp & DMA_IDLE))
		1659	reset_mask \|= RADEON_RESET_DMA1;
2997	Serge	1660
3764	Serge	1661	/* SRBM_STATUS2 */
		1662	tmp = RREG32(SRBM_STATUS2);
		1663	if (tmp & DMA_BUSY)
		1664	reset_mask \|= RADEON_RESET_DMA;
3192	Serge	1665
3764	Serge	1666	if (tmp & DMA1_BUSY)
		1667	reset_mask \|= RADEON_RESET_DMA1;
3192	Serge	1668
3764	Serge	1669	/* SRBM_STATUS */
		1670	tmp = RREG32(SRBM_STATUS);
		1671	if (tmp & (RLC_RQ_PENDING \| RLC_BUSY))
		1672	reset_mask \|= RADEON_RESET_RLC;
3192	Serge	1673
3764	Serge	1674	if (tmp & IH_BUSY)
		1675	reset_mask \|= RADEON_RESET_IH;
3192	Serge	1676
3764	Serge	1677	if (tmp & SEM_BUSY)
		1678	reset_mask \|= RADEON_RESET_SEM;
3192	Serge	1679
3764	Serge	1680	if (tmp & GRBM_RQ_PENDING)
		1681	reset_mask \|= RADEON_RESET_GRBM;
3192	Serge	1682
3764	Serge	1683	if (tmp & VMC_BUSY)
		1684	reset_mask \|= RADEON_RESET_VMC;
3192	Serge	1685
3764	Serge	1686	if (tmp & (MCB_BUSY \| MCB_NON_DISPLAY_BUSY \|
		1687	MCC_BUSY \| MCD_BUSY))
		1688	reset_mask \|= RADEON_RESET_MC;
3192	Serge	1689
3764	Serge	1690	if (evergreen_is_display_hung(rdev))
		1691	reset_mask \|= RADEON_RESET_DISPLAY;
3192	Serge	1692
3764	Serge	1693	/* VM_L2_STATUS */
		1694	tmp = RREG32(VM_L2_STATUS);
		1695	if (tmp & L2_BUSY)
		1696	reset_mask \|= RADEON_RESET_VMC;
		1697
		1698	/* Skip MC reset as it's mostly likely not hung, just busy */
		1699	if (reset_mask & RADEON_RESET_MC) {
		1700	DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
		1701	reset_mask &= ~RADEON_RESET_MC;
		1702	}
		1703
		1704	return reset_mask;
3192	Serge	1705	}
		1706
3764	Serge	1707	static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3192	Serge	1708	{
		1709	struct evergreen_mc_save save;
3764	Serge	1710	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
		1711	u32 tmp;
3192	Serge	1712
		1713	if (reset_mask == 0)
3764	Serge	1714	return;
3192	Serge	1715
		1716	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
		1717
3764	Serge	1718	evergreen_print_gpu_status_regs(rdev);
3192	Serge	1719	dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_ADDR 0x%08X\n",
		1720	RREG32(0x14F8));
		1721	dev_info(rdev->dev, " VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
		1722	RREG32(0x14D8));
		1723	dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
		1724	RREG32(0x14FC));
		1725	dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
		1726	RREG32(0x14DC));
		1727
3764	Serge	1728	/* Disable CP parsing/prefetching */
		1729	WREG32(CP_ME_CNTL, CP_ME_HALT \| CP_PFP_HALT);
		1730
		1731	if (reset_mask & RADEON_RESET_DMA) {
		1732	/* dma0 */
		1733	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
		1734	tmp &= ~DMA_RB_ENABLE;
		1735	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
		1736	}
		1737
		1738	if (reset_mask & RADEON_RESET_DMA1) {
		1739	/* dma1 */
		1740	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
		1741	tmp &= ~DMA_RB_ENABLE;
		1742	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
		1743	}
		1744
		1745	udelay(50);
		1746
3192	Serge	1747	evergreen_mc_stop(rdev, &save);
		1748	if (evergreen_mc_wait_for_idle(rdev)) {
		1749	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1750	}
		1751
3764	Serge	1752	if (reset_mask & (RADEON_RESET_GFX \| RADEON_RESET_COMPUTE)) {
		1753	grbm_soft_reset = SOFT_RESET_CB \|
		1754	SOFT_RESET_DB \|
		1755	SOFT_RESET_GDS \|
		1756	SOFT_RESET_PA \|
		1757	SOFT_RESET_SC \|
		1758	SOFT_RESET_SPI \|
		1759	SOFT_RESET_SH \|
		1760	SOFT_RESET_SX \|
		1761	SOFT_RESET_TC \|
		1762	SOFT_RESET_TA \|
		1763	SOFT_RESET_VGT \|
		1764	SOFT_RESET_IA;
		1765	}
3192	Serge	1766
3764	Serge	1767	if (reset_mask & RADEON_RESET_CP) {
		1768	grbm_soft_reset \|= SOFT_RESET_CP \| SOFT_RESET_VGT;
		1769
		1770	srbm_soft_reset \|= SOFT_RESET_GRBM;
		1771	}
		1772
3192	Serge	1773	if (reset_mask & RADEON_RESET_DMA)
3764	Serge	1774	srbm_soft_reset \|= SOFT_RESET_DMA;
3192	Serge	1775
3764	Serge	1776	if (reset_mask & RADEON_RESET_DMA1)
		1777	srbm_soft_reset \|= SOFT_RESET_DMA1;
		1778
		1779	if (reset_mask & RADEON_RESET_DISPLAY)
		1780	srbm_soft_reset \|= SOFT_RESET_DC;
		1781
		1782	if (reset_mask & RADEON_RESET_RLC)
		1783	srbm_soft_reset \|= SOFT_RESET_RLC;
		1784
		1785	if (reset_mask & RADEON_RESET_SEM)
		1786	srbm_soft_reset \|= SOFT_RESET_SEM;
		1787
		1788	if (reset_mask & RADEON_RESET_IH)
		1789	srbm_soft_reset \|= SOFT_RESET_IH;
		1790
		1791	if (reset_mask & RADEON_RESET_GRBM)
		1792	srbm_soft_reset \|= SOFT_RESET_GRBM;
		1793
		1794	if (reset_mask & RADEON_RESET_VMC)
		1795	srbm_soft_reset \|= SOFT_RESET_VMC;
		1796
		1797	if (!(rdev->flags & RADEON_IS_IGP)) {
		1798	if (reset_mask & RADEON_RESET_MC)
		1799	srbm_soft_reset \|= SOFT_RESET_MC;
		1800	}
		1801
		1802	if (grbm_soft_reset) {
		1803	tmp = RREG32(GRBM_SOFT_RESET);
		1804	tmp \|= grbm_soft_reset;
		1805	dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
		1806	WREG32(GRBM_SOFT_RESET, tmp);
		1807	tmp = RREG32(GRBM_SOFT_RESET);
		1808
		1809	udelay(50);
		1810
		1811	tmp &= ~grbm_soft_reset;
		1812	WREG32(GRBM_SOFT_RESET, tmp);
		1813	tmp = RREG32(GRBM_SOFT_RESET);
		1814	}
		1815
		1816	if (srbm_soft_reset) {
		1817	tmp = RREG32(SRBM_SOFT_RESET);
		1818	tmp \|= srbm_soft_reset;
		1819	dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
		1820	WREG32(SRBM_SOFT_RESET, tmp);
		1821	tmp = RREG32(SRBM_SOFT_RESET);
		1822
		1823	udelay(50);
		1824
		1825	tmp &= ~srbm_soft_reset;
		1826	WREG32(SRBM_SOFT_RESET, tmp);
		1827	tmp = RREG32(SRBM_SOFT_RESET);
		1828	}
		1829
3192	Serge	1830	/* Wait a little for things to settle down */
		1831	udelay(50);
		1832
1990	serge	1833	evergreen_mc_resume(rdev, &save);
3764	Serge	1834	udelay(50);
		1835
		1836	evergreen_print_gpu_status_regs(rdev);
1990	serge	1837	}
		1838
		1839	int cayman_asic_reset(struct radeon_device *rdev)
		1840	{
3764	Serge	1841	u32 reset_mask;
		1842
		1843	reset_mask = cayman_gpu_check_soft_reset(rdev);
		1844
		1845	if (reset_mask)
		1846	r600_set_bios_scratch_engine_hung(rdev, true);
		1847
		1848	cayman_gpu_soft_reset(rdev, reset_mask);
		1849
		1850	reset_mask = cayman_gpu_check_soft_reset(rdev);
		1851
		1852	if (!reset_mask)
		1853	r600_set_bios_scratch_engine_hung(rdev, false);
		1854
		1855	return 0;
		1856	}
		1857
		1858	/**
		1859	* cayman_gfx_is_lockup - Check if the GFX engine is locked up
		1860	*
		1861	* @rdev: radeon_device pointer
		1862	* @ring: radeon_ring structure holding ring information
		1863	*
		1864	* Check if the GFX engine is locked up.
		1865	* Returns true if the engine appears to be locked up, false if not.
		1866	*/
		1867	bool cayman_gfx_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
		1868	{
		1869	u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
		1870
		1871	if (!(reset_mask & (RADEON_RESET_GFX \|
3192	Serge	1872	RADEON_RESET_COMPUTE \|
3764	Serge	1873	RADEON_RESET_CP))) {
		1874	radeon_ring_lockup_update(ring);
		1875	return false;
		1876	}
		1877	/* force CP activities */
		1878	radeon_ring_force_activity(rdev, ring);
		1879	return radeon_ring_test_lockup(rdev, ring);
1990	serge	1880	}
		1881
3192	Serge	1882	/**
		1883	* cayman_dma_is_lockup - Check if the DMA engine is locked up
		1884	*
		1885	* @rdev: radeon_device pointer
		1886	* @ring: radeon_ring structure holding ring information
		1887	*
3764	Serge	1888	* Check if the async DMA engine is locked up.
3192	Serge	1889	* Returns true if the engine appears to be locked up, false if not.
		1890	*/
		1891	bool cayman_dma_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
		1892	{
3764	Serge	1893	u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
		1894	u32 mask;
3192	Serge	1895
		1896	if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3764	Serge	1897	mask = RADEON_RESET_DMA;
3192	Serge	1898	else
3764	Serge	1899	mask = RADEON_RESET_DMA1;
		1900
		1901	if (!(reset_mask & mask)) {
3192	Serge	1902	radeon_ring_lockup_update(ring);
		1903	return false;
		1904	}
		1905	/* force ring activities */
		1906	radeon_ring_force_activity(rdev, ring);
		1907	return radeon_ring_test_lockup(rdev, ring);
		1908	}
		1909
1990	serge	1910	static int cayman_startup(struct radeon_device *rdev)
		1911	{
2997	Serge	1912	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1990	serge	1913	int r;
		1914
2997	Serge	1915	/* enable pcie gen2 link */
		1916	evergreen_pcie_gen2_enable(rdev);
		1917
		1918	if (rdev->flags & RADEON_IS_IGP) {
		1919	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		1920	r = ni_init_microcode(rdev);
		1921	if (r) {
		1922	DRM_ERROR("Failed to load firmware!\n");
		1923	return r;
		1924	}
		1925	}
		1926	} else {
1990	serge	1927	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw \|\| !rdev->mc_fw) {
		1928	r = ni_init_microcode(rdev);
		1929	if (r) {
		1930	DRM_ERROR("Failed to load firmware!\n");
		1931	return r;
		1932	}
		1933	}
2997	Serge	1934
1990	serge	1935	r = ni_mc_load_microcode(rdev);
		1936	if (r) {
		1937	DRM_ERROR("Failed to load MC firmware!\n");
		1938	return r;
		1939	}
2997	Serge	1940	}
1990	serge	1941
2997	Serge	1942	r = r600_vram_scratch_init(rdev);
		1943	if (r)
		1944	return r;
		1945
1990	serge	1946	evergreen_mc_program(rdev);
		1947	r = cayman_pcie_gart_enable(rdev);
		1948	if (r)
		1949	return r;
		1950	cayman_gpu_init(rdev);
		1951
2005	serge	1952	r = evergreen_blit_init(rdev);
		1953	if (r) {
2997	Serge	1954	// r600_blit_fini(rdev);
		1955	rdev->asic->copy.copy = NULL;
2005	serge	1956	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		1957	}
1990	serge	1958
2997	Serge	1959	/* allocate rlc buffers */
		1960	if (rdev->flags & RADEON_IS_IGP) {
		1961	r = si_rlc_init(rdev);
		1962	if (r) {
		1963	DRM_ERROR("Failed to init rlc BOs!\n");
		1964	return r;
		1965	}
		1966	}
		1967
1990	serge	1968	/* allocate wb buffer */
2005	serge	1969	r = radeon_wb_init(rdev);
		1970	if (r)
		1971	return r;
1990	serge	1972
3192	Serge	1973	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
		1974	if (r) {
		1975	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		1976	return r;
		1977	}
		1978
3764	Serge	1979	// r = rv770_uvd_resume(rdev);
		1980	// if (!r) {
		1981	// r = radeon_fence_driver_start_ring(rdev,
		1982	// R600_RING_TYPE_UVD_INDEX);
		1983	// if (r)
		1984	// dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
		1985	// }
		1986	// if (r)
		1987	// rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
		1988
3192	Serge	1989	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
		1990	if (r) {
		1991	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		1992	return r;
		1993	}
		1994
		1995	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
		1996	if (r) {
		1997	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		1998	return r;
		1999	}
		2000
		2001	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
		2002	if (r) {
		2003	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		2004	return r;
		2005	}
		2006
		2007	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
		2008	if (r) {
		2009	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		2010	return r;
		2011	}
		2012
1990	serge	2013	/* Enable IRQ */
3764	Serge	2014	if (!rdev->irq.installed) {
		2015	r = radeon_irq_kms_init(rdev);
		2016	if (r)
		2017	return r;
		2018	}
		2019
2005	serge	2020	r = r600_irq_init(rdev);
		2021	if (r) {
		2022	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		2023	// radeon_irq_kms_fini(rdev);
		2024	return r;
		2025	}
		2026	evergreen_irq_set(rdev);
1990	serge	2027
2997	Serge	2028	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
		2029	CP_RB0_RPTR, CP_RB0_WPTR,
		2030	0, 0xfffff, RADEON_CP_PACKET2);
1990	serge	2031	if (r)
		2032	return r;
3192	Serge	2033
		2034	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		2035	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
		2036	DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
		2037	DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
		2038	2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
		2039	if (r)
		2040	return r;
		2041
		2042	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		2043	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
		2044	DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
		2045	DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
		2046	2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
		2047	if (r)
		2048	return r;
		2049
1990	serge	2050	r = cayman_cp_load_microcode(rdev);
		2051	if (r)
		2052	return r;
		2053	r = cayman_cp_resume(rdev);
		2054	if (r)
		2055	return r;
		2056
3192	Serge	2057	r = cayman_dma_resume(rdev);
		2058	if (r)
		2059	return r;
		2060
3764	Serge	2061	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
		2062	if (ring->ring_size) {
		2063	r = radeon_ring_init(rdev, ring, ring->ring_size,
		2064	R600_WB_UVD_RPTR_OFFSET,
		2065	UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
		2066	0, 0xfffff, RADEON_CP_PACKET2);
		2067	if (!r)
		2068	r = r600_uvd_init(rdev);
		2069	if (r)
		2070	DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
		2071	}
		2072
3192	Serge	2073	r = radeon_ib_pool_init(rdev);
		2074	if (r) {
		2075	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		2076	return r;
		2077	}
3764	Serge	2078
		2079	r = radeon_vm_manager_init(rdev);
		2080	if (r) {
		2081	dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
		2082	return r;
		2083	}
		2084
		2085
1990	serge	2086	return 0;
		2087	}
		2088
		2089
		2090
		2091
		2092
		2093	/* Plan is to move initialization in that function and use
		2094	* helper function so that radeon_device_init pretty much
		2095	* do nothing more than calling asic specific function. This
		2096	* should also allow to remove a bunch of callback function
		2097	* like vram_info.
		2098	*/
		2099	int cayman_init(struct radeon_device *rdev)
		2100	{
2997	Serge	2101	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1990	serge	2102	int r;
		2103
		2104	/* Read BIOS */
		2105	if (!radeon_get_bios(rdev)) {
		2106	if (ASIC_IS_AVIVO(rdev))
		2107	return -EINVAL;
		2108	}
		2109	/* Must be an ATOMBIOS */
		2110	if (!rdev->is_atom_bios) {
		2111	dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
		2112	return -EINVAL;
		2113	}
		2114	r = radeon_atombios_init(rdev);
		2115	if (r)
		2116	return r;
		2117
		2118	/* Post card if necessary */
		2119	if (!radeon_card_posted(rdev)) {
		2120	if (!rdev->bios) {
		2121	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		2122	return -EINVAL;
		2123	}
		2124	DRM_INFO("GPU not posted. posting now...\n");
		2125	atom_asic_init(rdev->mode_info.atom_context);
		2126	}
3764	Serge	2127	/* init golden registers */
		2128	ni_init_golden_registers(rdev);
1990	serge	2129	/* Initialize scratch registers */
		2130	r600_scratch_init(rdev);
		2131	/* Initialize surface registers */
		2132	radeon_surface_init(rdev);
		2133	/* Initialize clocks */
		2134	radeon_get_clock_info(rdev->ddev);
		2135	/* Fence driver */
2005	serge	2136	r = radeon_fence_driver_init(rdev);
		2137	if (r)
		2138	return r;
1990	serge	2139	/* initialize memory controller */
		2140	r = evergreen_mc_init(rdev);
		2141	if (r)
		2142	return r;
		2143	/* Memory manager */
		2144	r = radeon_bo_init(rdev);
		2145	if (r)
		2146	return r;
		2147
2997	Serge	2148	ring->ring_obj = NULL;
		2149	r600_ring_init(rdev, ring, 1024 * 1024);
1990	serge	2150
3192	Serge	2151	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		2152	ring->ring_obj = NULL;
		2153	r600_ring_init(rdev, ring, 64 * 1024);
		2154
		2155	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
		2156	ring->ring_obj = NULL;
		2157	r600_ring_init(rdev, ring, 64 * 1024);
		2158
3764	Serge	2159	// r = radeon_uvd_init(rdev);
		2160	// if (!r) {
		2161	// ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
		2162	// ring->ring_obj = NULL;
		2163	// r600_ring_init(rdev, ring, 4096);
		2164	// }
		2165
2005	serge	2166	rdev->ih.ring_obj = NULL;
		2167	r600_ih_ring_init(rdev, 64 * 1024);
1990	serge	2168
		2169	r = r600_pcie_gart_init(rdev);
		2170	if (r)
		2171	return r;
		2172
		2173	rdev->accel_working = true;
		2174	r = cayman_startup(rdev);
		2175	if (r) {
		2176	dev_err(rdev->dev, "disabling GPU acceleration\n");
		2177	rdev->accel_working = false;
		2178	}
		2179
		2180	/* Don't start up if the MC ucode is missing.
		2181	* The default clocks and voltages before the MC ucode
		2182	* is loaded are not suffient for advanced operations.
2997	Serge	2183	*
		2184	* We can skip this check for TN, because there is no MC
		2185	* ucode.
1990	serge	2186	*/
2997	Serge	2187	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
1990	serge	2188	DRM_ERROR("radeon: MC ucode required for NI+.\n");
		2189	return -EINVAL;
		2190	}
		2191
		2192	return 0;
		2193	}
		2194
2997	Serge	2195	/*
		2196	* vm
		2197	*/
		2198	int cayman_vm_init(struct radeon_device *rdev)
		2199	{
		2200	/* number of VMs */
		2201	rdev->vm_manager.nvm = 8;
		2202	/* base offset of vram pages */
		2203	if (rdev->flags & RADEON_IS_IGP) {
		2204	u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
		2205	tmp <<= 22;
		2206	rdev->vm_manager.vram_base_offset = tmp;
		2207	} else
		2208	rdev->vm_manager.vram_base_offset = 0;
		2209	return 0;
		2210	}
		2211
		2212	void cayman_vm_fini(struct radeon_device *rdev)
		2213	{
		2214	}
		2215
		2216	#define R600_ENTRY_VALID (1 << 0)
		2217	#define R600_PTE_SYSTEM (1 << 1)
		2218	#define R600_PTE_SNOOPED (1 << 2)
		2219	#define R600_PTE_READABLE (1 << 5)
		2220	#define R600_PTE_WRITEABLE (1 << 6)
		2221
		2222	uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags)
		2223	{
		2224	uint32_t r600_flags = 0;
		2225	r600_flags \|= (flags & RADEON_VM_PAGE_VALID) ? R600_ENTRY_VALID : 0;
		2226	r600_flags \|= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
		2227	r600_flags \|= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
		2228	if (flags & RADEON_VM_PAGE_SYSTEM) {
		2229	r600_flags \|= R600_PTE_SYSTEM;
		2230	r600_flags \|= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
		2231	}
		2232	return r600_flags;
		2233	}
		2234
		2235	/**
		2236	* cayman_vm_set_page - update the page tables using the CP
		2237	*
		2238	* @rdev: radeon_device pointer
3764	Serge	2239	* @ib: indirect buffer to fill with commands
2997	Serge	2240	* @pe: addr of the page entry
		2241	* @addr: dst addr to write into pe
		2242	* @count: number of page entries to update
		2243	* @incr: increase next addr by incr bytes
		2244	* @flags: access flags
		2245	*
3764	Serge	2246	* Update the page tables using the CP (cayman/TN).
2997	Serge	2247	*/
3764	Serge	2248	void cayman_vm_set_page(struct radeon_device *rdev,
		2249	struct radeon_ib *ib,
		2250	uint64_t pe,
2997	Serge	2251	uint64_t addr, unsigned count,
		2252	uint32_t incr, uint32_t flags)
		2253	{
		2254	uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
3192	Serge	2255	uint64_t value;
		2256	unsigned ndw;
2997	Serge	2257
3192	Serge	2258	if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
2997	Serge	2259	while (count) {
3192	Serge	2260	ndw = 1 + count * 2;
2997	Serge	2261	if (ndw > 0x3FFF)
		2262	ndw = 0x3FFF;
		2263
3764	Serge	2264	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_ME_WRITE, ndw);
		2265	ib->ptr[ib->length_dw++] = pe;
		2266	ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
2997	Serge	2267	for (; ndw > 1; ndw -= 2, --count, pe += 8) {
		2268	if (flags & RADEON_VM_PAGE_SYSTEM) {
		2269	value = radeon_vm_map_gart(rdev, addr);
		2270	value &= 0xFFFFFFFFFFFFF000ULL;
3192	Serge	2271	} else if (flags & RADEON_VM_PAGE_VALID) {
		2272	value = addr;
		2273	} else {
		2274	value = 0;
		2275	}
2997	Serge	2276	addr += incr;
3192	Serge	2277	value \|= r600_flags;
3764	Serge	2278	ib->ptr[ib->length_dw++] = value;
		2279	ib->ptr[ib->length_dw++] = upper_32_bits(value);
3192	Serge	2280	}
		2281	}
		2282	} else {
3764	Serge	2283	if ((flags & RADEON_VM_PAGE_SYSTEM) \|\|
		2284	(count == 1)) {
3192	Serge	2285	while (count) {
		2286	ndw = count * 2;
		2287	if (ndw > 0xFFFFE)
		2288	ndw = 0xFFFFE;
2997	Serge	2289
3192	Serge	2290	/* for non-physically contiguous pages (system) */
3764	Serge	2291	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, ndw);
		2292	ib->ptr[ib->length_dw++] = pe;
		2293	ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
3192	Serge	2294	for (; ndw > 0; ndw -= 2, --count, pe += 8) {
		2295	if (flags & RADEON_VM_PAGE_SYSTEM) {
		2296	value = radeon_vm_map_gart(rdev, addr);
		2297	value &= 0xFFFFFFFFFFFFF000ULL;
2997	Serge	2298	} else if (flags & RADEON_VM_PAGE_VALID) {
		2299	value = addr;
3192	Serge	2300	} else {
		2301	value = 0;
		2302	}
2997	Serge	2303	addr += incr;
		2304	value \|= r600_flags;
3764	Serge	2305	ib->ptr[ib->length_dw++] = value;
		2306	ib->ptr[ib->length_dw++] = upper_32_bits(value);
2997	Serge	2307	}
		2308	}
3764	Serge	2309	while (ib->length_dw & 0x7)
		2310	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0);
		2311	} else {
		2312	while (count) {
		2313	ndw = count * 2;
		2314	if (ndw > 0xFFFFE)
		2315	ndw = 0xFFFFE;
		2316
		2317	if (flags & RADEON_VM_PAGE_VALID)
		2318	value = addr;
		2319	else
		2320	value = 0;
		2321	/* for physically contiguous pages (vram) */
		2322	ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
		2323	ib->ptr[ib->length_dw++] = pe; /* dst addr */
		2324	ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
		2325	ib->ptr[ib->length_dw++] = r600_flags; /* mask */
		2326	ib->ptr[ib->length_dw++] = 0;
		2327	ib->ptr[ib->length_dw++] = value; /* value */
		2328	ib->ptr[ib->length_dw++] = upper_32_bits(value);
		2329	ib->ptr[ib->length_dw++] = incr; /* increment size */
		2330	ib->ptr[ib->length_dw++] = 0;
		2331	pe += ndw * 4;
		2332	addr += (ndw / 2) * incr;
		2333	count -= ndw / 2;
		2334	}
		2335	}
		2336	while (ib->length_dw & 0x7)
		2337	ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0);
3192	Serge	2338	}
2997	Serge	2339	}
		2340
		2341	/**
		2342	* cayman_vm_flush - vm flush using the CP
		2343	*
		2344	* @rdev: radeon_device pointer
		2345	*
		2346	* Update the page table base and flush the VM TLB
		2347	* using the CP (cayman-si).
		2348	*/
		2349	void cayman_vm_flush(struct radeon_device rdev, int ridx, struct radeon_vm vm)
		2350	{
		2351	struct radeon_ring *ring = &rdev->ring[ridx];
		2352
		2353	if (vm == NULL)
		2354	return;
		2355
		2356	radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
		2357	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
		2358
		2359	/* flush hdp cache */
		2360	radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
		2361	radeon_ring_write(ring, 0x1);
		2362
		2363	/* bits 0-7 are the VM contexts0-7 */
		2364	radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
		2365	radeon_ring_write(ring, 1 << vm->id);
		2366
		2367	/* sync PFP to ME, otherwise we might get invalid PFP reads */
		2368	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
		2369	radeon_ring_write(ring, 0x0);
		2370	}
3192	Serge	2371
		2372	void cayman_dma_vm_flush(struct radeon_device rdev, int ridx, struct radeon_vm vm)
		2373	{
		2374	struct radeon_ring *ring = &rdev->ring[ridx];
		2375
		2376	if (vm == NULL)
		2377	return;
		2378
		2379	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
		2380	radeon_ring_write(ring, (0xf << 16) \| ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
		2381	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
		2382
		2383	/* flush hdp cache */
		2384	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
		2385	radeon_ring_write(ring, (0xf << 16) \| (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
		2386	radeon_ring_write(ring, 1);
		2387
		2388	/* bits 0-7 are the VM contexts0-7 */
		2389	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0));
		2390	radeon_ring_write(ring, (0xf << 16) \| (VM_INVALIDATE_REQUEST >> 2));
		2391	radeon_ring_write(ring, 1 << vm->id);
		2392	}
		2393

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/ni.c – Rev 3764