WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/r600.c

Rev	Author	Line No.	Line
1128	serge	1	/*
		2	* Copyright 2008 Advanced Micro Devices, Inc.
		3	* Copyright 2008 Red Hat Inc.
		4	* Copyright 2009 Jerome Glisse.
		5	*
		6	* Permission is hereby granted, free of charge, to any person obtaining a
		7	* copy of this software and associated documentation files (the "Software"),
		8	* to deal in the Software without restriction, including without limitation
		9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		10	* and/or sell copies of the Software, and to permit persons to whom the
		11	* Software is furnished to do so, subject to the following conditions:
		12	*
		13	* The above copyright notice and this permission notice shall be included in
		14	* all copies or substantial portions of the Software.
		15	*
		16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		22	* OTHER DEALINGS IN THE SOFTWARE.
		23	*
		24	* Authors: Dave Airlie
		25	* Alex Deucher
		26	* Jerome Glisse
		27	*/
1963	serge	28	#include
1233	serge	29	#include
1221	serge	30	#include
2997	Serge	31	#include
		32	#include
		33	#include
1128	serge	34	#include "radeon.h"
1963	serge	35	#include "radeon_asic.h"
1221	serge	36	#include "radeon_mode.h"
		37	#include "r600d.h"
		38	#include "atom.h"
		39	#include "avivod.h"
1128	serge	40
1221	serge	41	#define PFP_UCODE_SIZE 576
		42	#define PM4_UCODE_SIZE 1792
1321	serge	43	#define RLC_UCODE_SIZE 768
1221	serge	44	#define R700_PFP_UCODE_SIZE 848
		45	#define R700_PM4_UCODE_SIZE 1360
1321	serge	46	#define R700_RLC_UCODE_SIZE 1024
1963	serge	47	#define EVERGREEN_PFP_UCODE_SIZE 1120
		48	#define EVERGREEN_PM4_UCODE_SIZE 1376
		49	#define EVERGREEN_RLC_UCODE_SIZE 768
		50	#define CAYMAN_RLC_UCODE_SIZE 1024
2997	Serge	51	#define ARUBA_RLC_UCODE_SIZE 1536
1128	serge	52
1221	serge	53	/* Firmware Names */
		54	MODULE_FIRMWARE("radeon/R600_pfp.bin");
		55	MODULE_FIRMWARE("radeon/R600_me.bin");
		56	MODULE_FIRMWARE("radeon/RV610_pfp.bin");
		57	MODULE_FIRMWARE("radeon/RV610_me.bin");
		58	MODULE_FIRMWARE("radeon/RV630_pfp.bin");
		59	MODULE_FIRMWARE("radeon/RV630_me.bin");
		60	MODULE_FIRMWARE("radeon/RV620_pfp.bin");
		61	MODULE_FIRMWARE("radeon/RV620_me.bin");
		62	MODULE_FIRMWARE("radeon/RV635_pfp.bin");
		63	MODULE_FIRMWARE("radeon/RV635_me.bin");
		64	MODULE_FIRMWARE("radeon/RV670_pfp.bin");
		65	MODULE_FIRMWARE("radeon/RV670_me.bin");
		66	MODULE_FIRMWARE("radeon/RS780_pfp.bin");
		67	MODULE_FIRMWARE("radeon/RS780_me.bin");
		68	MODULE_FIRMWARE("radeon/RV770_pfp.bin");
		69	MODULE_FIRMWARE("radeon/RV770_me.bin");
		70	MODULE_FIRMWARE("radeon/RV730_pfp.bin");
		71	MODULE_FIRMWARE("radeon/RV730_me.bin");
		72	MODULE_FIRMWARE("radeon/RV710_pfp.bin");
		73	MODULE_FIRMWARE("radeon/RV710_me.bin");
1321	serge	74	MODULE_FIRMWARE("radeon/R600_rlc.bin");
		75	MODULE_FIRMWARE("radeon/R700_rlc.bin");
1963	serge	76	MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
		77	MODULE_FIRMWARE("radeon/CEDAR_me.bin");
		78	MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
		79	MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
		80	MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
		81	MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
		82	MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
		83	MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
		84	MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
		85	MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
		86	MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
		87	MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
		88	MODULE_FIRMWARE("radeon/PALM_pfp.bin");
		89	MODULE_FIRMWARE("radeon/PALM_me.bin");
		90	MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
		91	MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
		92	MODULE_FIRMWARE("radeon/SUMO_me.bin");
		93	MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
		94	MODULE_FIRMWARE("radeon/SUMO2_me.bin");
1221	serge	95
		96	int r600_debugfs_mc_info_init(struct radeon_device *rdev);
		97
		98	/* r600,rv610,rv630,rv620,rv635,rv670 */
1128	serge	99	int r600_mc_wait_for_idle(struct radeon_device *rdev);
2997	Serge	100	static void r600_gpu_init(struct radeon_device *rdev);
1221	serge	101	void r600_fini(struct radeon_device *rdev);
1963	serge	102	void r600_irq_disable(struct radeon_device *rdev);
		103	static void r600_pcie_gen2_enable(struct radeon_device *rdev);
1128	serge	104
1963	serge	105	/* get temperature in millidegrees */
		106	int rv6xx_get_temp(struct radeon_device *rdev)
		107	{
		108	u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
		109	ASIC_T_SHIFT;
		110	int actual_temp = temp & 0xff;
		111
		112	if (temp & 0x100)
		113	actual_temp -= 256;
		114
		115	return actual_temp * 1000;
		116	}
		117
		118
		119
		120
		121
		122
		123	bool r600_gui_idle(struct radeon_device *rdev)
		124	{
		125	if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
		126	return false;
		127	else
		128	return true;
		129	}
		130
1321	serge	131	/* hpd for digital panel detect/disconnect */
		132	bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		133	{
		134	bool connected = false;
		135
		136	if (ASIC_IS_DCE3(rdev)) {
		137	switch (hpd) {
		138	case RADEON_HPD_1:
		139	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		140	connected = true;
		141	break;
		142	case RADEON_HPD_2:
		143	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		144	connected = true;
		145	break;
		146	case RADEON_HPD_3:
		147	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		148	connected = true;
		149	break;
		150	case RADEON_HPD_4:
		151	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		152	connected = true;
		153	break;
		154	/* DCE 3.2 */
		155	case RADEON_HPD_5:
		156	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		157	connected = true;
		158	break;
		159	case RADEON_HPD_6:
		160	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		161	connected = true;
		162	break;
		163	default:
		164	break;
		165	}
		166	} else {
		167	switch (hpd) {
		168	case RADEON_HPD_1:
		169	if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		170	connected = true;
		171	break;
		172	case RADEON_HPD_2:
		173	if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		174	connected = true;
		175	break;
		176	case RADEON_HPD_3:
		177	if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		178	connected = true;
		179	break;
		180	default:
		181	break;
		182	}
		183	}
		184	return connected;
		185	}
		186
		187	void r600_hpd_set_polarity(struct radeon_device *rdev,
		188	enum radeon_hpd_id hpd)
		189	{
		190	u32 tmp;
		191	bool connected = r600_hpd_sense(rdev, hpd);
		192
		193	if (ASIC_IS_DCE3(rdev)) {
		194	switch (hpd) {
		195	case RADEON_HPD_1:
		196	tmp = RREG32(DC_HPD1_INT_CONTROL);
		197	if (connected)
		198	tmp &= ~DC_HPDx_INT_POLARITY;
		199	else
		200	tmp \|= DC_HPDx_INT_POLARITY;
		201	WREG32(DC_HPD1_INT_CONTROL, tmp);
		202	break;
		203	case RADEON_HPD_2:
		204	tmp = RREG32(DC_HPD2_INT_CONTROL);
		205	if (connected)
		206	tmp &= ~DC_HPDx_INT_POLARITY;
		207	else
		208	tmp \|= DC_HPDx_INT_POLARITY;
		209	WREG32(DC_HPD2_INT_CONTROL, tmp);
		210	break;
		211	case RADEON_HPD_3:
		212	tmp = RREG32(DC_HPD3_INT_CONTROL);
		213	if (connected)
		214	tmp &= ~DC_HPDx_INT_POLARITY;
		215	else
		216	tmp \|= DC_HPDx_INT_POLARITY;
		217	WREG32(DC_HPD3_INT_CONTROL, tmp);
		218	break;
		219	case RADEON_HPD_4:
		220	tmp = RREG32(DC_HPD4_INT_CONTROL);
		221	if (connected)
		222	tmp &= ~DC_HPDx_INT_POLARITY;
		223	else
		224	tmp \|= DC_HPDx_INT_POLARITY;
		225	WREG32(DC_HPD4_INT_CONTROL, tmp);
		226	break;
		227	case RADEON_HPD_5:
		228	tmp = RREG32(DC_HPD5_INT_CONTROL);
		229	if (connected)
		230	tmp &= ~DC_HPDx_INT_POLARITY;
		231	else
		232	tmp \|= DC_HPDx_INT_POLARITY;
		233	WREG32(DC_HPD5_INT_CONTROL, tmp);
		234	break;
		235	/* DCE 3.2 */
		236	case RADEON_HPD_6:
		237	tmp = RREG32(DC_HPD6_INT_CONTROL);
		238	if (connected)
		239	tmp &= ~DC_HPDx_INT_POLARITY;
		240	else
		241	tmp \|= DC_HPDx_INT_POLARITY;
		242	WREG32(DC_HPD6_INT_CONTROL, tmp);
		243	break;
		244	default:
		245	break;
		246	}
		247	} else {
		248	switch (hpd) {
		249	case RADEON_HPD_1:
		250	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		251	if (connected)
		252	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		253	else
		254	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		255	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		256	break;
		257	case RADEON_HPD_2:
		258	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		259	if (connected)
		260	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		261	else
		262	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		263	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		264	break;
		265	case RADEON_HPD_3:
		266	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		267	if (connected)
		268	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		269	else
		270	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		271	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		272	break;
		273	default:
		274	break;
		275	}
		276	}
		277	}
		278
		279	void r600_hpd_init(struct radeon_device *rdev)
		280	{
		281	struct drm_device *dev = rdev->ddev;
		282	struct drm_connector *connector;
2997	Serge	283	unsigned enable = 0;
1321	serge	284
2997	Serge	285	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		286	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		287
		288	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP \|\|
		289	connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
		290	/* don't try to enable hpd on eDP or LVDS avoid breaking the
		291	* aux dp channel on imac and help (but not completely fix)
		292	* https://bugzilla.redhat.com/show_bug.cgi?id=726143
		293	*/
		294	continue;
		295	}
1321	serge	296	if (ASIC_IS_DCE3(rdev)) {
		297	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \| DC_HPDx_RX_INT_TIMER(0xfa);
		298	if (ASIC_IS_DCE32(rdev))
		299	tmp \|= DC_HPDx_EN;
		300
		301	switch (radeon_connector->hpd.hpd) {
		302	case RADEON_HPD_1:
		303	WREG32(DC_HPD1_CONTROL, tmp);
		304	break;
		305	case RADEON_HPD_2:
		306	WREG32(DC_HPD2_CONTROL, tmp);
		307	break;
		308	case RADEON_HPD_3:
		309	WREG32(DC_HPD3_CONTROL, tmp);
		310	break;
		311	case RADEON_HPD_4:
		312	WREG32(DC_HPD4_CONTROL, tmp);
		313	break;
		314	/* DCE 3.2 */
		315	case RADEON_HPD_5:
		316	WREG32(DC_HPD5_CONTROL, tmp);
		317	break;
		318	case RADEON_HPD_6:
		319	WREG32(DC_HPD6_CONTROL, tmp);
		320	break;
		321	default:
		322	break;
		323	}
		324	} else {
		325	switch (radeon_connector->hpd.hpd) {
		326	case RADEON_HPD_1:
		327	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		328	break;
		329	case RADEON_HPD_2:
		330	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		331	break;
		332	case RADEON_HPD_3:
		333	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		334	break;
		335	default:
		336	break;
		337	}
		338	}
2997	Serge	339	enable \|= 1 << radeon_connector->hpd.hpd;
		340	radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1321	serge	341	}
2997	Serge	342	// radeon_irq_kms_enable_hpd(rdev, enable);
1321	serge	343	}
		344
		345	void r600_hpd_fini(struct radeon_device *rdev)
		346	{
		347	struct drm_device *dev = rdev->ddev;
		348	struct drm_connector *connector;
2997	Serge	349	unsigned disable = 0;
1321	serge	350
		351	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		352	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2997	Serge	353	if (ASIC_IS_DCE3(rdev)) {
1321	serge	354	switch (radeon_connector->hpd.hpd) {
		355	case RADEON_HPD_1:
		356	WREG32(DC_HPD1_CONTROL, 0);
		357	break;
		358	case RADEON_HPD_2:
		359	WREG32(DC_HPD2_CONTROL, 0);
		360	break;
		361	case RADEON_HPD_3:
		362	WREG32(DC_HPD3_CONTROL, 0);
		363	break;
		364	case RADEON_HPD_4:
		365	WREG32(DC_HPD4_CONTROL, 0);
		366	break;
		367	/* DCE 3.2 */
		368	case RADEON_HPD_5:
		369	WREG32(DC_HPD5_CONTROL, 0);
		370	break;
		371	case RADEON_HPD_6:
		372	WREG32(DC_HPD6_CONTROL, 0);
		373	break;
		374	default:
		375	break;
		376	}
		377	} else {
		378	switch (radeon_connector->hpd.hpd) {
		379	case RADEON_HPD_1:
		380	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
		381	break;
		382	case RADEON_HPD_2:
		383	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
		384	break;
		385	case RADEON_HPD_3:
		386	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
		387	break;
		388	default:
		389	break;
		390	}
		391	}
2997	Serge	392	disable \|= 1 << radeon_connector->hpd.hpd;
1321	serge	393	}
2997	Serge	394	// radeon_irq_kms_disable_hpd(rdev, disable);
1321	serge	395	}
		396
1128	serge	397	/*
1221	serge	398	* R600 PCIE GART
1128	serge	399	*/
1221	serge	400	void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
		401	{
		402	unsigned i;
		403	u32 tmp;
1128	serge	404
1430	serge	405	/* flush hdp cache so updates hit vram */
1963	serge	406	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		407	!(rdev->flags & RADEON_IS_AGP)) {
2997	Serge	408	void __iomem ptr = (void )rdev->gart.ptr;
1963	serge	409	u32 tmp;
		410
		411	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		412	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
		413	* This seems to cause problems on some AGP cards. Just use the old
		414	* method for them.
		415	*/
		416	WREG32(HDP_DEBUG1, 0);
		417	tmp = readl((void __iomem *)ptr);
		418	} else
1430	serge	419	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		420
1221	serge	421	WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
		422	WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
		423	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		424	for (i = 0; i < rdev->usec_timeout; i++) {
		425	/* read MC_STATUS */
		426	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		427	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		428	if (tmp == 2) {
		429	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		430	return;
		431	}
		432	if (tmp) {
		433	return;
		434	}
		435	udelay(1);
1128	serge	436	}
1221	serge	437	}
1128	serge	438
1221	serge	439	int r600_pcie_gart_init(struct radeon_device *rdev)
		440	{
		441	int r;
		442
2997	Serge	443	if (rdev->gart.robj) {
1963	serge	444	WARN(1, "R600 PCIE GART already initialized\n");
1221	serge	445	return 0;
		446	}
		447	/* Initialize common gart structure */
		448	r = radeon_gart_init(rdev);
		449	if (r)
		450	return r;
		451	rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
		452	return radeon_gart_table_vram_alloc(rdev);
		453	}
		454
2997	Serge	455	static int r600_pcie_gart_enable(struct radeon_device *rdev)
1221	serge	456	{
		457	u32 tmp;
		458	int r, i;
		459
2997	Serge	460	if (rdev->gart.robj == NULL) {
1221	serge	461	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		462	return -EINVAL;
		463	}
		464	r = radeon_gart_table_vram_pin(rdev);
		465	if (r)
		466	return r;
1430	serge	467	radeon_gart_restore(rdev);
1221	serge	468
		469	/* Setup L2 cache */
		470	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		471	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		472	EFFECTIVE_L2_QUEUE_SIZE(7));
		473	WREG32(VM_L2_CNTL2, 0);
		474	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		475	/* Setup TLB control */
		476	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		477	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		478	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		479	ENABLE_WAIT_L2_QUERY;
		480	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		481	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		482	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		483	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		484	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		485	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		486	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		487	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		488	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		489	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		490	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		491	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		492	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		493	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		494	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		495	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		496	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		497	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		498	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		499	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		500	(u32)(rdev->dummy_page.addr >> 12));
		501	for (i = 1; i < 7; i++)
		502	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		503
		504	r600_pcie_gart_tlb_flush(rdev);
2997	Serge	505	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		506	(unsigned)(rdev->mc.gtt_size >> 20),
		507	(unsigned long long)rdev->gart.table_addr);
1221	serge	508	rdev->gart.ready = true;
1128	serge	509	return 0;
		510	}
		511
2997	Serge	512	static void r600_pcie_gart_disable(struct radeon_device *rdev)
1128	serge	513	{
1221	serge	514	u32 tmp;
2997	Serge	515	int i;
1221	serge	516
		517	/* Disable all tables */
		518	for (i = 0; i < 7; i++)
		519	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		520
		521	/* Disable L2 cache */
		522	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		523	EFFECTIVE_L2_QUEUE_SIZE(7));
		524	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		525	/* Setup L1 TLB control */
		526	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		527	ENABLE_WAIT_L2_QUERY;
		528	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		529	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		530	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		531	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		532	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		533	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		534	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		535	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		536	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
		537	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
		538	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		539	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		540	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
		541	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
2997	Serge	542	radeon_gart_table_vram_unpin(rdev);
1128	serge	543	}
		544
2997	Serge	545	static void r600_pcie_gart_fini(struct radeon_device *rdev)
1221	serge	546	{
1963	serge	547	radeon_gart_fini(rdev);
1221	serge	548	r600_pcie_gart_disable(rdev);
		549	radeon_gart_table_vram_free(rdev);
		550	}
1128	serge	551
2997	Serge	552	static void r600_agp_enable(struct radeon_device *rdev)
1128	serge	553	{
1221	serge	554	u32 tmp;
		555	int i;
		556
		557	/* Setup L2 cache */
		558	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		559	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		560	EFFECTIVE_L2_QUEUE_SIZE(7));
		561	WREG32(VM_L2_CNTL2, 0);
		562	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		563	/* Setup TLB control */
		564	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		565	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		566	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		567	ENABLE_WAIT_L2_QUERY;
		568	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		569	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		570	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		571	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		572	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		573	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		574	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		575	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		576	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		577	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		578	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		579	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		580	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		581	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		582	for (i = 0; i < 7; i++)
		583	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1128	serge	584	}
		585
		586	int r600_mc_wait_for_idle(struct radeon_device *rdev)
		587	{
1221	serge	588	unsigned i;
		589	u32 tmp;
		590
		591	for (i = 0; i < rdev->usec_timeout; i++) {
		592	/* read MC_STATUS */
		593	tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
		594	if (!tmp)
1128	serge	595	return 0;
1221	serge	596	udelay(1);
		597	}
		598	return -1;
1128	serge	599	}
		600
1221	serge	601	static void r600_mc_program(struct radeon_device *rdev)
1128	serge	602	{
1221	serge	603	struct rv515_mc_save save;
		604	u32 tmp;
		605	int i, j;
		606
		607	/* Initialize HDP */
		608	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		609	WREG32((0x2c14 + j), 0x00000000);
		610	WREG32((0x2c18 + j), 0x00000000);
		611	WREG32((0x2c1c + j), 0x00000000);
		612	WREG32((0x2c20 + j), 0x00000000);
		613	WREG32((0x2c24 + j), 0x00000000);
		614	}
		615	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		616
		617	rv515_mc_stop(rdev, &save);
		618	if (r600_mc_wait_for_idle(rdev)) {
		619	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		620	}
		621	/* Lockout access through VGA aperture (doesn't exist before R600) */
		622	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		623	/* Update configuration */
		624	if (rdev->flags & RADEON_IS_AGP) {
		625	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		626	/* VRAM before AGP */
		627	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		628	rdev->mc.vram_start >> 12);
		629	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		630	rdev->mc.gtt_end >> 12);
		631	} else {
		632	/* VRAM after AGP */
		633	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		634	rdev->mc.gtt_start >> 12);
		635	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		636	rdev->mc.vram_end >> 12);
		637	}
		638	} else {
		639	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
		640	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
		641	}
2997	Serge	642	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1221	serge	643	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		644	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		645	WREG32(MC_VM_FB_LOCATION, tmp);
		646	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
		647	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1963	serge	648	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1221	serge	649	if (rdev->flags & RADEON_IS_AGP) {
		650	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
		651	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
		652	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		653	} else {
		654	WREG32(MC_VM_AGP_BASE, 0);
		655	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		656	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		657	}
		658	if (r600_mc_wait_for_idle(rdev)) {
		659	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		660	}
		661	rv515_mc_resume(rdev, &save);
		662	/* we need to own VRAM, so turn off the VGA renderer here
		663	* to stop it overwriting our objects */
		664	rv515_vga_render_disable(rdev);
1128	serge	665	}
		666
1430	serge	667	/**
		668	* r600_vram_gtt_location - try to find VRAM & GTT location
		669	* @rdev: radeon device structure holding all necessary informations
		670	* @mc: memory controller structure holding memory informations
		671	*
		672	* Function will place try to place VRAM at same place as in CPU (PCI)
		673	* address space as some GPU seems to have issue when we reprogram at
		674	* different address space.
		675	*
		676	* If there is not enough space to fit the unvisible VRAM after the
		677	* aperture then we limit the VRAM size to the aperture.
		678	*
		679	* If we are using AGP then place VRAM adjacent to AGP aperture are we need
		680	* them to be in one from GPU point of view so that we can program GPU to
		681	* catch access outside them (weird GPU policy see ??).
		682	*
		683	* This function will never fails, worst case are limiting VRAM or GTT.
		684	*
		685	* Note: GTT start, end, size should be initialized before calling this
		686	* function on AGP platform.
		687	*/
1963	serge	688	static void r600_vram_gtt_location(struct radeon_device rdev, struct radeon_mc mc)
1430	serge	689	{
		690	u64 size_bf, size_af;
		691
		692	if (mc->mc_vram_size > 0xE0000000) {
		693	/* leave room for at least 512M GTT */
		694	dev_warn(rdev->dev, "limiting VRAM\n");
		695	mc->real_vram_size = 0xE0000000;
		696	mc->mc_vram_size = 0xE0000000;
		697	}
		698	if (rdev->flags & RADEON_IS_AGP) {
		699	size_bf = mc->gtt_start;
2997	Serge	700	size_af = 0xFFFFFFFF - mc->gtt_end;
1430	serge	701	if (size_bf > size_af) {
		702	if (mc->mc_vram_size > size_bf) {
		703	dev_warn(rdev->dev, "limiting VRAM\n");
		704	mc->real_vram_size = size_bf;
		705	mc->mc_vram_size = size_bf;
		706	}
		707	mc->vram_start = mc->gtt_start - mc->mc_vram_size;
		708	} else {
		709	if (mc->mc_vram_size > size_af) {
		710	dev_warn(rdev->dev, "limiting VRAM\n");
		711	mc->real_vram_size = size_af;
		712	mc->mc_vram_size = size_af;
		713	}
2997	Serge	714	mc->vram_start = mc->gtt_end + 1;
1430	serge	715	}
		716	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
		717	dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
		718	mc->mc_vram_size >> 20, mc->vram_start,
		719	mc->vram_end, mc->real_vram_size >> 20);
		720	} else {
		721	u64 base = 0;
1963	serge	722	if (rdev->flags & RADEON_IS_IGP) {
		723	base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
		724	base <<= 24;
		725	}
1430	serge	726	radeon_vram_location(rdev, &rdev->mc, base);
1963	serge	727	rdev->mc.gtt_base_align = 0;
1430	serge	728	radeon_gtt_location(rdev, mc);
		729	}
		730	}
		731
2997	Serge	732	static int r600_mc_init(struct radeon_device *rdev)
1128	serge	733	{
1221	serge	734	u32 tmp;
1268	serge	735	int chansize, numchan;
1128	serge	736
1221	serge	737	/* Get VRAM informations */
1128	serge	738	rdev->mc.vram_is_ddr = true;
1221	serge	739	tmp = RREG32(RAMCFG);
		740	if (tmp & CHANSIZE_OVERRIDE) {
1128	serge	741	chansize = 16;
1221	serge	742	} else if (tmp & CHANSIZE_MASK) {
1128	serge	743	chansize = 64;
		744	} else {
		745	chansize = 32;
		746	}
1268	serge	747	tmp = RREG32(CHMAP);
		748	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		749	case 0:
		750	default:
		751	numchan = 1;
		752	break;
		753	case 1:
		754	numchan = 2;
		755	break;
		756	case 2:
		757	numchan = 4;
		758	break;
		759	case 3:
		760	numchan = 8;
		761	break;
1128	serge	762	}
1268	serge	763	rdev->mc.vram_width = numchan * chansize;
1221	serge	764	/* Could aper size report 0 ? */
1963	serge	765	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		766	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1221	serge	767	/* Setup GPU memory space */
		768	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		769	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1430	serge	770	rdev->mc.visible_vram_size = rdev->mc.aper_size;
		771	r600_vram_gtt_location(rdev, &rdev->mc);
1963	serge	772
		773	if (rdev->flags & RADEON_IS_IGP) {
		774	rs690_pm_info(rdev);
1403	serge	775	rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1963	serge	776	}
		777	radeon_update_bandwidth_info(rdev);
1221	serge	778	return 0;
1128	serge	779	}
		780
2997	Serge	781	int r600_vram_scratch_init(struct radeon_device *rdev)
		782	{
		783	int r;
		784
		785	if (rdev->vram_scratch.robj == NULL) {
		786	r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
		787	PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
		788	NULL, &rdev->vram_scratch.robj);
		789	if (r) {
		790	return r;
		791	}
		792	}
		793
		794	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
		795	if (unlikely(r != 0))
		796	return r;
		797	r = radeon_bo_pin(rdev->vram_scratch.robj,
		798	RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
		799	if (r) {
		800	radeon_bo_unreserve(rdev->vram_scratch.robj);
		801	return r;
		802	}
		803	r = radeon_bo_kmap(rdev->vram_scratch.robj,
		804	(void **)&rdev->vram_scratch.ptr);
		805	if (r)
		806	radeon_bo_unpin(rdev->vram_scratch.robj);
		807	radeon_bo_unreserve(rdev->vram_scratch.robj);
		808
		809	return r;
		810	}
1221	serge	811	/* We doesn't check that the GPU really needs a reset we simply do the
		812	* reset, it's up to the caller to determine if the GPU needs one. We
		813	* might add an helper function to check that.
		814	*/
3192	Serge	815	static void r600_gpu_soft_reset_gfx(struct radeon_device *rdev)
1128	serge	816	{
1221	serge	817	u32 grbm_busy_mask = S_008010_VC_BUSY(1) \| S_008010_VGT_BUSY_NO_DMA(1) \|
		818	S_008010_VGT_BUSY(1) \| S_008010_TA03_BUSY(1) \|
		819	S_008010_TC_BUSY(1) \| S_008010_SX_BUSY(1) \|
		820	S_008010_SH_BUSY(1) \| S_008010_SPI03_BUSY(1) \|
		821	S_008010_SMX_BUSY(1) \| S_008010_SC_BUSY(1) \|
		822	S_008010_PA_BUSY(1) \| S_008010_DB03_BUSY(1) \|
		823	S_008010_CR_BUSY(1) \| S_008010_CB03_BUSY(1) \|
		824	S_008010_GUI_ACTIVE(1);
		825	u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) \| S_008014_SPI1_BUSY(1) \|
		826	S_008014_SPI2_BUSY(1) \| S_008014_SPI3_BUSY(1) \|
		827	S_008014_TA0_BUSY(1) \| S_008014_TA1_BUSY(1) \|
		828	S_008014_TA2_BUSY(1) \| S_008014_TA3_BUSY(1) \|
		829	S_008014_DB0_BUSY(1) \| S_008014_DB1_BUSY(1) \|
		830	S_008014_DB2_BUSY(1) \| S_008014_DB3_BUSY(1) \|
		831	S_008014_CB0_BUSY(1) \| S_008014_CB1_BUSY(1) \|
		832	S_008014_CB2_BUSY(1) \| S_008014_CB3_BUSY(1);
		833	u32 tmp;
1128	serge	834
1963	serge	835	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
3192	Serge	836	return;
1963	serge	837
3192	Serge	838	dev_info(rdev->dev, " R_008010_GRBM_STATUS = 0x%08X\n",
1221	serge	839	RREG32(R_008010_GRBM_STATUS));
3192	Serge	840	dev_info(rdev->dev, " R_008014_GRBM_STATUS2 = 0x%08X\n",
1221	serge	841	RREG32(R_008014_GRBM_STATUS2));
3192	Serge	842	dev_info(rdev->dev, " R_000E50_SRBM_STATUS = 0x%08X\n",
1221	serge	843	RREG32(R_000E50_SRBM_STATUS));
2997	Serge	844	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		845	RREG32(CP_STALLED_STAT1));
		846	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		847	RREG32(CP_STALLED_STAT2));
		848	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		849	RREG32(CP_BUSY_STAT));
		850	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		851	RREG32(CP_STAT));
3192	Serge	852
1221	serge	853	/* Disable CP parsing/prefetching */
1963	serge	854	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
3192	Serge	855
1221	serge	856	/* Check if any of the rendering block is busy and reset it */
		857	if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) \|\|
		858	(RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
		859	tmp = S_008020_SOFT_RESET_CR(1) \|
		860	S_008020_SOFT_RESET_DB(1) \|
		861	S_008020_SOFT_RESET_CB(1) \|
		862	S_008020_SOFT_RESET_PA(1) \|
		863	S_008020_SOFT_RESET_SC(1) \|
		864	S_008020_SOFT_RESET_SMX(1) \|
		865	S_008020_SOFT_RESET_SPI(1) \|
		866	S_008020_SOFT_RESET_SX(1) \|
		867	S_008020_SOFT_RESET_SH(1) \|
		868	S_008020_SOFT_RESET_TC(1) \|
		869	S_008020_SOFT_RESET_TA(1) \|
		870	S_008020_SOFT_RESET_VC(1) \|
		871	S_008020_SOFT_RESET_VGT(1);
		872	dev_info(rdev->dev, " R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		873	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1963	serge	874	RREG32(R_008020_GRBM_SOFT_RESET);
		875	mdelay(15);
1221	serge	876	WREG32(R_008020_GRBM_SOFT_RESET, 0);
		877	}
		878	/* Reset CP (we always reset CP) */
		879	tmp = S_008020_SOFT_RESET_CP(1);
		880	dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		881	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1963	serge	882	RREG32(R_008020_GRBM_SOFT_RESET);
		883	mdelay(15);
1221	serge	884	WREG32(R_008020_GRBM_SOFT_RESET, 0);
3192	Serge	885
		886	dev_info(rdev->dev, " R_008010_GRBM_STATUS = 0x%08X\n",
1221	serge	887	RREG32(R_008010_GRBM_STATUS));
3192	Serge	888	dev_info(rdev->dev, " R_008014_GRBM_STATUS2 = 0x%08X\n",
1221	serge	889	RREG32(R_008014_GRBM_STATUS2));
3192	Serge	890	dev_info(rdev->dev, " R_000E50_SRBM_STATUS = 0x%08X\n",
1221	serge	891	RREG32(R_000E50_SRBM_STATUS));
2997	Serge	892	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		893	RREG32(CP_STALLED_STAT1));
		894	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		895	RREG32(CP_STALLED_STAT2));
		896	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		897	RREG32(CP_BUSY_STAT));
		898	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		899	RREG32(CP_STAT));
3192	Serge	900
		901	}
		902
		903	static void r600_gpu_soft_reset_dma(struct radeon_device *rdev)
		904	{
		905	u32 tmp;
		906
		907	if (RREG32(DMA_STATUS_REG) & DMA_IDLE)
		908	return;
		909
		910	dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
		911	RREG32(DMA_STATUS_REG));
		912
		913	/* Disable DMA */
		914	tmp = RREG32(DMA_RB_CNTL);
		915	tmp &= ~DMA_RB_ENABLE;
		916	WREG32(DMA_RB_CNTL, tmp);
		917
		918	/* Reset dma */
		919	if (rdev->family >= CHIP_RV770)
		920	WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
		921	else
		922	WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
		923	RREG32(SRBM_SOFT_RESET);
		924	udelay(50);
		925	WREG32(SRBM_SOFT_RESET, 0);
		926
		927	dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
		928	RREG32(DMA_STATUS_REG));
		929	}
		930
		931	static int r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
		932	{
		933	struct rv515_mc_save save;
		934
		935	if (reset_mask == 0)
		936	return 0;
		937
		938	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
		939
		940	rv515_mc_stop(rdev, &save);
		941	if (r600_mc_wait_for_idle(rdev)) {
		942	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		943	}
		944
		945	if (reset_mask & (RADEON_RESET_GFX \| RADEON_RESET_COMPUTE))
		946	r600_gpu_soft_reset_gfx(rdev);
		947
		948	if (reset_mask & RADEON_RESET_DMA)
		949	r600_gpu_soft_reset_dma(rdev);
		950
		951	/* Wait a little for things to settle down */
		952	mdelay(1);
		953
1221	serge	954	rv515_mc_resume(rdev, &save);
		955	return 0;
1128	serge	956	}
		957
2997	Serge	958	bool r600_gpu_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
1221	serge	959	{
1963	serge	960	u32 srbm_status;
		961	u32 grbm_status;
		962	u32 grbm_status2;
		963
		964	srbm_status = RREG32(R_000E50_SRBM_STATUS);
		965	grbm_status = RREG32(R_008010_GRBM_STATUS);
		966	grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
		967	if (!G_008010_GUI_ACTIVE(grbm_status)) {
2997	Serge	968	radeon_ring_lockup_update(ring);
1963	serge	969	return false;
		970	}
		971	/* force CP activities */
2997	Serge	972	radeon_ring_force_activity(rdev, ring);
		973	return radeon_ring_test_lockup(rdev, ring);
1963	serge	974	}
		975
3192	Serge	976	/**
		977	* r600_dma_is_lockup - Check if the DMA engine is locked up
		978	*
		979	* @rdev: radeon_device pointer
		980	* @ring: radeon_ring structure holding ring information
		981	*
		982	* Check if the async DMA engine is locked up (r6xx-evergreen).
		983	* Returns true if the engine appears to be locked up, false if not.
		984	*/
		985	bool r600_dma_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
		986	{
		987	u32 dma_status_reg;
		988
		989	dma_status_reg = RREG32(DMA_STATUS_REG);
		990	if (dma_status_reg & DMA_IDLE) {
		991	radeon_ring_lockup_update(ring);
		992	return false;
		993	}
		994	/* force ring activities */
		995	radeon_ring_force_activity(rdev, ring);
		996	return radeon_ring_test_lockup(rdev, ring);
		997	}
		998
1963	serge	999	int r600_asic_reset(struct radeon_device *rdev)
		1000	{
3192	Serge	1001	return r600_gpu_soft_reset(rdev, (RADEON_RESET_GFX \|
		1002	RADEON_RESET_COMPUTE \|
		1003	RADEON_RESET_DMA));
1221	serge	1004	}
		1005
2997	Serge	1006	u32 r6xx_remap_render_backend(struct radeon_device *rdev,
		1007	u32 tiling_pipe_num,
		1008	u32 max_rb_num,
		1009	u32 total_max_rb_num,
		1010	u32 disabled_rb_mask)
1221	serge	1011	{
2997	Serge	1012	u32 rendering_pipe_num, rb_num_width, req_rb_num;
		1013	u32 pipe_rb_ratio, pipe_rb_remain;
		1014	u32 data = 0, mask = 1 << (max_rb_num - 1);
		1015	unsigned i, j;
1221	serge	1016
2997	Serge	1017	/* mask out the RBs that don't exist on that asic */
		1018	disabled_rb_mask \|= (0xff << max_rb_num) & 0xff;
1221	serge	1019
2997	Serge	1020	rendering_pipe_num = 1 << tiling_pipe_num;
		1021	req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
		1022	BUG_ON(rendering_pipe_num < req_rb_num);
1221	serge	1023
2997	Serge	1024	pipe_rb_ratio = rendering_pipe_num / req_rb_num;
		1025	pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1221	serge	1026
2997	Serge	1027	if (rdev->family <= CHIP_RV740) {
		1028	/* r6xx/r7xx */
		1029	rb_num_width = 2;
		1030	} else {
		1031	/* eg+ */
		1032	rb_num_width = 4;
		1033	}
1221	serge	1034
2997	Serge	1035	for (i = 0; i < max_rb_num; i++) {
		1036	if (!(mask & disabled_rb_mask)) {
		1037	for (j = 0; j < pipe_rb_ratio; j++) {
		1038	data <<= rb_num_width;
		1039	data \|= max_rb_num - i - 1;
1221	serge	1040	}
2997	Serge	1041	if (pipe_rb_remain) {
		1042	data <<= rb_num_width;
		1043	data \|= max_rb_num - i - 1;
		1044	pipe_rb_remain--;
1221	serge	1045	}
2997	Serge	1046	}
		1047	mask >>= 1;
		1048	}
1221	serge	1049
2997	Serge	1050	return data;
1221	serge	1051	}
		1052
		1053	int r600_count_pipe_bits(uint32_t val)
		1054	{
3192	Serge	1055	return hweight32(val);
1221	serge	1056	}
		1057
2997	Serge	1058	static void r600_gpu_init(struct radeon_device *rdev)
1221	serge	1059	{
		1060	u32 tiling_config;
		1061	u32 ramcfg;
1430	serge	1062	u32 cc_rb_backend_disable;
		1063	u32 cc_gc_shader_pipe_config;
1221	serge	1064	u32 tmp;
		1065	int i, j;
		1066	u32 sq_config;
		1067	u32 sq_gpr_resource_mgmt_1 = 0;
		1068	u32 sq_gpr_resource_mgmt_2 = 0;
		1069	u32 sq_thread_resource_mgmt = 0;
		1070	u32 sq_stack_resource_mgmt_1 = 0;
		1071	u32 sq_stack_resource_mgmt_2 = 0;
2997	Serge	1072	u32 disabled_rb_mask;
1221	serge	1073
2997	Serge	1074	rdev->config.r600.tiling_group_size = 256;
1221	serge	1075	switch (rdev->family) {
		1076	case CHIP_R600:
		1077	rdev->config.r600.max_pipes = 4;
		1078	rdev->config.r600.max_tile_pipes = 8;
		1079	rdev->config.r600.max_simds = 4;
		1080	rdev->config.r600.max_backends = 4;
		1081	rdev->config.r600.max_gprs = 256;
		1082	rdev->config.r600.max_threads = 192;
		1083	rdev->config.r600.max_stack_entries = 256;
		1084	rdev->config.r600.max_hw_contexts = 8;
		1085	rdev->config.r600.max_gs_threads = 16;
		1086	rdev->config.r600.sx_max_export_size = 128;
		1087	rdev->config.r600.sx_max_export_pos_size = 16;
		1088	rdev->config.r600.sx_max_export_smx_size = 128;
		1089	rdev->config.r600.sq_num_cf_insts = 2;
		1090	break;
		1091	case CHIP_RV630:
		1092	case CHIP_RV635:
		1093	rdev->config.r600.max_pipes = 2;
		1094	rdev->config.r600.max_tile_pipes = 2;
		1095	rdev->config.r600.max_simds = 3;
		1096	rdev->config.r600.max_backends = 1;
		1097	rdev->config.r600.max_gprs = 128;
		1098	rdev->config.r600.max_threads = 192;
		1099	rdev->config.r600.max_stack_entries = 128;
		1100	rdev->config.r600.max_hw_contexts = 8;
		1101	rdev->config.r600.max_gs_threads = 4;
		1102	rdev->config.r600.sx_max_export_size = 128;
		1103	rdev->config.r600.sx_max_export_pos_size = 16;
		1104	rdev->config.r600.sx_max_export_smx_size = 128;
		1105	rdev->config.r600.sq_num_cf_insts = 2;
		1106	break;
		1107	case CHIP_RV610:
		1108	case CHIP_RV620:
		1109	case CHIP_RS780:
		1110	case CHIP_RS880:
		1111	rdev->config.r600.max_pipes = 1;
		1112	rdev->config.r600.max_tile_pipes = 1;
		1113	rdev->config.r600.max_simds = 2;
		1114	rdev->config.r600.max_backends = 1;
		1115	rdev->config.r600.max_gprs = 128;
		1116	rdev->config.r600.max_threads = 192;
		1117	rdev->config.r600.max_stack_entries = 128;
		1118	rdev->config.r600.max_hw_contexts = 4;
		1119	rdev->config.r600.max_gs_threads = 4;
		1120	rdev->config.r600.sx_max_export_size = 128;
		1121	rdev->config.r600.sx_max_export_pos_size = 16;
		1122	rdev->config.r600.sx_max_export_smx_size = 128;
		1123	rdev->config.r600.sq_num_cf_insts = 1;
		1124	break;
		1125	case CHIP_RV670:
		1126	rdev->config.r600.max_pipes = 4;
		1127	rdev->config.r600.max_tile_pipes = 4;
		1128	rdev->config.r600.max_simds = 4;
		1129	rdev->config.r600.max_backends = 4;
		1130	rdev->config.r600.max_gprs = 192;
		1131	rdev->config.r600.max_threads = 192;
		1132	rdev->config.r600.max_stack_entries = 256;
		1133	rdev->config.r600.max_hw_contexts = 8;
		1134	rdev->config.r600.max_gs_threads = 16;
		1135	rdev->config.r600.sx_max_export_size = 128;
		1136	rdev->config.r600.sx_max_export_pos_size = 16;
		1137	rdev->config.r600.sx_max_export_smx_size = 128;
		1138	rdev->config.r600.sq_num_cf_insts = 2;
		1139	break;
		1140	default:
		1141	break;
		1142	}
		1143
		1144	/* Initialize HDP */
		1145	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1146	WREG32((0x2c14 + j), 0x00000000);
		1147	WREG32((0x2c18 + j), 0x00000000);
		1148	WREG32((0x2c1c + j), 0x00000000);
		1149	WREG32((0x2c20 + j), 0x00000000);
		1150	WREG32((0x2c24 + j), 0x00000000);
		1151	}
		1152
		1153	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1154
		1155	/* Setup tiling */
		1156	tiling_config = 0;
		1157	ramcfg = RREG32(RAMCFG);
		1158	switch (rdev->config.r600.max_tile_pipes) {
		1159	case 1:
		1160	tiling_config \|= PIPE_TILING(0);
		1161	break;
		1162	case 2:
		1163	tiling_config \|= PIPE_TILING(1);
		1164	break;
		1165	case 4:
		1166	tiling_config \|= PIPE_TILING(2);
		1167	break;
		1168	case 8:
		1169	tiling_config \|= PIPE_TILING(3);
		1170	break;
		1171	default:
		1172	break;
		1173	}
1430	serge	1174	rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
		1175	rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1221	serge	1176	tiling_config \|= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1963	serge	1177	tiling_config \|= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
2997	Serge	1178
1221	serge	1179	tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
		1180	if (tmp > 3) {
		1181	tiling_config \|= ROW_TILING(3);
		1182	tiling_config \|= SAMPLE_SPLIT(3);
		1183	} else {
		1184	tiling_config \|= ROW_TILING(tmp);
		1185	tiling_config \|= SAMPLE_SPLIT(tmp);
		1186	}
		1187	tiling_config \|= BANK_SWAPS(1);
1430	serge	1188
		1189	cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
2997	Serge	1190	tmp = R6XX_MAX_BACKENDS -
		1191	r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
		1192	if (tmp < rdev->config.r600.max_backends) {
		1193	rdev->config.r600.max_backends = tmp;
		1194	}
1430	serge	1195
2997	Serge	1196	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
		1197	tmp = R6XX_MAX_PIPES -
		1198	r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
		1199	if (tmp < rdev->config.r600.max_pipes) {
		1200	rdev->config.r600.max_pipes = tmp;
		1201	}
		1202	tmp = R6XX_MAX_SIMDS -
		1203	r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
		1204	if (tmp < rdev->config.r600.max_simds) {
		1205	rdev->config.r600.max_simds = tmp;
		1206	}
1430	serge	1207
2997	Serge	1208	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
		1209	tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
		1210	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
		1211	R6XX_MAX_BACKENDS, disabled_rb_mask);
		1212	tiling_config \|= tmp << 16;
		1213	rdev->config.r600.backend_map = tmp;
		1214
1963	serge	1215	rdev->config.r600.tile_config = tiling_config;
1221	serge	1216	WREG32(GB_TILING_CONFIG, tiling_config);
		1217	WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
		1218	WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
3192	Serge	1219	WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
1221	serge	1220
1430	serge	1221	tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1221	serge	1222	WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
		1223	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
		1224
		1225	/* Setup some CP states */
		1226	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \| ROQ_IB2_START(0x2b)));
		1227	WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) \| ROQ_END(0x40)));
		1228
		1229	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \| SYNC_GRADIENT \|
		1230	SYNC_WALKER \| SYNC_ALIGNER));
		1231	/* Setup various GPU states */
		1232	if (rdev->family == CHIP_RV670)
		1233	WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
		1234
		1235	tmp = RREG32(SX_DEBUG_1);
		1236	tmp \|= SMX_EVENT_RELEASE;
		1237	if ((rdev->family > CHIP_R600))
		1238	tmp \|= ENABLE_NEW_SMX_ADDRESS;
		1239	WREG32(SX_DEBUG_1, tmp);
		1240
		1241	if (((rdev->family) == CHIP_R600) \|\|
		1242	((rdev->family) == CHIP_RV630) \|\|
		1243	((rdev->family) == CHIP_RV610) \|\|
		1244	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1245	((rdev->family) == CHIP_RS780) \|\|
		1246	((rdev->family) == CHIP_RS880)) {
1221	serge	1247	WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
		1248	} else {
		1249	WREG32(DB_DEBUG, 0);
		1250	}
		1251	WREG32(DB_WATERMARKS, (DEPTH_FREE(4) \| DEPTH_CACHELINE_FREE(16) \|
		1252	DEPTH_FLUSH(16) \| DEPTH_PENDING_FREE(4)));
		1253
		1254	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1255	WREG32(VGT_NUM_INSTANCES, 0);
		1256
		1257	WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
		1258	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
		1259
		1260	tmp = RREG32(SQ_MS_FIFO_SIZES);
		1261	if (((rdev->family) == CHIP_RV610) \|\|
		1262	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1263	((rdev->family) == CHIP_RS780) \|\|
		1264	((rdev->family) == CHIP_RS880)) {
1221	serge	1265	tmp = (CACHE_FIFO_SIZE(0xa) \|
		1266	FETCH_FIFO_HIWATER(0xa) \|
		1267	DONE_FIFO_HIWATER(0xe0) \|
		1268	ALU_UPDATE_FIFO_HIWATER(0x8));
		1269	} else if (((rdev->family) == CHIP_R600) \|\|
		1270	((rdev->family) == CHIP_RV630)) {
		1271	tmp &= ~DONE_FIFO_HIWATER(0xff);
		1272	tmp \|= DONE_FIFO_HIWATER(0x4);
		1273	}
		1274	WREG32(SQ_MS_FIFO_SIZES, tmp);
		1275
		1276	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
		1277	* should be adjusted as needed by the 2D/3D drivers. This just sets default values
		1278	*/
		1279	sq_config = RREG32(SQ_CONFIG);
		1280	sq_config &= ~(PS_PRIO(3) \|
		1281	VS_PRIO(3) \|
		1282	GS_PRIO(3) \|
		1283	ES_PRIO(3));
		1284	sq_config \|= (DX9_CONSTS \|
		1285	VC_ENABLE \|
		1286	PS_PRIO(0) \|
		1287	VS_PRIO(1) \|
		1288	GS_PRIO(2) \|
		1289	ES_PRIO(3));
		1290
		1291	if ((rdev->family) == CHIP_R600) {
		1292	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) \|
		1293	NUM_VS_GPRS(124) \|
		1294	NUM_CLAUSE_TEMP_GPRS(4));
		1295	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) \|
		1296	NUM_ES_GPRS(0));
		1297	sq_thread_resource_mgmt = (NUM_PS_THREADS(136) \|
		1298	NUM_VS_THREADS(48) \|
		1299	NUM_GS_THREADS(4) \|
		1300	NUM_ES_THREADS(4));
		1301	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) \|
		1302	NUM_VS_STACK_ENTRIES(128));
		1303	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) \|
		1304	NUM_ES_STACK_ENTRIES(0));
		1305	} else if (((rdev->family) == CHIP_RV610) \|\|
		1306	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1307	((rdev->family) == CHIP_RS780) \|\|
		1308	((rdev->family) == CHIP_RS880)) {
1221	serge	1309	/* no vertex cache */
		1310	sq_config &= ~VC_ENABLE;
		1311
		1312	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1313	NUM_VS_GPRS(44) \|
		1314	NUM_CLAUSE_TEMP_GPRS(2));
		1315	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1316	NUM_ES_GPRS(17));
		1317	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1318	NUM_VS_THREADS(78) \|
		1319	NUM_GS_THREADS(4) \|
		1320	NUM_ES_THREADS(31));
		1321	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1322	NUM_VS_STACK_ENTRIES(40));
		1323	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1324	NUM_ES_STACK_ENTRIES(16));
		1325	} else if (((rdev->family) == CHIP_RV630) \|\|
		1326	((rdev->family) == CHIP_RV635)) {
		1327	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1328	NUM_VS_GPRS(44) \|
		1329	NUM_CLAUSE_TEMP_GPRS(2));
		1330	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) \|
		1331	NUM_ES_GPRS(18));
		1332	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1333	NUM_VS_THREADS(78) \|
		1334	NUM_GS_THREADS(4) \|
		1335	NUM_ES_THREADS(31));
		1336	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1337	NUM_VS_STACK_ENTRIES(40));
		1338	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1339	NUM_ES_STACK_ENTRIES(16));
		1340	} else if ((rdev->family) == CHIP_RV670) {
		1341	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1342	NUM_VS_GPRS(44) \|
		1343	NUM_CLAUSE_TEMP_GPRS(2));
		1344	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1345	NUM_ES_GPRS(17));
		1346	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1347	NUM_VS_THREADS(78) \|
		1348	NUM_GS_THREADS(4) \|
		1349	NUM_ES_THREADS(31));
		1350	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) \|
		1351	NUM_VS_STACK_ENTRIES(64));
		1352	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) \|
		1353	NUM_ES_STACK_ENTRIES(64));
		1354	}
		1355
		1356	WREG32(SQ_CONFIG, sq_config);
		1357	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		1358	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		1359	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		1360	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		1361	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		1362
		1363	if (((rdev->family) == CHIP_RV610) \|\|
		1364	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1365	((rdev->family) == CHIP_RS780) \|\|
		1366	((rdev->family) == CHIP_RS880)) {
1221	serge	1367	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
		1368	} else {
		1369	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
		1370	}
		1371
		1372	/* More default values. 2D/3D driver should adjust as needed */
		1373	WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) \| S0_Y(0x4) \|
		1374	S1_X(0x4) \| S1_Y(0xc)));
		1375	WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) \| S0_Y(0xe) \|
		1376	S1_X(0x2) \| S1_Y(0x2) \|
		1377	S2_X(0xa) \| S2_Y(0x6) \|
		1378	S3_X(0x6) \| S3_Y(0xa)));
		1379	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) \| S0_Y(0xb) \|
		1380	S1_X(0x4) \| S1_Y(0xc) \|
		1381	S2_X(0x1) \| S2_Y(0x6) \|
		1382	S3_X(0xa) \| S3_Y(0xe)));
		1383	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) \| S4_Y(0x1) \|
		1384	S5_X(0x0) \| S5_Y(0x0) \|
		1385	S6_X(0xb) \| S6_Y(0x4) \|
		1386	S7_X(0x7) \| S7_Y(0x8)));
		1387
		1388	WREG32(VGT_STRMOUT_EN, 0);
		1389	tmp = rdev->config.r600.max_pipes * 16;
		1390	switch (rdev->family) {
		1391	case CHIP_RV610:
1268	serge	1392	case CHIP_RV620:
1221	serge	1393	case CHIP_RS780:
1268	serge	1394	case CHIP_RS880:
1221	serge	1395	tmp += 32;
		1396	break;
		1397	case CHIP_RV670:
		1398	tmp += 128;
		1399	break;
		1400	default:
		1401	break;
		1402	}
		1403	if (tmp > 256) {
		1404	tmp = 256;
		1405	}
		1406	WREG32(VGT_ES_PER_GS, 128);
		1407	WREG32(VGT_GS_PER_ES, tmp);
		1408	WREG32(VGT_GS_PER_VS, 2);
		1409	WREG32(VGT_GS_VERTEX_REUSE, 16);
		1410
		1411	/* more default values. 2D/3D driver should adjust as needed */
		1412	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		1413	WREG32(VGT_STRMOUT_EN, 0);
		1414	WREG32(SX_MISC, 0);
		1415	WREG32(PA_SC_MODE_CNTL, 0);
		1416	WREG32(PA_SC_AA_CONFIG, 0);
		1417	WREG32(PA_SC_LINE_STIPPLE, 0);
		1418	WREG32(SPI_INPUT_Z, 0);
		1419	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
		1420	WREG32(CB_COLOR7_FRAG, 0);
		1421
		1422	/* Clear render buffer base addresses */
		1423	WREG32(CB_COLOR0_BASE, 0);
		1424	WREG32(CB_COLOR1_BASE, 0);
		1425	WREG32(CB_COLOR2_BASE, 0);
		1426	WREG32(CB_COLOR3_BASE, 0);
		1427	WREG32(CB_COLOR4_BASE, 0);
		1428	WREG32(CB_COLOR5_BASE, 0);
		1429	WREG32(CB_COLOR6_BASE, 0);
		1430	WREG32(CB_COLOR7_BASE, 0);
		1431	WREG32(CB_COLOR7_FRAG, 0);
		1432
		1433	switch (rdev->family) {
		1434	case CHIP_RV610:
1268	serge	1435	case CHIP_RV620:
1221	serge	1436	case CHIP_RS780:
1268	serge	1437	case CHIP_RS880:
1221	serge	1438	tmp = TC_L2_SIZE(8);
		1439	break;
		1440	case CHIP_RV630:
		1441	case CHIP_RV635:
		1442	tmp = TC_L2_SIZE(4);
		1443	break;
		1444	case CHIP_R600:
		1445	tmp = TC_L2_SIZE(0) \| L2_DISABLE_LATE_HIT;
		1446	break;
		1447	default:
		1448	tmp = TC_L2_SIZE(0);
		1449	break;
		1450	}
		1451	WREG32(TC_CNTL, tmp);
		1452
		1453	tmp = RREG32(HDP_HOST_PATH_CNTL);
		1454	WREG32(HDP_HOST_PATH_CNTL, tmp);
		1455
		1456	tmp = RREG32(ARB_POP);
		1457	tmp \|= ENABLE_TC128;
		1458	WREG32(ARB_POP, tmp);
		1459
		1460	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1461	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA \|
		1462	NUM_CLIP_SEQ(3)));
		1463	WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
2997	Serge	1464	WREG32(VC_ENHANCE, 0);
1221	serge	1465	}
		1466
		1467
1128	serge	1468	/*
		1469	* Indirect registers accessor
		1470	*/
1221	serge	1471	u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1128	serge	1472	{
1221	serge	1473	u32 r;
1128	serge	1474
1221	serge	1475	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1476	(void)RREG32(PCIE_PORT_INDEX);
		1477	r = RREG32(PCIE_PORT_DATA);
1128	serge	1478	return r;
		1479	}
		1480
1221	serge	1481	void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1128	serge	1482	{
1221	serge	1483	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1484	(void)RREG32(PCIE_PORT_INDEX);
		1485	WREG32(PCIE_PORT_DATA, (v));
		1486	(void)RREG32(PCIE_PORT_DATA);
1128	serge	1487	}
1221	serge	1488
		1489	/*
		1490	* CP & Ring
		1491	*/
		1492	void r600_cp_stop(struct radeon_device *rdev)
		1493	{
3192	Serge	1494	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1221	serge	1495	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1963	serge	1496	WREG32(SCRATCH_UMSK, 0);
3192	Serge	1497	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1221	serge	1498	}
1413	serge	1499
		1500	int r600_init_microcode(struct radeon_device *rdev)
		1501	{
		1502	struct platform_device *pdev;
		1503	const char *chip_name;
		1504	const char *rlc_chip_name;
		1505	size_t pfp_req_size, me_req_size, rlc_req_size;
		1506	char fw_name[30];
		1507	int err;
		1508
		1509	DRM_DEBUG("\n");
		1510
		1511	pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
		1512	err = IS_ERR(pdev);
		1513	if (err) {
		1514	printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
		1515	return -EINVAL;
		1516	}
		1517
		1518	switch (rdev->family) {
		1519	case CHIP_R600:
		1520	chip_name = "R600";
		1521	rlc_chip_name = "R600";
		1522	break;
		1523	case CHIP_RV610:
		1524	chip_name = "RV610";
		1525	rlc_chip_name = "R600";
		1526	break;
		1527	case CHIP_RV630:
		1528	chip_name = "RV630";
		1529	rlc_chip_name = "R600";
		1530	break;
		1531	case CHIP_RV620:
		1532	chip_name = "RV620";
		1533	rlc_chip_name = "R600";
		1534	break;
		1535	case CHIP_RV635:
		1536	chip_name = "RV635";
		1537	rlc_chip_name = "R600";
		1538	break;
		1539	case CHIP_RV670:
		1540	chip_name = "RV670";
		1541	rlc_chip_name = "R600";
		1542	break;
		1543	case CHIP_RS780:
		1544	case CHIP_RS880:
		1545	chip_name = "RS780";
		1546	rlc_chip_name = "R600";
		1547	break;
		1548	case CHIP_RV770:
		1549	chip_name = "RV770";
		1550	rlc_chip_name = "R700";
		1551	break;
		1552	case CHIP_RV730:
		1553	case CHIP_RV740:
		1554	chip_name = "RV730";
		1555	rlc_chip_name = "R700";
		1556	break;
		1557	case CHIP_RV710:
		1558	chip_name = "RV710";
		1559	rlc_chip_name = "R700";
		1560	break;
1963	serge	1561	case CHIP_CEDAR:
		1562	chip_name = "CEDAR";
		1563	rlc_chip_name = "CEDAR";
		1564	break;
		1565	case CHIP_REDWOOD:
		1566	chip_name = "REDWOOD";
		1567	rlc_chip_name = "REDWOOD";
		1568	break;
		1569	case CHIP_JUNIPER:
		1570	chip_name = "JUNIPER";
		1571	rlc_chip_name = "JUNIPER";
		1572	break;
		1573	case CHIP_CYPRESS:
		1574	case CHIP_HEMLOCK:
		1575	chip_name = "CYPRESS";
		1576	rlc_chip_name = "CYPRESS";
		1577	break;
		1578	case CHIP_PALM:
		1579	chip_name = "PALM";
		1580	rlc_chip_name = "SUMO";
		1581	break;
1986	serge	1582	case CHIP_SUMO:
		1583	chip_name = "SUMO";
		1584	rlc_chip_name = "SUMO";
		1585	break;
		1586	case CHIP_SUMO2:
		1587	chip_name = "SUMO2";
		1588	rlc_chip_name = "SUMO";
		1589	break;
1413	serge	1590	default: BUG();
		1591	}
		1592
1963	serge	1593	if (rdev->family >= CHIP_CEDAR) {
		1594	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		1595	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		1596	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		1597	} else if (rdev->family >= CHIP_RV770) {
1413	serge	1598	pfp_req_size = R700_PFP_UCODE_SIZE * 4;
		1599	me_req_size = R700_PM4_UCODE_SIZE * 4;
		1600	rlc_req_size = R700_RLC_UCODE_SIZE * 4;
		1601	} else {
		1602	pfp_req_size = PFP_UCODE_SIZE * 4;
		1603	me_req_size = PM4_UCODE_SIZE * 12;
		1604	rlc_req_size = RLC_UCODE_SIZE * 4;
		1605	}
		1606
		1607	DRM_INFO("Loading %s Microcode\n", chip_name);
		1608
		1609	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
		1610	err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
		1611	if (err)
		1612	goto out;
		1613	if (rdev->pfp_fw->size != pfp_req_size) {
		1614	printk(KERN_ERR
		1615	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1616	rdev->pfp_fw->size, fw_name);
		1617	err = -EINVAL;
		1618	goto out;
		1619	}
		1620
		1621	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
		1622	err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
		1623	if (err)
		1624	goto out;
		1625	if (rdev->me_fw->size != me_req_size) {
		1626	printk(KERN_ERR
		1627	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1628	rdev->me_fw->size, fw_name);
		1629	err = -EINVAL;
		1630	}
		1631
		1632	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
		1633	err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
		1634	if (err)
		1635	goto out;
		1636	if (rdev->rlc_fw->size != rlc_req_size) {
		1637	printk(KERN_ERR
		1638	"r600_rlc: Bogus length %zu in firmware \"%s\"\n",
		1639	rdev->rlc_fw->size, fw_name);
		1640	err = -EINVAL;
		1641	}
		1642
		1643	out:
		1644	platform_device_unregister(pdev);
		1645
		1646	if (err) {
		1647	if (err != -EINVAL)
		1648	printk(KERN_ERR
		1649	"r600_cp: Failed to load firmware \"%s\"\n",
		1650	fw_name);
		1651	release_firmware(rdev->pfp_fw);
		1652	rdev->pfp_fw = NULL;
		1653	release_firmware(rdev->me_fw);
		1654	rdev->me_fw = NULL;
		1655	release_firmware(rdev->rlc_fw);
		1656	rdev->rlc_fw = NULL;
		1657	}
		1658	return err;
		1659	}
		1660
		1661	static int r600_cp_load_microcode(struct radeon_device *rdev)
		1662	{
		1663	const __be32 *fw_data;
		1664	int i;
		1665
		1666	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1667	return -EINVAL;
		1668
		1669	r600_cp_stop(rdev);
		1670
1963	serge	1671	WREG32(CP_RB_CNTL,
		1672	#ifdef __BIG_ENDIAN
		1673	BUF_SWAP_32BIT \|
		1674	#endif
		1675	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
1413	serge	1676
		1677	/* Reset cp */
		1678	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1679	RREG32(GRBM_SOFT_RESET);
		1680	mdelay(15);
		1681	WREG32(GRBM_SOFT_RESET, 0);
		1682
		1683	WREG32(CP_ME_RAM_WADDR, 0);
		1684
		1685	fw_data = (const __be32 *)rdev->me_fw->data;
		1686	WREG32(CP_ME_RAM_WADDR, 0);
		1687	for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
		1688	WREG32(CP_ME_RAM_DATA,
		1689	be32_to_cpup(fw_data++));
		1690
		1691	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1692	WREG32(CP_PFP_UCODE_ADDR, 0);
		1693	for (i = 0; i < PFP_UCODE_SIZE; i++)
		1694	WREG32(CP_PFP_UCODE_DATA,
		1695	be32_to_cpup(fw_data++));
		1696
		1697	WREG32(CP_PFP_UCODE_ADDR, 0);
		1698	WREG32(CP_ME_RAM_WADDR, 0);
		1699	WREG32(CP_ME_RAM_RADDR, 0);
		1700	return 0;
		1701	}
		1702
1221	serge	1703	int r600_cp_start(struct radeon_device *rdev)
		1704	{
2997	Serge	1705	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1221	serge	1706	int r;
		1707	uint32_t cp_me;
		1708
2997	Serge	1709	r = radeon_ring_lock(rdev, ring, 7);
1221	serge	1710	if (r) {
		1711	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1712	return r;
		1713	}
2997	Serge	1714	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1715	radeon_ring_write(ring, 0x1);
1963	serge	1716	if (rdev->family >= CHIP_RV770) {
2997	Serge	1717	radeon_ring_write(ring, 0x0);
		1718	radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
1963	serge	1719	} else {
2997	Serge	1720	radeon_ring_write(ring, 0x3);
		1721	radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
1221	serge	1722	}
2997	Serge	1723	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1724	radeon_ring_write(ring, 0);
		1725	radeon_ring_write(ring, 0);
		1726	radeon_ring_unlock_commit(rdev, ring);
1221	serge	1727
		1728	cp_me = 0xff;
		1729	WREG32(R_0086D8_CP_ME_CNTL, cp_me);
		1730	return 0;
		1731	}
1413	serge	1732
		1733	int r600_cp_resume(struct radeon_device *rdev)
		1734	{
2997	Serge	1735	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1413	serge	1736	u32 tmp;
		1737	u32 rb_bufsz;
		1738	int r;
		1739
		1740	/* Reset cp */
		1741	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1742	RREG32(GRBM_SOFT_RESET);
		1743	mdelay(15);
		1744	WREG32(GRBM_SOFT_RESET, 0);
		1745
		1746	/* Set ring buffer size */
2997	Serge	1747	rb_bufsz = drm_order(ring->ring_size / 8);
1963	serge	1748	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
1413	serge	1749	#ifdef __BIG_ENDIAN
		1750	tmp \|= BUF_SWAP_32BIT;
		1751	#endif
		1752	WREG32(CP_RB_CNTL, tmp);
2997	Serge	1753	WREG32(CP_SEM_WAIT_TIMER, 0x0);
1413	serge	1754
		1755	/* Set the write pointer delay */
		1756	WREG32(CP_RB_WPTR_DELAY, 0);
		1757
		1758	/* Initialize the ring buffer's read and write pointers */
		1759	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1760	WREG32(CP_RB_RPTR_WR, 0);
2997	Serge	1761	ring->wptr = 0;
		1762	WREG32(CP_RB_WPTR, ring->wptr);
1963	serge	1763
		1764	/* set the wb address whether it's enabled or not */
		1765	WREG32(CP_RB_RPTR_ADDR,
		1766	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1767	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1768	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1769
		1770	if (rdev->wb.enabled)
		1771	WREG32(SCRATCH_UMSK, 0xff);
		1772	else {
		1773	tmp \|= RB_NO_UPDATE;
		1774	WREG32(SCRATCH_UMSK, 0);
		1775	}
		1776
1413	serge	1777	mdelay(1);
		1778	WREG32(CP_RB_CNTL, tmp);
		1779
2997	Serge	1780	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1413	serge	1781	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1782
2997	Serge	1783	ring->rptr = RREG32(CP_RB_RPTR);
1413	serge	1784
		1785	r600_cp_start(rdev);
2997	Serge	1786	ring->ready = true;
		1787	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
1413	serge	1788	if (r) {
2997	Serge	1789	ring->ready = false;
1413	serge	1790	return r;
		1791	}
		1792	return 0;
		1793	}
		1794
2997	Serge	1795	void r600_ring_init(struct radeon_device rdev, struct radeon_ring ring, unsigned ring_size)
1221	serge	1796	{
1233	serge	1797	u32 rb_bufsz;
2997	Serge	1798	int r;
1221	serge	1799
1233	serge	1800	/* Align ring size */
		1801	rb_bufsz = drm_order(ring_size / 8);
		1802	ring_size = (1 << (rb_bufsz + 1)) * 4;
2997	Serge	1803	ring->ring_size = ring_size;
		1804	ring->align_mask = 16 - 1;
		1805
		1806	if (radeon_ring_supports_scratch_reg(rdev, ring)) {
		1807	r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
		1808	if (r) {
		1809	DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
		1810	ring->rptr_save_reg = 0;
		1811	}
		1812	}
1233	serge	1813	}
		1814
1963	serge	1815	void r600_cp_fini(struct radeon_device *rdev)
		1816	{
2997	Serge	1817	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	1818	r600_cp_stop(rdev);
2997	Serge	1819	radeon_ring_fini(rdev, ring);
		1820	radeon_scratch_free(rdev, ring->rptr_save_reg);
1963	serge	1821	}
1233	serge	1822
3192	Serge	1823	/*
		1824	* DMA
		1825	* Starting with R600, the GPU has an asynchronous
		1826	* DMA engine. The programming model is very similar
		1827	* to the 3D engine (ring buffer, IBs, etc.), but the
		1828	* DMA controller has it's own packet format that is
		1829	* different form the PM4 format used by the 3D engine.
		1830	* It supports copying data, writing embedded data,
		1831	* solid fills, and a number of other things. It also
		1832	* has support for tiling/detiling of buffers.
		1833	*/
		1834	/**
		1835	* r600_dma_stop - stop the async dma engine
		1836	*
		1837	* @rdev: radeon_device pointer
		1838	*
		1839	* Stop the async dma engine (r6xx-evergreen).
		1840	*/
		1841	void r600_dma_stop(struct radeon_device *rdev)
		1842	{
		1843	u32 rb_cntl = RREG32(DMA_RB_CNTL);
1963	serge	1844
3192	Serge	1845	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
		1846
		1847	rb_cntl &= ~DMA_RB_ENABLE;
		1848	WREG32(DMA_RB_CNTL, rb_cntl);
		1849
		1850	rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
		1851	}
		1852
		1853	/**
		1854	* r600_dma_resume - setup and start the async dma engine
		1855	*
		1856	* @rdev: radeon_device pointer
		1857	*
		1858	* Set up the DMA ring buffer and enable it. (r6xx-evergreen).
		1859	* Returns 0 for success, error for failure.
		1860	*/
		1861	int r600_dma_resume(struct radeon_device *rdev)
		1862	{
		1863	struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		1864	u32 rb_cntl, dma_cntl;
		1865	u32 rb_bufsz;
		1866	int r;
		1867
		1868	/* Reset dma */
		1869	if (rdev->family >= CHIP_RV770)
		1870	WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
		1871	else
		1872	WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
		1873	RREG32(SRBM_SOFT_RESET);
		1874	udelay(50);
		1875	WREG32(SRBM_SOFT_RESET, 0);
		1876
		1877	WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0);
		1878	WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
		1879
		1880	/* Set ring buffer size in dwords */
		1881	rb_bufsz = drm_order(ring->ring_size / 4);
		1882	rb_cntl = rb_bufsz << 1;
		1883	#ifdef __BIG_ENDIAN
		1884	rb_cntl \|= DMA_RB_SWAP_ENABLE \| DMA_RPTR_WRITEBACK_SWAP_ENABLE;
		1885	#endif
		1886	WREG32(DMA_RB_CNTL, rb_cntl);
		1887
		1888	/* Initialize the ring buffer's read and write pointers */
		1889	WREG32(DMA_RB_RPTR, 0);
		1890	WREG32(DMA_RB_WPTR, 0);
		1891
		1892	/* set the wb address whether it's enabled or not */
		1893	WREG32(DMA_RB_RPTR_ADDR_HI,
		1894	upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF);
		1895	WREG32(DMA_RB_RPTR_ADDR_LO,
		1896	((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC));
		1897
		1898	if (rdev->wb.enabled)
		1899	rb_cntl \|= DMA_RPTR_WRITEBACK_ENABLE;
		1900
		1901	WREG32(DMA_RB_BASE, ring->gpu_addr >> 8);
		1902
		1903	/* enable DMA IBs */
		1904	WREG32(DMA_IB_CNTL, DMA_IB_ENABLE);
		1905
		1906	dma_cntl = RREG32(DMA_CNTL);
		1907	dma_cntl &= ~CTXEMPTY_INT_ENABLE;
		1908	WREG32(DMA_CNTL, dma_cntl);
		1909
		1910	if (rdev->family >= CHIP_RV770)
		1911	WREG32(DMA_MODE, 1);
		1912
		1913	ring->wptr = 0;
		1914	WREG32(DMA_RB_WPTR, ring->wptr << 2);
		1915
		1916	ring->rptr = RREG32(DMA_RB_RPTR) >> 2;
		1917
		1918	WREG32(DMA_RB_CNTL, rb_cntl \| DMA_RB_ENABLE);
		1919
		1920	ring->ready = true;
		1921
		1922	r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring);
		1923	if (r) {
		1924	ring->ready = false;
		1925	return r;
		1926	}
		1927
		1928	radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
		1929
		1930	return 0;
		1931	}
		1932
		1933	/**
		1934	* r600_dma_fini - tear down the async dma engine
		1935	*
		1936	* @rdev: radeon_device pointer
		1937	*
		1938	* Stop the async dma engine and free the ring (r6xx-evergreen).
		1939	*/
		1940	void r600_dma_fini(struct radeon_device *rdev)
		1941	{
		1942	r600_dma_stop(rdev);
		1943	radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
		1944	}
		1945
1233	serge	1946	/*
		1947	* GPU scratch registers helpers function.
		1948	*/
		1949	void r600_scratch_init(struct radeon_device *rdev)
		1950	{
		1951	int i;
		1952
		1953	rdev->scratch.num_reg = 7;
1963	serge	1954	rdev->scratch.reg_base = SCRATCH_REG0;
1233	serge	1955	for (i = 0; i < rdev->scratch.num_reg; i++) {
		1956	rdev->scratch.free[i] = true;
1963	serge	1957	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
1233	serge	1958	}
		1959	}
1413	serge	1960
2997	Serge	1961	int r600_ring_test(struct radeon_device rdev, struct radeon_ring ring)
1413	serge	1962	{
		1963	uint32_t scratch;
		1964	uint32_t tmp = 0;
		1965	unsigned i;
		1966	int r;
		1967
		1968	r = radeon_scratch_get(rdev, &scratch);
		1969	if (r) {
		1970	DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
		1971	return r;
		1972	}
		1973	WREG32(scratch, 0xCAFEDEAD);
2997	Serge	1974	r = radeon_ring_lock(rdev, ring, 3);
1413	serge	1975	if (r) {
2997	Serge	1976	DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
1413	serge	1977	radeon_scratch_free(rdev, scratch);
		1978	return r;
		1979	}
2997	Serge	1980	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1981	radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		1982	radeon_ring_write(ring, 0xDEADBEEF);
		1983	radeon_ring_unlock_commit(rdev, ring);
1413	serge	1984	for (i = 0; i < rdev->usec_timeout; i++) {
		1985	tmp = RREG32(scratch);
		1986	if (tmp == 0xDEADBEEF)
		1987	break;
		1988	DRM_UDELAY(1);
		1989	}
		1990	if (i < rdev->usec_timeout) {
2997	Serge	1991	DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
1413	serge	1992	} else {
2997	Serge	1993	DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
		1994	ring->idx, scratch, tmp);
1413	serge	1995	r = -EINVAL;
		1996	}
		1997	radeon_scratch_free(rdev, scratch);
		1998	return r;
		1999	}
1963	serge	2000
3192	Serge	2001	/**
		2002	* r600_dma_ring_test - simple async dma engine test
		2003	*
		2004	* @rdev: radeon_device pointer
		2005	* @ring: radeon_ring structure holding ring information
		2006	*
		2007	* Test the DMA engine by writing using it to write an
		2008	* value to memory. (r6xx-SI).
		2009	* Returns 0 for success, error for failure.
		2010	*/
		2011	int r600_dma_ring_test(struct radeon_device *rdev,
		2012	struct radeon_ring *ring)
		2013	{
		2014	unsigned i;
		2015	int r;
		2016	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		2017	u32 tmp;
		2018
		2019	if (!ptr) {
		2020	DRM_ERROR("invalid vram scratch pointer\n");
		2021	return -EINVAL;
		2022	}
		2023
		2024	tmp = 0xCAFEDEAD;
		2025	writel(tmp, ptr);
		2026
		2027	r = radeon_ring_lock(rdev, ring, 4);
		2028	if (r) {
		2029	DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
		2030	return r;
		2031	}
		2032	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
		2033	radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
		2034	radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff);
		2035	radeon_ring_write(ring, 0xDEADBEEF);
		2036	radeon_ring_unlock_commit(rdev, ring);
		2037
		2038	for (i = 0; i < rdev->usec_timeout; i++) {
		2039	tmp = readl(ptr);
		2040	if (tmp == 0xDEADBEEF)
		2041	break;
		2042	DRM_UDELAY(1);
		2043	}
		2044
		2045	if (i < rdev->usec_timeout) {
		2046	DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
		2047	} else {
		2048	DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
		2049	ring->idx, tmp);
		2050	r = -EINVAL;
		2051	}
		2052	return r;
		2053	}
		2054
		2055	/*
		2056	* CP fences/semaphores
		2057	*/
		2058
1413	serge	2059	void r600_fence_ring_emit(struct radeon_device *rdev,
		2060	struct radeon_fence *fence)
		2061	{
2997	Serge	2062	struct radeon_ring *ring = &rdev->ring[fence->ring];
		2063
1963	serge	2064	if (rdev->wb.use_event) {
2997	Serge	2065	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		2066	/* flush read cache over gart */
		2067	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		2068	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA \|
		2069	PACKET3_VC_ACTION_ENA \|
		2070	PACKET3_SH_ACTION_ENA);
		2071	radeon_ring_write(ring, 0xFFFFFFFF);
		2072	radeon_ring_write(ring, 0);
		2073	radeon_ring_write(ring, 10); /* poll interval */
1963	serge	2074	/* EVENT_WRITE_EOP - flush caches, send int */
2997	Serge	2075	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		2076	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) \| EVENT_INDEX(5));
		2077	radeon_ring_write(ring, addr & 0xffffffff);
		2078	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		2079	radeon_ring_write(ring, fence->seq);
		2080	radeon_ring_write(ring, 0);
1963	serge	2081	} else {
2997	Serge	2082	/* flush read cache over gart */
		2083	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		2084	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA \|
		2085	PACKET3_VC_ACTION_ENA \|
		2086	PACKET3_SH_ACTION_ENA);
		2087	radeon_ring_write(ring, 0xFFFFFFFF);
		2088	radeon_ring_write(ring, 0);
		2089	radeon_ring_write(ring, 10); /* poll interval */
		2090	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
		2091	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) \| EVENT_INDEX(0));
1430	serge	2092	/* wait for 3D idle clean */
2997	Serge	2093	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2094	radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		2095	radeon_ring_write(ring, WAIT_3D_IDLE_bit \| WAIT_3D_IDLECLEAN_bit);
1413	serge	2096	/* Emit fence sequence & fire IRQ */
2997	Serge	2097	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2098	radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		2099	radeon_ring_write(ring, fence->seq);
1413	serge	2100	/* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2997	Serge	2101	radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
		2102	radeon_ring_write(ring, RB_INT_STAT);
1963	serge	2103	}
1413	serge	2104	}
1963	serge	2105
2997	Serge	2106	void r600_semaphore_ring_emit(struct radeon_device *rdev,
		2107	struct radeon_ring *ring,
		2108	struct radeon_semaphore *semaphore,
		2109	bool emit_wait)
		2110	{
		2111	uint64_t addr = semaphore->gpu_addr;
		2112	unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
		2113
		2114	if (rdev->family < CHIP_CAYMAN)
		2115	sel \|= PACKET3_SEM_WAIT_ON_SIGNAL;
		2116
		2117	radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
		2118	radeon_ring_write(ring, addr & 0xffffffff);
		2119	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| sel);
		2120	}
		2121
3192	Serge	2122	/*
		2123	* DMA fences/semaphores
		2124	*/
		2125
		2126	/**
		2127	* r600_dma_fence_ring_emit - emit a fence on the DMA ring
		2128	*
		2129	* @rdev: radeon_device pointer
		2130	* @fence: radeon fence object
		2131	*
		2132	* Add a DMA fence packet to the ring to write
		2133	* the fence seq number and DMA trap packet to generate
		2134	* an interrupt if needed (r6xx-r7xx).
		2135	*/
		2136	void r600_dma_fence_ring_emit(struct radeon_device *rdev,
		2137	struct radeon_fence *fence)
		2138	{
		2139	struct radeon_ring *ring = &rdev->ring[fence->ring];
		2140	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		2141
		2142	/* write the fence */
		2143	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0));
		2144	radeon_ring_write(ring, addr & 0xfffffffc);
		2145	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
		2146	radeon_ring_write(ring, lower_32_bits(fence->seq));
		2147	/* generate an interrupt */
		2148	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0));
		2149	}
		2150
		2151	/**
		2152	* r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring
		2153	*
		2154	* @rdev: radeon_device pointer
		2155	* @ring: radeon_ring structure holding ring information
		2156	* @semaphore: radeon semaphore object
		2157	* @emit_wait: wait or signal semaphore
		2158	*
		2159	* Add a DMA semaphore packet to the ring wait on or signal
		2160	* other rings (r6xx-SI).
		2161	*/
		2162	void r600_dma_semaphore_ring_emit(struct radeon_device *rdev,
		2163	struct radeon_ring *ring,
		2164	struct radeon_semaphore *semaphore,
		2165	bool emit_wait)
		2166	{
		2167	u64 addr = semaphore->gpu_addr;
		2168	u32 s = emit_wait ? 0 : 1;
		2169
		2170	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0));
		2171	radeon_ring_write(ring, addr & 0xfffffffc);
		2172	radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
		2173	}
		2174
2005	serge	2175	int r600_copy_blit(struct radeon_device *rdev,
2997	Serge	2176	uint64_t src_offset,
		2177	uint64_t dst_offset,
		2178	unsigned num_gpu_pages,
		2179	struct radeon_fence **fence)
2005	serge	2180	{
2997	Serge	2181	struct radeon_semaphore *sem = NULL;
		2182	struct radeon_sa_bo *vb = NULL;
2005	serge	2183	int r;
1963	serge	2184
2997	Serge	2185	r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem);
2005	serge	2186	if (r) {
		2187	return r;
		2188	}
2997	Serge	2189	r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb);
		2190	r600_blit_done_copy(rdev, fence, vb, sem);
2005	serge	2191	return 0;
		2192	}
		2193
3192	Serge	2194	/**
		2195	* r600_copy_dma - copy pages using the DMA engine
		2196	*
		2197	* @rdev: radeon_device pointer
		2198	* @src_offset: src GPU address
		2199	* @dst_offset: dst GPU address
		2200	* @num_gpu_pages: number of GPU pages to xfer
		2201	* @fence: radeon fence object
		2202	*
		2203	* Copy GPU paging using the DMA engine (r6xx).
		2204	* Used by the radeon ttm implementation to move pages if
		2205	* registered as the asic copy callback.
		2206	*/
		2207	int r600_copy_dma(struct radeon_device *rdev,
		2208	uint64_t src_offset, uint64_t dst_offset,
		2209	unsigned num_gpu_pages,
		2210	struct radeon_fence **fence)
		2211	{
		2212	struct radeon_semaphore *sem = NULL;
		2213	int ring_index = rdev->asic->copy.dma_ring_index;
		2214	struct radeon_ring *ring = &rdev->ring[ring_index];
		2215	u32 size_in_dw, cur_size_in_dw;
		2216	int i, num_loops;
		2217	int r = 0;
		2218
		2219	r = radeon_semaphore_create(rdev, &sem);
		2220	if (r) {
		2221	DRM_ERROR("radeon: moving bo (%d).\n", r);
		2222	return r;
		2223	}
		2224
		2225	size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
		2226	num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE);
		2227	r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8);
		2228	if (r) {
		2229	DRM_ERROR("radeon: moving bo (%d).\n", r);
		2230	radeon_semaphore_free(rdev, &sem, NULL);
		2231	return r;
		2232	}
		2233
		2234	if (radeon_fence_need_sync(*fence, ring->idx)) {
		2235	radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
		2236	ring->idx);
		2237	radeon_fence_note_sync(*fence, ring->idx);
		2238	} else {
		2239	radeon_semaphore_free(rdev, &sem, NULL);
		2240	}
		2241
		2242	for (i = 0; i < num_loops; i++) {
		2243	cur_size_in_dw = size_in_dw;
		2244	if (cur_size_in_dw > 0xFFFE)
		2245	cur_size_in_dw = 0xFFFE;
		2246	size_in_dw -= cur_size_in_dw;
		2247	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
		2248	radeon_ring_write(ring, dst_offset & 0xfffffffc);
		2249	radeon_ring_write(ring, src_offset & 0xfffffffc);
		2250	radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) \|
		2251	(upper_32_bits(src_offset) & 0xff)));
		2252	src_offset += cur_size_in_dw * 4;
		2253	dst_offset += cur_size_in_dw * 4;
		2254	}
		2255
		2256	r = radeon_fence_emit(rdev, fence, ring->idx);
		2257	if (r) {
		2258	radeon_ring_unlock_undo(rdev, ring);
		2259	return r;
		2260	}
		2261
		2262	radeon_ring_unlock_commit(rdev, ring);
		2263	radeon_semaphore_free(rdev, &sem, *fence);
		2264
		2265	return r;
		2266	}
		2267
1221	serge	2268	int r600_set_surface_reg(struct radeon_device *rdev, int reg,
		2269	uint32_t tiling_flags, uint32_t pitch,
		2270	uint32_t offset, uint32_t obj_size)
		2271	{
		2272	/* FIXME: implement */
		2273	return 0;
		2274	}
		2275
		2276	void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
		2277	{
		2278	/* FIXME: implement */
		2279	}
		2280
2997	Serge	2281	static int r600_startup(struct radeon_device *rdev)
1221	serge	2282	{
3192	Serge	2283	struct radeon_ring *ring;
1221	serge	2284	int r;
		2285
1963	serge	2286	/* enable pcie gen2 link */
		2287	r600_pcie_gen2_enable(rdev);
		2288
1413	serge	2289	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		2290	r = r600_init_microcode(rdev);
		2291	if (r) {
		2292	DRM_ERROR("Failed to load firmware!\n");
		2293	return r;
		2294	}
		2295	}
		2296
1221	serge	2297	r600_mc_program(rdev);
		2298	if (rdev->flags & RADEON_IS_AGP) {
		2299	r600_agp_enable(rdev);
		2300	} else {
		2301	r = r600_pcie_gart_enable(rdev);
		2302	if (r)
		2303	return r;
		2304	}
		2305	r600_gpu_init(rdev);
2005	serge	2306	r = r600_blit_init(rdev);
		2307	if (r) {
		2308	// r600_blit_fini(rdev);
2997	Serge	2309	rdev->asic->copy.copy = NULL;
2005	serge	2310	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		2311	}
1221	serge	2312
2005	serge	2313	/* allocate wb buffer */
		2314	r = radeon_wb_init(rdev);
		2315	if (r)
		2316	return r;
		2317
3192	Serge	2318	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
		2319	if (r) {
		2320	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		2321	return r;
		2322	}
		2323
		2324	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
		2325	if (r) {
		2326	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		2327	return r;
		2328	}
		2329
2005	serge	2330	/* Enable IRQ */
		2331	r = r600_irq_init(rdev);
		2332	if (r) {
		2333	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		2334	// radeon_irq_kms_fini(rdev);
		2335	return r;
		2336	}
		2337	r600_irq_set(rdev);
		2338
3192	Serge	2339	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2997	Serge	2340	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
		2341	R600_CP_RB_RPTR, R600_CP_RB_WPTR,
		2342	0, 0xfffff, RADEON_CP_PACKET2);
3192	Serge	2343	if (r)
		2344	return r;
2997	Serge	2345
3192	Serge	2346	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		2347	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
		2348	DMA_RB_RPTR, DMA_RB_WPTR,
		2349	2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1413	serge	2350	if (r)
		2351	return r;
3192	Serge	2352
1413	serge	2353	r = r600_cp_load_microcode(rdev);
		2354	if (r)
		2355	return r;
		2356	r = r600_cp_resume(rdev);
		2357	if (r)
		2358	return r;
1963	serge	2359
3192	Serge	2360	r = r600_dma_resume(rdev);
		2361	if (r)
		2362	return r;
		2363
		2364	r = radeon_ib_pool_init(rdev);
		2365	if (r) {
		2366	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		2367	return r;
		2368	}
1221	serge	2369	return 0;
		2370	}
		2371
		2372	void r600_vga_set_state(struct radeon_device *rdev, bool state)
		2373	{
		2374	uint32_t temp;
		2375
		2376	temp = RREG32(CONFIG_CNTL);
		2377	if (state == false) {
		2378	temp &= ~(1<<0);
		2379	temp \|= (1<<1);
		2380	} else {
		2381	temp &= ~(1<<1);
		2382	}
		2383	WREG32(CONFIG_CNTL, temp);
		2384	}
		2385
		2386
		2387
		2388
		2389
		2390	/* Plan is to move initialization in that function and use
		2391	* helper function so that radeon_device_init pretty much
		2392	* do nothing more than calling asic specific function. This
		2393	* should also allow to remove a bunch of callback function
		2394	* like vram_info.
		2395	*/
		2396	int r600_init(struct radeon_device *rdev)
		2397	{
		2398	int r;
		2399
		2400	if (r600_debugfs_mc_info_init(rdev)) {
		2401	DRM_ERROR("Failed to register debugfs file for mc !\n");
		2402	}
		2403	/* Read BIOS */
		2404	if (!radeon_get_bios(rdev)) {
		2405	if (ASIC_IS_AVIVO(rdev))
		2406	return -EINVAL;
		2407	}
		2408	/* Must be an ATOMBIOS */
		2409	if (!rdev->is_atom_bios) {
		2410	dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
		2411	return -EINVAL;
		2412	}
		2413	r = radeon_atombios_init(rdev);
		2414	if (r)
		2415	return r;
		2416	/* Post card if necessary */
1963	serge	2417	if (!radeon_card_posted(rdev)) {
1321	serge	2418	if (!rdev->bios) {
		2419	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		2420	return -EINVAL;
		2421	}
1221	serge	2422	DRM_INFO("GPU not posted. posting now...\n");
		2423	atom_asic_init(rdev->mode_info.atom_context);
		2424	}
		2425	/* Initialize scratch registers */
		2426	r600_scratch_init(rdev);
		2427	/* Initialize surface registers */
		2428	radeon_surface_init(rdev);
1268	serge	2429	/* Initialize clocks */
1221	serge	2430	radeon_get_clock_info(rdev->ddev);
		2431	/* Fence driver */
2004	serge	2432	r = radeon_fence_driver_init(rdev);
		2433	if (r)
		2434	return r;
1403	serge	2435	if (rdev->flags & RADEON_IS_AGP) {
		2436	r = radeon_agp_init(rdev);
		2437	if (r)
		2438	radeon_agp_disable(rdev);
		2439	}
1221	serge	2440	r = r600_mc_init(rdev);
		2441	if (r)
		2442	return r;
		2443	/* Memory manager */
1321	serge	2444	r = radeon_bo_init(rdev);
1221	serge	2445	if (r)
		2446	return r;
1321	serge	2447
2004	serge	2448	r = radeon_irq_kms_init(rdev);
		2449	if (r)
		2450	return r;
1321	serge	2451
2997	Serge	2452	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
		2453	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1221	serge	2454
3192	Serge	2455	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
		2456	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
		2457
2004	serge	2458	rdev->ih.ring_obj = NULL;
		2459	r600_ih_ring_init(rdev, 64 * 1024);
1221	serge	2460
		2461	r = r600_pcie_gart_init(rdev);
		2462	if (r)
		2463	return r;
		2464
1321	serge	2465	rdev->accel_working = true;
1221	serge	2466	r = r600_startup(rdev);
		2467	if (r) {
1428	serge	2468	dev_err(rdev->dev, "disabling GPU acceleration\n");
1221	serge	2469	r600_pcie_gart_fini(rdev);
		2470	rdev->accel_working = false;
		2471	}
2005	serge	2472
1221	serge	2473	return 0;
		2474	}
		2475
2004	serge	2476	/*
		2477	* CS stuff
		2478	*/
		2479	void r600_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		2480	{
2997	Serge	2481	struct radeon_ring *ring = &rdev->ring[ib->ring];
		2482	u32 next_rptr;
		2483
		2484	if (ring->rptr_save_reg) {
		2485	next_rptr = ring->wptr + 3 + 4;
		2486	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2487	radeon_ring_write(ring, ((ring->rptr_save_reg -
		2488	PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		2489	radeon_ring_write(ring, next_rptr);
		2490	} else if (rdev->wb.enabled) {
		2491	next_rptr = ring->wptr + 5 + 4;
		2492	radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
		2493	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		2494	radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) \| (1 << 18));
		2495	radeon_ring_write(ring, next_rptr);
		2496	radeon_ring_write(ring, 0);
		2497	}
		2498
		2499	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		2500	radeon_ring_write(ring,
2004	serge	2501	#ifdef __BIG_ENDIAN
		2502	(2 << 0) \|
		2503	#endif
		2504	(ib->gpu_addr & 0xFFFFFFFC));
2997	Serge	2505	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
		2506	radeon_ring_write(ring, ib->length_dw);
2004	serge	2507	}
		2508
2997	Serge	2509	int r600_ib_test(struct radeon_device rdev, struct radeon_ring ring)
2004	serge	2510	{
2997	Serge	2511	struct radeon_ib ib;
2004	serge	2512	uint32_t scratch;
		2513	uint32_t tmp = 0;
		2514	unsigned i;
		2515	int r;
		2516
		2517	r = radeon_scratch_get(rdev, &scratch);
		2518	if (r) {
		2519	DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
		2520	return r;
		2521	}
		2522	WREG32(scratch, 0xCAFEDEAD);
2997	Serge	2523	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
2004	serge	2524	if (r) {
		2525	DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2997	Serge	2526	goto free_scratch;
2004	serge	2527	}
2997	Serge	2528	ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
		2529	ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		2530	ib.ptr[2] = 0xDEADBEEF;
		2531	ib.length_dw = 3;
		2532	r = radeon_ib_schedule(rdev, &ib, NULL);
2004	serge	2533	if (r) {
		2534	DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2997	Serge	2535	goto free_ib;
2004	serge	2536	}
2997	Serge	2537	r = radeon_fence_wait(ib.fence, false);
2004	serge	2538	if (r) {
		2539	DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2997	Serge	2540	goto free_ib;
2004	serge	2541	}
		2542	for (i = 0; i < rdev->usec_timeout; i++) {
		2543	tmp = RREG32(scratch);
		2544	if (tmp == 0xDEADBEEF)
		2545	break;
		2546	DRM_UDELAY(1);
		2547	}
		2548	if (i < rdev->usec_timeout) {
2997	Serge	2549	DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
2004	serge	2550	} else {
		2551	DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
		2552	scratch, tmp);
		2553	r = -EINVAL;
		2554	}
2997	Serge	2555	free_ib:
		2556	radeon_ib_free(rdev, &ib);
		2557	free_scratch:
2004	serge	2558	radeon_scratch_free(rdev, scratch);
		2559	return r;
		2560	}
		2561
3192	Serge	2562	/**
		2563	* r600_dma_ib_test - test an IB on the DMA engine
		2564	*
		2565	* @rdev: radeon_device pointer
		2566	* @ring: radeon_ring structure holding ring information
		2567	*
		2568	* Test a simple IB in the DMA ring (r6xx-SI).
		2569	* Returns 0 on success, error on failure.
		2570	*/
		2571	int r600_dma_ib_test(struct radeon_device rdev, struct radeon_ring ring)
		2572	{
		2573	struct radeon_ib ib;
		2574	unsigned i;
		2575	int r;
		2576	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		2577	u32 tmp = 0;
		2578
		2579	ENTER();
		2580
		2581	if (!ptr) {
		2582	DRM_ERROR("invalid vram scratch pointer\n");
		2583	return -EINVAL;
		2584	}
		2585
		2586	tmp = 0xCAFEDEAD;
		2587	writel(tmp, ptr);
		2588
		2589	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
		2590	if (r) {
		2591	DRM_ERROR("radeon: failed to get ib (%d).\n", r);
		2592	return r;
		2593	}
		2594
		2595	ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1);
		2596	ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
		2597	ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff;
		2598	ib.ptr[3] = 0xDEADBEEF;
		2599	ib.length_dw = 4;
		2600
		2601	r = radeon_ib_schedule(rdev, &ib, NULL);
		2602	if (r) {
		2603	radeon_ib_free(rdev, &ib);
		2604	DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
		2605	return r;
		2606	}
		2607	r = radeon_fence_wait(ib.fence, false);
		2608	if (r) {
		2609	DRM_ERROR("radeon: fence wait failed (%d).\n", r);
		2610	return r;
		2611	}
		2612	for (i = 0; i < rdev->usec_timeout; i++) {
		2613	tmp = readl(ptr);
		2614	if (tmp == 0xDEADBEEF)
		2615	break;
		2616	DRM_UDELAY(1);
		2617	}
		2618	if (i < rdev->usec_timeout) {
		2619	DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
		2620	} else {
		2621	DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
		2622	r = -EINVAL;
		2623	}
		2624	radeon_ib_free(rdev, &ib);
		2625
		2626	LEAVE();
		2627
		2628	return r;
		2629	}
		2630
		2631	/**
		2632	* r600_dma_ring_ib_execute - Schedule an IB on the DMA engine
		2633	*
		2634	* @rdev: radeon_device pointer
		2635	* @ib: IB object to schedule
		2636	*
		2637	* Schedule an IB in the DMA ring (r6xx-r7xx).
		2638	*/
		2639	void r600_dma_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		2640	{
		2641	struct radeon_ring *ring = &rdev->ring[ib->ring];
		2642
		2643	if (rdev->wb.enabled) {
		2644	u32 next_rptr = ring->wptr + 4;
		2645	while ((next_rptr & 7) != 5)
		2646	next_rptr++;
		2647	next_rptr += 3;
		2648	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
		2649	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		2650	radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
		2651	radeon_ring_write(ring, next_rptr);
		2652	}
		2653
		2654	/* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
		2655	* Pad as necessary with NOPs.
		2656	*/
		2657	while ((ring->wptr & 7) != 5)
		2658	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
		2659	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0));
		2660	radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
		2661	radeon_ring_write(ring, (ib->length_dw << 16) \| (upper_32_bits(ib->gpu_addr) & 0xFF));
		2662
		2663	}
		2664
2004	serge	2665	/*
		2666	* Interrupts
		2667	*
		2668	* Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
		2669	* the same as the CP ring buffer, but in reverse. Rather than the CPU
		2670	* writing to the ring and the GPU consuming, the GPU writes to the ring
		2671	* and host consumes. As the host irq handler processes interrupts, it
		2672	* increments the rptr. When the rptr catches up with the wptr, all the
		2673	* current interrupts have been processed.
		2674	*/
		2675
		2676	void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
		2677	{
		2678	u32 rb_bufsz;
		2679
		2680	/* Align ring size */
		2681	rb_bufsz = drm_order(ring_size / 4);
		2682	ring_size = (1 << rb_bufsz) * 4;
		2683	rdev->ih.ring_size = ring_size;
		2684	rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
		2685	rdev->ih.rptr = 0;
		2686	}
		2687
2997	Serge	2688	int r600_ih_ring_alloc(struct radeon_device *rdev)
2004	serge	2689	{
		2690	int r;
		2691
		2692	/* Allocate ring buffer */
		2693	if (rdev->ih.ring_obj == NULL) {
		2694	r = radeon_bo_create(rdev, rdev->ih.ring_size,
		2695	PAGE_SIZE, true,
		2696	RADEON_GEM_DOMAIN_GTT,
2997	Serge	2697	NULL, &rdev->ih.ring_obj);
2004	serge	2698	if (r) {
		2699	DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
		2700	return r;
		2701	}
		2702	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		2703	if (unlikely(r != 0))
		2704	return r;
		2705	r = radeon_bo_pin(rdev->ih.ring_obj,
		2706	RADEON_GEM_DOMAIN_GTT,
		2707	&rdev->ih.gpu_addr);
		2708	if (r) {
		2709	radeon_bo_unreserve(rdev->ih.ring_obj);
		2710	DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
		2711	return r;
		2712	}
		2713	r = radeon_bo_kmap(rdev->ih.ring_obj,
		2714	(void **)&rdev->ih.ring);
		2715	radeon_bo_unreserve(rdev->ih.ring_obj);
		2716	if (r) {
		2717	DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
		2718	return r;
		2719	}
		2720	}
		2721	return 0;
		2722	}
		2723
2997	Serge	2724	void r600_ih_ring_fini(struct radeon_device *rdev)
2004	serge	2725	{
		2726	int r;
		2727	if (rdev->ih.ring_obj) {
		2728	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		2729	if (likely(r == 0)) {
		2730	radeon_bo_kunmap(rdev->ih.ring_obj);
		2731	radeon_bo_unpin(rdev->ih.ring_obj);
		2732	radeon_bo_unreserve(rdev->ih.ring_obj);
		2733	}
		2734	radeon_bo_unref(&rdev->ih.ring_obj);
		2735	rdev->ih.ring = NULL;
		2736	rdev->ih.ring_obj = NULL;
		2737	}
		2738	}
		2739
		2740	void r600_rlc_stop(struct radeon_device *rdev)
		2741	{
		2742
		2743	if ((rdev->family >= CHIP_RV770) &&
		2744	(rdev->family <= CHIP_RV740)) {
		2745	/* r7xx asics need to soft reset RLC before halting */
		2746	WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
		2747	RREG32(SRBM_SOFT_RESET);
2997	Serge	2748	mdelay(15);
2004	serge	2749	WREG32(SRBM_SOFT_RESET, 0);
		2750	RREG32(SRBM_SOFT_RESET);
		2751	}
		2752
		2753	WREG32(RLC_CNTL, 0);
		2754	}
		2755
		2756	static void r600_rlc_start(struct radeon_device *rdev)
		2757	{
		2758	WREG32(RLC_CNTL, RLC_ENABLE);
		2759	}
		2760
		2761	static int r600_rlc_init(struct radeon_device *rdev)
		2762	{
		2763	u32 i;
		2764	const __be32 *fw_data;
		2765
		2766	if (!rdev->rlc_fw)
		2767	return -EINVAL;
		2768
		2769	r600_rlc_stop(rdev);
		2770
2997	Serge	2771	WREG32(RLC_HB_CNTL, 0);
		2772
		2773	if (rdev->family == CHIP_ARUBA) {
		2774	WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		2775	WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		2776	}
		2777	if (rdev->family <= CHIP_CAYMAN) {
2004	serge	2778	WREG32(RLC_HB_BASE, 0);
		2779	WREG32(RLC_HB_RPTR, 0);
		2780	WREG32(RLC_HB_WPTR, 0);
2997	Serge	2781	}
2004	serge	2782	if (rdev->family <= CHIP_CAICOS) {
		2783	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
		2784	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
		2785	}
		2786	WREG32(RLC_MC_CNTL, 0);
		2787	WREG32(RLC_UCODE_CNTL, 0);
		2788
		2789	fw_data = (const __be32 *)rdev->rlc_fw->data;
2997	Serge	2790	if (rdev->family >= CHIP_ARUBA) {
		2791	for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
		2792	WREG32(RLC_UCODE_ADDR, i);
		2793	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2794	}
		2795	} else if (rdev->family >= CHIP_CAYMAN) {
2004	serge	2796	for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
		2797	WREG32(RLC_UCODE_ADDR, i);
		2798	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2799	}
		2800	} else if (rdev->family >= CHIP_CEDAR) {
		2801	for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
		2802	WREG32(RLC_UCODE_ADDR, i);
		2803	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2804	}
		2805	} else if (rdev->family >= CHIP_RV770) {
		2806	for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
		2807	WREG32(RLC_UCODE_ADDR, i);
		2808	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2809	}
		2810	} else {
		2811	for (i = 0; i < RLC_UCODE_SIZE; i++) {
		2812	WREG32(RLC_UCODE_ADDR, i);
		2813	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2814	}
		2815	}
		2816	WREG32(RLC_UCODE_ADDR, 0);
		2817
		2818	r600_rlc_start(rdev);
		2819
		2820	return 0;
		2821	}
		2822
		2823	static void r600_enable_interrupts(struct radeon_device *rdev)
		2824	{
		2825	u32 ih_cntl = RREG32(IH_CNTL);
		2826	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		2827
		2828	ih_cntl \|= ENABLE_INTR;
		2829	ih_rb_cntl \|= IH_RB_ENABLE;
		2830	WREG32(IH_CNTL, ih_cntl);
		2831	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2832	rdev->ih.enabled = true;
		2833	}
		2834
		2835	void r600_disable_interrupts(struct radeon_device *rdev)
		2836	{
		2837	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		2838	u32 ih_cntl = RREG32(IH_CNTL);
		2839
		2840	ih_rb_cntl &= ~IH_RB_ENABLE;
		2841	ih_cntl &= ~ENABLE_INTR;
		2842	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2843	WREG32(IH_CNTL, ih_cntl);
		2844	/* set rptr, wptr to 0 */
		2845	WREG32(IH_RB_RPTR, 0);
		2846	WREG32(IH_RB_WPTR, 0);
		2847	rdev->ih.enabled = false;
		2848	rdev->ih.rptr = 0;
		2849	}
		2850
1963	serge	2851	static void r600_disable_interrupt_state(struct radeon_device *rdev)
		2852	{
		2853	u32 tmp;
1221	serge	2854
1963	serge	2855	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
3192	Serge	2856	tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
		2857	WREG32(DMA_CNTL, tmp);
1963	serge	2858	WREG32(GRBM_INT_CNTL, 0);
		2859	WREG32(DxMODE_INT_MASK, 0);
		2860	WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
		2861	WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
		2862	if (ASIC_IS_DCE3(rdev)) {
		2863	WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
		2864	WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
		2865	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2866	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2867	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2868	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2869	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2870	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2871	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2872	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2873	if (ASIC_IS_DCE32(rdev)) {
		2874	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2875	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2876	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2877	WREG32(DC_HPD6_INT_CONTROL, tmp);
2997	Serge	2878	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2879	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
		2880	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2881	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
		2882	} else {
		2883	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2884	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
		2885	tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2886	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
1963	serge	2887	}
		2888	} else {
		2889	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		2890	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2891	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2892	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		2893	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2894	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		2895	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2896	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2997	Serge	2897	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2898	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
		2899	tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2900	WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
1963	serge	2901	}
		2902	}
1221	serge	2903
2004	serge	2904	int r600_irq_init(struct radeon_device *rdev)
		2905	{
		2906	int ret = 0;
		2907	int rb_bufsz;
		2908	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
1221	serge	2909
2004	serge	2910	/* allocate ring */
		2911	ret = r600_ih_ring_alloc(rdev);
		2912	if (ret)
		2913	return ret;
1221	serge	2914
2004	serge	2915	/* disable irqs */
		2916	r600_disable_interrupts(rdev);
1221	serge	2917
2004	serge	2918	/* init rlc */
		2919	ret = r600_rlc_init(rdev);
		2920	if (ret) {
		2921	r600_ih_ring_fini(rdev);
		2922	return ret;
		2923	}
1221	serge	2924
2004	serge	2925	/* setup interrupt control */
		2926	/* set dummy read address to ring address */
		2927	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
		2928	interrupt_cntl = RREG32(INTERRUPT_CNTL);
		2929	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
		2930	* IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
		2931	*/
		2932	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
		2933	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
		2934	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
		2935	WREG32(INTERRUPT_CNTL, interrupt_cntl);
1221	serge	2936
2004	serge	2937	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
		2938	rb_bufsz = drm_order(rdev->ih.ring_size / 4);
1221	serge	2939
2004	serge	2940	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE \|
		2941	IH_WPTR_OVERFLOW_CLEAR \|
		2942	(rb_bufsz << 1));
1963	serge	2943
2004	serge	2944	if (rdev->wb.enabled)
		2945	ih_rb_cntl \|= IH_WPTR_WRITEBACK_ENABLE;
		2946
		2947	/* set the writeback address whether it's enabled or not */
		2948	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
		2949	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
		2950
		2951	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2952
		2953	/* set rptr, wptr to 0 */
		2954	WREG32(IH_RB_RPTR, 0);
		2955	WREG32(IH_RB_WPTR, 0);
		2956
		2957	/* Default settings for IH_CNTL (disabled at first) */
		2958	ih_cntl = MC_WRREQ_CREDIT(0x10) \| MC_WR_CLEAN_CNT(0x10);
		2959	/* RPTR_REARM only works if msi's are enabled */
		2960	if (rdev->msi_enabled)
		2961	ih_cntl \|= RPTR_REARM;
		2962	WREG32(IH_CNTL, ih_cntl);
		2963
		2964	/* force the active interrupt state to all disabled */
		2965	if (rdev->family >= CHIP_CEDAR)
		2966	evergreen_disable_interrupt_state(rdev);
		2967	else
		2968	r600_disable_interrupt_state(rdev);
		2969
2997	Serge	2970	/* at this point everything should be setup correctly to enable master */
		2971	pci_set_master(rdev->pdev);
		2972
2004	serge	2973	/* enable irqs */
		2974	r600_enable_interrupts(rdev);
		2975
		2976	return ret;
		2977	}
		2978	int r600_irq_set(struct radeon_device *rdev)
		2979	{
		2980	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
		2981	u32 mode_int = 0;
		2982	u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
		2983	u32 grbm_int_cntl = 0;
2997	Serge	2984	u32 hdmi0, hdmi1;
2004	serge	2985	u32 d1grph = 0, d2grph = 0;
3192	Serge	2986	u32 dma_cntl;
2004	serge	2987
		2988	if (!rdev->irq.installed) {
		2989	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		2990	return -EINVAL;
		2991	}
		2992	/* don't enable anything if the ih is disabled */
		2993	if (!rdev->ih.enabled) {
		2994	r600_disable_interrupts(rdev);
		2995	/* force the active interrupt state to all disabled */
		2996	r600_disable_interrupt_state(rdev);
		2997	return 0;
		2998	}
		2999
		3000	if (ASIC_IS_DCE3(rdev)) {
		3001	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3002	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3003	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3004	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3005	if (ASIC_IS_DCE32(rdev)) {
		3006	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3007	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2997	Serge	3008	hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		3009	hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		3010	} else {
		3011	hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3012	hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3013	}
		3014	} else {
		3015	hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3016	hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3017	hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2997	Serge	3018	hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3019	hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3020	}
3192	Serge	3021	dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
2004	serge	3022
2997	Serge	3023	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
2004	serge	3024	DRM_DEBUG("r600_irq_set: sw int\n");
		3025	cp_int_cntl \|= RB_INT_ENABLE;
		3026	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		3027	}
3192	Serge	3028
		3029	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
		3030	DRM_DEBUG("r600_irq_set: sw int dma\n");
		3031	dma_cntl \|= TRAP_ENABLE;
		3032	}
		3033
2004	serge	3034	if (rdev->irq.crtc_vblank_int[0] \|\|
2997	Serge	3035	atomic_read(&rdev->irq.pflip[0])) {
2004	serge	3036	DRM_DEBUG("r600_irq_set: vblank 0\n");
		3037	mode_int \|= D1MODE_VBLANK_INT_MASK;
		3038	}
		3039	if (rdev->irq.crtc_vblank_int[1] \|\|
2997	Serge	3040	atomic_read(&rdev->irq.pflip[1])) {
2004	serge	3041	DRM_DEBUG("r600_irq_set: vblank 1\n");
		3042	mode_int \|= D2MODE_VBLANK_INT_MASK;
		3043	}
		3044	if (rdev->irq.hpd[0]) {
		3045	DRM_DEBUG("r600_irq_set: hpd 1\n");
		3046	hpd1 \|= DC_HPDx_INT_EN;
		3047	}
		3048	if (rdev->irq.hpd[1]) {
		3049	DRM_DEBUG("r600_irq_set: hpd 2\n");
		3050	hpd2 \|= DC_HPDx_INT_EN;
		3051	}
		3052	if (rdev->irq.hpd[2]) {
		3053	DRM_DEBUG("r600_irq_set: hpd 3\n");
		3054	hpd3 \|= DC_HPDx_INT_EN;
		3055	}
		3056	if (rdev->irq.hpd[3]) {
		3057	DRM_DEBUG("r600_irq_set: hpd 4\n");
		3058	hpd4 \|= DC_HPDx_INT_EN;
		3059	}
		3060	if (rdev->irq.hpd[4]) {
		3061	DRM_DEBUG("r600_irq_set: hpd 5\n");
		3062	hpd5 \|= DC_HPDx_INT_EN;
		3063	}
		3064	if (rdev->irq.hpd[5]) {
		3065	DRM_DEBUG("r600_irq_set: hpd 6\n");
		3066	hpd6 \|= DC_HPDx_INT_EN;
		3067	}
2997	Serge	3068	if (rdev->irq.afmt[0]) {
		3069	DRM_DEBUG("r600_irq_set: hdmi 0\n");
		3070	hdmi0 \|= HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3071	}
2997	Serge	3072	if (rdev->irq.afmt[1]) {
		3073	DRM_DEBUG("r600_irq_set: hdmi 0\n");
		3074	hdmi1 \|= HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3075	}
		3076
		3077	WREG32(CP_INT_CNTL, cp_int_cntl);
3192	Serge	3078	WREG32(DMA_CNTL, dma_cntl);
2004	serge	3079	WREG32(DxMODE_INT_MASK, mode_int);
		3080	WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
		3081	WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
		3082	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		3083	if (ASIC_IS_DCE3(rdev)) {
		3084	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		3085	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		3086	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		3087	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		3088	if (ASIC_IS_DCE32(rdev)) {
		3089	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		3090	WREG32(DC_HPD6_INT_CONTROL, hpd6);
2997	Serge	3091	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
		3092	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
		3093	} else {
		3094	WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
		3095	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
2004	serge	3096	}
		3097	} else {
		3098	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
		3099	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
		3100	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
2997	Serge	3101	WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
		3102	WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
2004	serge	3103	}
		3104
		3105	return 0;
		3106	}
		3107
2997	Serge	3108	static void r600_irq_ack(struct radeon_device *rdev)
2004	serge	3109	{
		3110	u32 tmp;
		3111
		3112	if (ASIC_IS_DCE3(rdev)) {
		3113	rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
		3114	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
		3115	rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
2997	Serge	3116	if (ASIC_IS_DCE32(rdev)) {
		3117	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
		3118	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
		3119	} else {
		3120	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
		3121	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
		3122	}
2004	serge	3123	} else {
		3124	rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		3125	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		3126	rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
2997	Serge	3127	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
		3128	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
2004	serge	3129	}
		3130	rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
		3131	rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
		3132
		3133	if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		3134	WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		3135	if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		3136	WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		3137	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
		3138	WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		3139	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
		3140	WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		3141	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
		3142	WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		3143	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
		3144	WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		3145	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		3146	if (ASIC_IS_DCE3(rdev)) {
		3147	tmp = RREG32(DC_HPD1_INT_CONTROL);
		3148	tmp \|= DC_HPDx_INT_ACK;
		3149	WREG32(DC_HPD1_INT_CONTROL, tmp);
		3150	} else {
		3151	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		3152	tmp \|= DC_HPDx_INT_ACK;
		3153	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		3154	}
		3155	}
		3156	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		3157	if (ASIC_IS_DCE3(rdev)) {
		3158	tmp = RREG32(DC_HPD2_INT_CONTROL);
		3159	tmp \|= DC_HPDx_INT_ACK;
		3160	WREG32(DC_HPD2_INT_CONTROL, tmp);
		3161	} else {
		3162	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		3163	tmp \|= DC_HPDx_INT_ACK;
		3164	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		3165	}
		3166	}
		3167	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		3168	if (ASIC_IS_DCE3(rdev)) {
		3169	tmp = RREG32(DC_HPD3_INT_CONTROL);
		3170	tmp \|= DC_HPDx_INT_ACK;
		3171	WREG32(DC_HPD3_INT_CONTROL, tmp);
		3172	} else {
		3173	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		3174	tmp \|= DC_HPDx_INT_ACK;
		3175	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		3176	}
		3177	}
		3178	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		3179	tmp = RREG32(DC_HPD4_INT_CONTROL);
		3180	tmp \|= DC_HPDx_INT_ACK;
		3181	WREG32(DC_HPD4_INT_CONTROL, tmp);
		3182	}
		3183	if (ASIC_IS_DCE32(rdev)) {
		3184	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		3185	tmp = RREG32(DC_HPD5_INT_CONTROL);
		3186	tmp \|= DC_HPDx_INT_ACK;
		3187	WREG32(DC_HPD5_INT_CONTROL, tmp);
		3188	}
		3189	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		3190	tmp = RREG32(DC_HPD5_INT_CONTROL);
		3191	tmp \|= DC_HPDx_INT_ACK;
		3192	WREG32(DC_HPD6_INT_CONTROL, tmp);
		3193	}
2997	Serge	3194	if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
		3195	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
		3196	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		3197	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
		3198	}
		3199	if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
		3200	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
		3201	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		3202	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
2004	serge	3203	}
2997	Serge	3204	} else {
		3205	if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
		3206	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
		3207	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		3208	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
2004	serge	3209	}
2997	Serge	3210	if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
2004	serge	3211	if (ASIC_IS_DCE3(rdev)) {
2997	Serge	3212	tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
		3213	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		3214	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
		3215	} else {
		3216	tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
		3217	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		3218	WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
2004	serge	3219	}
		3220	}
		3221	}
		3222	}
		3223
3192	Serge	3224	void r600_irq_disable(struct radeon_device *rdev)
		3225	{
		3226	r600_disable_interrupts(rdev);
		3227	/* Wait and acknowledge irq */
		3228	mdelay(1);
		3229	r600_irq_ack(rdev);
		3230	r600_disable_interrupt_state(rdev);
		3231	}
		3232
2997	Serge	3233	static u32 r600_get_ih_wptr(struct radeon_device *rdev)
2004	serge	3234	{
		3235	u32 wptr, tmp;
		3236
		3237	if (rdev->wb.enabled)
		3238	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		3239	else
		3240	wptr = RREG32(IH_RB_WPTR);
		3241
		3242	if (wptr & RB_OVERFLOW) {
		3243	/* When a ring buffer overflow happen start parsing interrupt
		3244	* from the last not overwritten vector (wptr + 16). Hopefully
		3245	* this should allow us to catchup.
		3246	*/
		3247	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
		3248	wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
		3249	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		3250	tmp = RREG32(IH_RB_CNTL);
		3251	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		3252	WREG32(IH_RB_CNTL, tmp);
		3253	}
		3254	return (wptr & rdev->ih.ptr_mask);
		3255	}
		3256
		3257	/* r600 IV Ring
		3258	* Each IV ring entry is 128 bits:
		3259	* [7:0] - interrupt source id
		3260	* [31:8] - reserved
		3261	* [59:32] - interrupt source data
		3262	* [127:60] - reserved
		3263	*
		3264	* The basic interrupt vector entries
		3265	* are decoded as follows:
		3266	* src_id src_data description
		3267	* 1 0 D1 Vblank
		3268	* 1 1 D1 Vline
		3269	* 5 0 D2 Vblank
		3270	* 5 1 D2 Vline
		3271	* 19 0 FP Hot plug detection A
		3272	* 19 1 FP Hot plug detection B
		3273	* 19 2 DAC A auto-detection
		3274	* 19 3 DAC B auto-detection
		3275	* 21 4 HDMI block A
		3276	* 21 5 HDMI block B
		3277	* 176 - CP_INT RB
		3278	* 177 - CP_INT IB1
		3279	* 178 - CP_INT IB2
		3280	* 181 - EOP Interrupt
		3281	* 233 - GUI Idle
		3282	*
		3283	* Note, these are based on r600 and may need to be
		3284	* adjusted or added to on newer asics
		3285	*/
		3286
2160	serge	3287	#define DRM_DEBUG(...)
		3288
2004	serge	3289	int r600_irq_process(struct radeon_device *rdev)
		3290	{
		3291	u32 wptr;
		3292	u32 rptr;
		3293	u32 src_id, src_data;
		3294	u32 ring_index;
		3295	bool queue_hotplug = false;
2997	Serge	3296	bool queue_hdmi = false;
2004	serge	3297
		3298	if (!rdev->ih.enabled \|\| rdev->shutdown)
		3299	return IRQ_NONE;
		3300
2160	serge	3301	/* No MSIs, need a dummy read to flush PCI DMAs */
		3302	if (!rdev->msi_enabled)
		3303	RREG32(IH_RB_WPTR);
		3304
2004	serge	3305	wptr = r600_get_ih_wptr(rdev);
		3306
2997	Serge	3307	restart_ih:
		3308	/* is somebody else already processing irqs? */
		3309	if (atomic_xchg(&rdev->ih.lock, 1))
2004	serge	3310	return IRQ_NONE;
		3311
2997	Serge	3312	rptr = rdev->ih.rptr;
		3313	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		3314
2160	serge	3315	/* Order reading of wptr vs. reading of IH ring data */
		3316	rmb();
		3317
2004	serge	3318	/* display interrupts */
		3319	r600_irq_ack(rdev);
		3320
		3321	while (rptr != wptr) {
		3322	/* wptr/rptr are in bytes! */
		3323	ring_index = rptr / 4;
		3324	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		3325	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		3326
		3327	switch (src_id) {
		3328	case 1: /* D1 vblank/vline */
		3329	switch (src_data) {
		3330	case 0: /* D1 vblank */
		3331	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
		3332	if (rdev->irq.crtc_vblank_int[0]) {
		3333	// drm_handle_vblank(rdev->ddev, 0);
		3334	rdev->pm.vblank_sync = true;
		3335	// wake_up(&rdev->irq.vblank_queue);
		3336	}
		3337	// if (rdev->irq.pflip[0])
		3338	// radeon_crtc_handle_flip(rdev, 0);
		3339	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		3340	DRM_DEBUG("IH: D1 vblank\n");
		3341	}
		3342	break;
		3343	case 1: /* D1 vline */
		3344	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
		3345	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		3346	DRM_DEBUG("IH: D1 vline\n");
		3347	}
		3348	break;
		3349	default:
		3350	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3351	break;
		3352	}
		3353	break;
		3354	case 5: /* D2 vblank/vline */
		3355	switch (src_data) {
		3356	case 0: /* D2 vblank */
		3357	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
		3358	if (rdev->irq.crtc_vblank_int[1]) {
		3359	// drm_handle_vblank(rdev->ddev, 1);
		3360	rdev->pm.vblank_sync = true;
		3361	// wake_up(&rdev->irq.vblank_queue);
		3362	}
		3363	// if (rdev->irq.pflip[1])
		3364	// radeon_crtc_handle_flip(rdev, 1);
		3365	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
		3366	DRM_DEBUG("IH: D2 vblank\n");
		3367	}
		3368	break;
		3369	case 1: /* D1 vline */
		3370	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
		3371	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
		3372	DRM_DEBUG("IH: D2 vline\n");
		3373	}
		3374	break;
		3375	default:
		3376	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3377	break;
		3378	}
		3379	break;
		3380	case 19: /* HPD/DAC hotplug */
		3381	switch (src_data) {
		3382	case 0:
		3383	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		3384	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
		3385	queue_hotplug = true;
		3386	DRM_DEBUG("IH: HPD1\n");
		3387	}
		3388	break;
		3389	case 1:
		3390	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		3391	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
		3392	queue_hotplug = true;
		3393	DRM_DEBUG("IH: HPD2\n");
		3394	}
		3395	break;
		3396	case 4:
		3397	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		3398	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
		3399	queue_hotplug = true;
		3400	DRM_DEBUG("IH: HPD3\n");
		3401	}
		3402	break;
		3403	case 5:
		3404	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		3405	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
		3406	queue_hotplug = true;
		3407	DRM_DEBUG("IH: HPD4\n");
		3408	}
		3409	break;
		3410	case 10:
		3411	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		3412	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
		3413	queue_hotplug = true;
		3414	DRM_DEBUG("IH: HPD5\n");
		3415	}
		3416	break;
		3417	case 12:
		3418	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		3419	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
		3420	queue_hotplug = true;
		3421	DRM_DEBUG("IH: HPD6\n");
		3422	}
		3423	break;
		3424	default:
		3425	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3426	break;
		3427	}
		3428	break;
2997	Serge	3429	case 21: /* hdmi */
		3430	switch (src_data) {
		3431	case 4:
		3432	if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
		3433	rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
		3434	queue_hdmi = true;
		3435	DRM_DEBUG("IH: HDMI0\n");
		3436	}
		3437	break;
		3438	case 5:
		3439	if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
		3440	rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
		3441	queue_hdmi = true;
		3442	DRM_DEBUG("IH: HDMI1\n");
		3443	}
		3444	break;
		3445	default:
		3446	DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
		3447	break;
		3448	}
2004	serge	3449	break;
		3450	case 176: /* CP_INT in ring buffer */
		3451	case 177: /* CP_INT in IB1 */
		3452	case 178: /* CP_INT in IB2 */
		3453	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2997	Serge	3454	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2004	serge	3455	break;
		3456	case 181: /* CP EOP event */
		3457	DRM_DEBUG("IH: CP EOP\n");
2997	Serge	3458	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2004	serge	3459	break;
3192	Serge	3460	case 224: /* DMA trap event */
		3461	DRM_DEBUG("IH: DMA trap\n");
		3462	radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
		3463	break;
2004	serge	3464	case 233: /* GUI IDLE */
		3465	DRM_DEBUG("IH: GUI idle\n");
		3466	break;
		3467	default:
		3468	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3469	break;
		3470	}
		3471
		3472	/* wptr/rptr are in bytes! */
		3473	rptr += 16;
		3474	rptr &= rdev->ih.ptr_mask;
		3475	}
2997	Serge	3476	rdev->ih.rptr = rptr;
		3477	WREG32(IH_RB_RPTR, rdev->ih.rptr);
		3478	atomic_set(&rdev->ih.lock, 0);
		3479
2004	serge	3480	/* make sure wptr hasn't changed while processing */
		3481	wptr = r600_get_ih_wptr(rdev);
2997	Serge	3482	if (wptr != rptr)
2004	serge	3483	goto restart_ih;
2997	Serge	3484
2004	serge	3485	return IRQ_HANDLED;
		3486	}
		3487
1221	serge	3488	/*
		3489	* Debugfs info
		3490	*/
		3491	#if defined(CONFIG_DEBUG_FS)
		3492
		3493	static int r600_debugfs_mc_info(struct seq_file m, void data)
		3494	{
		3495	struct drm_info_node node = (struct drm_info_node ) m->private;
		3496	struct drm_device *dev = node->minor->dev;
		3497	struct radeon_device *rdev = dev->dev_private;
		3498
		3499	DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
		3500	DREG32_SYS(m, rdev, VM_L2_STATUS);
		3501	return 0;
		3502	}
		3503
		3504	static struct drm_info_list r600_mc_info_list[] = {
		3505	{"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
		3506	};
		3507	#endif
		3508
		3509	int r600_debugfs_mc_info_init(struct radeon_device *rdev)
		3510	{
		3511	#if defined(CONFIG_DEBUG_FS)
		3512	return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
		3513	#else
		3514	return 0;
		3515	#endif
		3516	}
1404	serge	3517
		3518	/**
		3519	* r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
		3520	* rdev: radeon device structure
		3521	* bo: buffer object struct which userspace is waiting for idle
		3522	*
		3523	* Some R6XX/R7XX doesn't seems to take into account HDP flush performed
		3524	* through ring buffer, this leads to corruption in rendering, see
		3525	* http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
		3526	* directly perform HDP flush by writing register through MMIO.
		3527	*/
		3528	void r600_ioctl_wait_idle(struct radeon_device rdev, struct radeon_bo bo)
		3529	{
1963	serge	3530	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		3531	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
		3532	* This seems to cause problems on some AGP cards. Just use the old
		3533	* method for them.
		3534	*/
		3535	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		3536	rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
		3537	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		3538	u32 tmp;
		3539
		3540	WREG32(HDP_DEBUG1, 0);
		3541	tmp = readl((void __iomem *)ptr);
		3542	} else
1404	serge	3543	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		3544	}
1963	serge	3545
		3546	void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
		3547	{
		3548	u32 link_width_cntl, mask, target_reg;
		3549
		3550	if (rdev->flags & RADEON_IS_IGP)
		3551	return;
		3552
		3553	if (!(rdev->flags & RADEON_IS_PCIE))
		3554	return;
		3555
		3556	/* x2 cards have a special sequence */
		3557	if (ASIC_IS_X2(rdev))
		3558	return;
		3559
		3560	/* FIXME wait for idle */
		3561
		3562	switch (lanes) {
		3563	case 0:
		3564	mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
		3565	break;
		3566	case 1:
		3567	mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
		3568	break;
		3569	case 2:
		3570	mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
		3571	break;
		3572	case 4:
		3573	mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
		3574	break;
		3575	case 8:
		3576	mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
		3577	break;
		3578	case 12:
		3579	mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
		3580	break;
		3581	case 16:
		3582	default:
		3583	mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
		3584	break;
		3585	}
		3586
		3587	link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		3588
		3589	if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
		3590	(mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
		3591	return;
		3592
		3593	if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
		3594	return;
		3595
		3596	link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK \|
		3597	RADEON_PCIE_LC_RECONFIG_NOW \|
		3598	R600_PCIE_LC_RENEGOTIATE_EN \|
		3599	R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
		3600	link_width_cntl \|= mask;
		3601
		3602	WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3603
2005	serge	3604	/* some northbridges can renegotiate the link rather than requiring
		3605	* a complete re-config.
		3606	* e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)
1963	serge	3607	*/
		3608	if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
		3609	link_width_cntl \|= R600_PCIE_LC_RENEGOTIATE_EN \| R600_PCIE_LC_UPCONFIGURE_SUPPORT;
		3610	else
		3611	link_width_cntl \|= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
		3612
		3613	WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl \|
		3614	RADEON_PCIE_LC_RECONFIG_NOW));
		3615
		3616	if (rdev->family >= CHIP_RV770)
		3617	target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
		3618	else
		3619	target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
		3620
		3621	/* wait for lane set to complete */
		3622	link_width_cntl = RREG32(target_reg);
		3623	while (link_width_cntl == 0xffffffff)
		3624	link_width_cntl = RREG32(target_reg);
		3625
		3626	}
		3627
		3628	int r600_get_pcie_lanes(struct radeon_device *rdev)
		3629	{
		3630	u32 link_width_cntl;
		3631
		3632	if (rdev->flags & RADEON_IS_IGP)
		3633	return 0;
		3634
		3635	if (!(rdev->flags & RADEON_IS_PCIE))
		3636	return 0;
		3637
		3638	/* x2 cards have a special sequence */
		3639	if (ASIC_IS_X2(rdev))
		3640	return 0;
		3641
		3642	/* FIXME wait for idle */
		3643
		3644	link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		3645
		3646	switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
		3647	case RADEON_PCIE_LC_LINK_WIDTH_X0:
		3648	return 0;
		3649	case RADEON_PCIE_LC_LINK_WIDTH_X1:
		3650	return 1;
		3651	case RADEON_PCIE_LC_LINK_WIDTH_X2:
		3652	return 2;
		3653	case RADEON_PCIE_LC_LINK_WIDTH_X4:
		3654	return 4;
		3655	case RADEON_PCIE_LC_LINK_WIDTH_X8:
		3656	return 8;
		3657	case RADEON_PCIE_LC_LINK_WIDTH_X16:
		3658	default:
		3659	return 16;
		3660	}
		3661	}
		3662
		3663	static void r600_pcie_gen2_enable(struct radeon_device *rdev)
		3664	{
		3665	u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
		3666	u16 link_cntl2;
2997	Serge	3667	u32 mask;
		3668	int ret;
1963	serge	3669
		3670	if (radeon_pcie_gen2 == 0)
		3671	return;
		3672
		3673	if (rdev->flags & RADEON_IS_IGP)
		3674	return;
		3675
		3676	if (!(rdev->flags & RADEON_IS_PCIE))
		3677	return;
		3678
		3679	/* x2 cards have a special sequence */
		3680	if (ASIC_IS_X2(rdev))
		3681	return;
		3682
		3683	/* only RV6xx+ chips are supported */
		3684	if (rdev->family <= CHIP_R600)
		3685	return;
		3686
2997	Serge	3687	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
		3688	if (ret != 0)
		3689	return;
		3690
		3691	if (!(mask & DRM_PCIE_SPEED_50))
		3692	return;
		3693
		3694	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3695	if (speed_cntl & LC_CURRENT_DATA_RATE) {
		3696	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
		3697	return;
		3698	}
		3699
		3700	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
		3701
1963	serge	3702	/* 55 nm r6xx asics */
		3703	if ((rdev->family == CHIP_RV670) \|\|
		3704	(rdev->family == CHIP_RV620) \|\|
		3705	(rdev->family == CHIP_RV635)) {
		3706	/* advertise upconfig capability */
		3707	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3708	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3709	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3710	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3711	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
		3712	lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
		3713	link_width_cntl &= ~(LC_LINK_WIDTH_MASK \|
		3714	LC_RECONFIG_ARC_MISSING_ESCAPE);
		3715	link_width_cntl \|= lanes \| LC_RECONFIG_NOW \| LC_RENEGOTIATE_EN;
		3716	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3717	} else {
		3718	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3719	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3720	}
		3721	}
		3722
		3723	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3724	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
		3725	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		3726
		3727	/* 55 nm r6xx asics */
		3728	if ((rdev->family == CHIP_RV670) \|\|
		3729	(rdev->family == CHIP_RV620) \|\|
		3730	(rdev->family == CHIP_RV635)) {
		3731	WREG32(MM_CFGREGS_CNTL, 0x8);
		3732	link_cntl2 = RREG32(0x4088);
		3733	WREG32(MM_CFGREGS_CNTL, 0);
		3734	/* not supported yet */
		3735	if (link_cntl2 & SELECTABLE_DEEMPHASIS)
		3736	return;
		3737	}
		3738
		3739	speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
		3740	speed_cntl \|= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
		3741	speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
		3742	speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
		3743	speed_cntl \|= LC_FORCE_EN_HW_SPEED_CHANGE;
		3744	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3745
		3746	tmp = RREG32(0x541c);
		3747	WREG32(0x541c, tmp \| 0x8);
		3748	WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
		3749	link_cntl2 = RREG16(0x4088);
		3750	link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
		3751	link_cntl2 \|= 0x2;
		3752	WREG16(0x4088, link_cntl2);
		3753	WREG32(MM_CFGREGS_CNTL, 0);
		3754
		3755	if ((rdev->family == CHIP_RV670) \|\|
		3756	(rdev->family == CHIP_RV620) \|\|
		3757	(rdev->family == CHIP_RV635)) {
		3758	training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
		3759	training_cntl &= ~LC_POINT_7_PLUS_EN;
		3760	WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
		3761	} else {
		3762	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3763	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		3764	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3765	}
		3766
		3767	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3768	speed_cntl \|= LC_GEN2_EN_STRAP;
		3769	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3770
		3771	} else {
		3772	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3773	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		3774	if (1)
		3775	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3776	else
		3777	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3778	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3779	}
		3780	}
2997	Serge	3781
		3782	/**
		3783	* r600_get_gpu_clock - return GPU clock counter snapshot
		3784	*
		3785	* @rdev: radeon_device pointer
		3786	*
		3787	* Fetches a GPU clock counter snapshot (R6xx-cayman).
		3788	* Returns the 64 bit clock counter snapshot.
		3789	*/
		3790	uint64_t r600_get_gpu_clock(struct radeon_device *rdev)
		3791	{
		3792	uint64_t clock;
		3793
		3794	mutex_lock(&rdev->gpu_clock_mutex);
		3795	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
		3796	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) \|
		3797	((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
		3798	mutex_unlock(&rdev->gpu_clock_mutex);
		3799	return clock;
		3800	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/r600.c – Rev 3192