WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/r600.c

Rev	Author	Line No.	Line
1128	serge	1	/*
		2	* Copyright 2008 Advanced Micro Devices, Inc.
		3	* Copyright 2008 Red Hat Inc.
		4	* Copyright 2009 Jerome Glisse.
		5	*
		6	* Permission is hereby granted, free of charge, to any person obtaining a
		7	* copy of this software and associated documentation files (the "Software"),
		8	* to deal in the Software without restriction, including without limitation
		9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		10	* and/or sell copies of the Software, and to permit persons to whom the
		11	* Software is furnished to do so, subject to the following conditions:
		12	*
		13	* The above copyright notice and this permission notice shall be included in
		14	* all copies or substantial portions of the Software.
		15	*
		16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		22	* OTHER DEALINGS IN THE SOFTWARE.
		23	*
		24	* Authors: Dave Airlie
		25	* Alex Deucher
		26	* Jerome Glisse
		27	*/
1963	serge	28	#include
1233	serge	29	#include
1221	serge	30	#include
2997	Serge	31	#include
		32	#include
		33	#include
1128	serge	34	#include "radeon.h"
1963	serge	35	#include "radeon_asic.h"
1221	serge	36	#include "radeon_mode.h"
		37	#include "r600d.h"
		38	#include "atom.h"
		39	#include "avivod.h"
1128	serge	40
1221	serge	41	#define PFP_UCODE_SIZE 576
		42	#define PM4_UCODE_SIZE 1792
1321	serge	43	#define RLC_UCODE_SIZE 768
1221	serge	44	#define R700_PFP_UCODE_SIZE 848
		45	#define R700_PM4_UCODE_SIZE 1360
1321	serge	46	#define R700_RLC_UCODE_SIZE 1024
1963	serge	47	#define EVERGREEN_PFP_UCODE_SIZE 1120
		48	#define EVERGREEN_PM4_UCODE_SIZE 1376
		49	#define EVERGREEN_RLC_UCODE_SIZE 768
		50	#define CAYMAN_RLC_UCODE_SIZE 1024
2997	Serge	51	#define ARUBA_RLC_UCODE_SIZE 1536
1128	serge	52
1221	serge	53	/* Firmware Names */
		54	MODULE_FIRMWARE("radeon/R600_pfp.bin");
		55	MODULE_FIRMWARE("radeon/R600_me.bin");
		56	MODULE_FIRMWARE("radeon/RV610_pfp.bin");
		57	MODULE_FIRMWARE("radeon/RV610_me.bin");
		58	MODULE_FIRMWARE("radeon/RV630_pfp.bin");
		59	MODULE_FIRMWARE("radeon/RV630_me.bin");
		60	MODULE_FIRMWARE("radeon/RV620_pfp.bin");
		61	MODULE_FIRMWARE("radeon/RV620_me.bin");
		62	MODULE_FIRMWARE("radeon/RV635_pfp.bin");
		63	MODULE_FIRMWARE("radeon/RV635_me.bin");
		64	MODULE_FIRMWARE("radeon/RV670_pfp.bin");
		65	MODULE_FIRMWARE("radeon/RV670_me.bin");
		66	MODULE_FIRMWARE("radeon/RS780_pfp.bin");
		67	MODULE_FIRMWARE("radeon/RS780_me.bin");
		68	MODULE_FIRMWARE("radeon/RV770_pfp.bin");
		69	MODULE_FIRMWARE("radeon/RV770_me.bin");
		70	MODULE_FIRMWARE("radeon/RV730_pfp.bin");
		71	MODULE_FIRMWARE("radeon/RV730_me.bin");
		72	MODULE_FIRMWARE("radeon/RV710_pfp.bin");
		73	MODULE_FIRMWARE("radeon/RV710_me.bin");
1321	serge	74	MODULE_FIRMWARE("radeon/R600_rlc.bin");
		75	MODULE_FIRMWARE("radeon/R700_rlc.bin");
1963	serge	76	MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
		77	MODULE_FIRMWARE("radeon/CEDAR_me.bin");
		78	MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
		79	MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
		80	MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
		81	MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
		82	MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
		83	MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
		84	MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
		85	MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
		86	MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
		87	MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
		88	MODULE_FIRMWARE("radeon/PALM_pfp.bin");
		89	MODULE_FIRMWARE("radeon/PALM_me.bin");
		90	MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
		91	MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
		92	MODULE_FIRMWARE("radeon/SUMO_me.bin");
		93	MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
		94	MODULE_FIRMWARE("radeon/SUMO2_me.bin");
1221	serge	95
		96	int r600_debugfs_mc_info_init(struct radeon_device *rdev);
		97
		98	/* r600,rv610,rv630,rv620,rv635,rv670 */
1128	serge	99	int r600_mc_wait_for_idle(struct radeon_device *rdev);
2997	Serge	100	static void r600_gpu_init(struct radeon_device *rdev);
1221	serge	101	void r600_fini(struct radeon_device *rdev);
1963	serge	102	void r600_irq_disable(struct radeon_device *rdev);
		103	static void r600_pcie_gen2_enable(struct radeon_device *rdev);
1128	serge	104
1963	serge	105	/* get temperature in millidegrees */
		106	int rv6xx_get_temp(struct radeon_device *rdev)
		107	{
		108	u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
		109	ASIC_T_SHIFT;
		110	int actual_temp = temp & 0xff;
		111
		112	if (temp & 0x100)
		113	actual_temp -= 256;
		114
		115	return actual_temp * 1000;
		116	}
		117
		118
		119
		120
		121
		122
		123	bool r600_gui_idle(struct radeon_device *rdev)
		124	{
		125	if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
		126	return false;
		127	else
		128	return true;
		129	}
		130
1321	serge	131	/* hpd for digital panel detect/disconnect */
		132	bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		133	{
		134	bool connected = false;
		135
		136	if (ASIC_IS_DCE3(rdev)) {
		137	switch (hpd) {
		138	case RADEON_HPD_1:
		139	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		140	connected = true;
		141	break;
		142	case RADEON_HPD_2:
		143	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		144	connected = true;
		145	break;
		146	case RADEON_HPD_3:
		147	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		148	connected = true;
		149	break;
		150	case RADEON_HPD_4:
		151	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		152	connected = true;
		153	break;
		154	/* DCE 3.2 */
		155	case RADEON_HPD_5:
		156	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		157	connected = true;
		158	break;
		159	case RADEON_HPD_6:
		160	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		161	connected = true;
		162	break;
		163	default:
		164	break;
		165	}
		166	} else {
		167	switch (hpd) {
		168	case RADEON_HPD_1:
		169	if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		170	connected = true;
		171	break;
		172	case RADEON_HPD_2:
		173	if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		174	connected = true;
		175	break;
		176	case RADEON_HPD_3:
		177	if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		178	connected = true;
		179	break;
		180	default:
		181	break;
		182	}
		183	}
		184	return connected;
		185	}
		186
		187	void r600_hpd_set_polarity(struct radeon_device *rdev,
		188	enum radeon_hpd_id hpd)
		189	{
		190	u32 tmp;
		191	bool connected = r600_hpd_sense(rdev, hpd);
		192
		193	if (ASIC_IS_DCE3(rdev)) {
		194	switch (hpd) {
		195	case RADEON_HPD_1:
		196	tmp = RREG32(DC_HPD1_INT_CONTROL);
		197	if (connected)
		198	tmp &= ~DC_HPDx_INT_POLARITY;
		199	else
		200	tmp \|= DC_HPDx_INT_POLARITY;
		201	WREG32(DC_HPD1_INT_CONTROL, tmp);
		202	break;
		203	case RADEON_HPD_2:
		204	tmp = RREG32(DC_HPD2_INT_CONTROL);
		205	if (connected)
		206	tmp &= ~DC_HPDx_INT_POLARITY;
		207	else
		208	tmp \|= DC_HPDx_INT_POLARITY;
		209	WREG32(DC_HPD2_INT_CONTROL, tmp);
		210	break;
		211	case RADEON_HPD_3:
		212	tmp = RREG32(DC_HPD3_INT_CONTROL);
		213	if (connected)
		214	tmp &= ~DC_HPDx_INT_POLARITY;
		215	else
		216	tmp \|= DC_HPDx_INT_POLARITY;
		217	WREG32(DC_HPD3_INT_CONTROL, tmp);
		218	break;
		219	case RADEON_HPD_4:
		220	tmp = RREG32(DC_HPD4_INT_CONTROL);
		221	if (connected)
		222	tmp &= ~DC_HPDx_INT_POLARITY;
		223	else
		224	tmp \|= DC_HPDx_INT_POLARITY;
		225	WREG32(DC_HPD4_INT_CONTROL, tmp);
		226	break;
		227	case RADEON_HPD_5:
		228	tmp = RREG32(DC_HPD5_INT_CONTROL);
		229	if (connected)
		230	tmp &= ~DC_HPDx_INT_POLARITY;
		231	else
		232	tmp \|= DC_HPDx_INT_POLARITY;
		233	WREG32(DC_HPD5_INT_CONTROL, tmp);
		234	break;
		235	/* DCE 3.2 */
		236	case RADEON_HPD_6:
		237	tmp = RREG32(DC_HPD6_INT_CONTROL);
		238	if (connected)
		239	tmp &= ~DC_HPDx_INT_POLARITY;
		240	else
		241	tmp \|= DC_HPDx_INT_POLARITY;
		242	WREG32(DC_HPD6_INT_CONTROL, tmp);
		243	break;
		244	default:
		245	break;
		246	}
		247	} else {
		248	switch (hpd) {
		249	case RADEON_HPD_1:
		250	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		251	if (connected)
		252	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		253	else
		254	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		255	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		256	break;
		257	case RADEON_HPD_2:
		258	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		259	if (connected)
		260	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		261	else
		262	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		263	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		264	break;
		265	case RADEON_HPD_3:
		266	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		267	if (connected)
		268	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		269	else
		270	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		271	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		272	break;
		273	default:
		274	break;
		275	}
		276	}
		277	}
		278
		279	void r600_hpd_init(struct radeon_device *rdev)
		280	{
		281	struct drm_device *dev = rdev->ddev;
		282	struct drm_connector *connector;
2997	Serge	283	unsigned enable = 0;
1321	serge	284
2997	Serge	285	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		286	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		287
		288	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP \|\|
		289	connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
		290	/* don't try to enable hpd on eDP or LVDS avoid breaking the
		291	* aux dp channel on imac and help (but not completely fix)
		292	* https://bugzilla.redhat.com/show_bug.cgi?id=726143
		293	*/
		294	continue;
		295	}
1321	serge	296	if (ASIC_IS_DCE3(rdev)) {
		297	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \| DC_HPDx_RX_INT_TIMER(0xfa);
		298	if (ASIC_IS_DCE32(rdev))
		299	tmp \|= DC_HPDx_EN;
		300
		301	switch (radeon_connector->hpd.hpd) {
		302	case RADEON_HPD_1:
		303	WREG32(DC_HPD1_CONTROL, tmp);
		304	break;
		305	case RADEON_HPD_2:
		306	WREG32(DC_HPD2_CONTROL, tmp);
		307	break;
		308	case RADEON_HPD_3:
		309	WREG32(DC_HPD3_CONTROL, tmp);
		310	break;
		311	case RADEON_HPD_4:
		312	WREG32(DC_HPD4_CONTROL, tmp);
		313	break;
		314	/* DCE 3.2 */
		315	case RADEON_HPD_5:
		316	WREG32(DC_HPD5_CONTROL, tmp);
		317	break;
		318	case RADEON_HPD_6:
		319	WREG32(DC_HPD6_CONTROL, tmp);
		320	break;
		321	default:
		322	break;
		323	}
		324	} else {
		325	switch (radeon_connector->hpd.hpd) {
		326	case RADEON_HPD_1:
		327	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		328	break;
		329	case RADEON_HPD_2:
		330	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		331	break;
		332	case RADEON_HPD_3:
		333	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		334	break;
		335	default:
		336	break;
		337	}
		338	}
2997	Serge	339	enable \|= 1 << radeon_connector->hpd.hpd;
		340	radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1321	serge	341	}
2997	Serge	342	// radeon_irq_kms_enable_hpd(rdev, enable);
1321	serge	343	}
		344
		345	void r600_hpd_fini(struct radeon_device *rdev)
		346	{
		347	struct drm_device *dev = rdev->ddev;
		348	struct drm_connector *connector;
2997	Serge	349	unsigned disable = 0;
1321	serge	350
		351	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		352	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2997	Serge	353	if (ASIC_IS_DCE3(rdev)) {
1321	serge	354	switch (radeon_connector->hpd.hpd) {
		355	case RADEON_HPD_1:
		356	WREG32(DC_HPD1_CONTROL, 0);
		357	break;
		358	case RADEON_HPD_2:
		359	WREG32(DC_HPD2_CONTROL, 0);
		360	break;
		361	case RADEON_HPD_3:
		362	WREG32(DC_HPD3_CONTROL, 0);
		363	break;
		364	case RADEON_HPD_4:
		365	WREG32(DC_HPD4_CONTROL, 0);
		366	break;
		367	/* DCE 3.2 */
		368	case RADEON_HPD_5:
		369	WREG32(DC_HPD5_CONTROL, 0);
		370	break;
		371	case RADEON_HPD_6:
		372	WREG32(DC_HPD6_CONTROL, 0);
		373	break;
		374	default:
		375	break;
		376	}
		377	} else {
		378	switch (radeon_connector->hpd.hpd) {
		379	case RADEON_HPD_1:
		380	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
		381	break;
		382	case RADEON_HPD_2:
		383	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
		384	break;
		385	case RADEON_HPD_3:
		386	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
		387	break;
		388	default:
		389	break;
		390	}
		391	}
2997	Serge	392	disable \|= 1 << radeon_connector->hpd.hpd;
1321	serge	393	}
2997	Serge	394	// radeon_irq_kms_disable_hpd(rdev, disable);
1321	serge	395	}
		396
1128	serge	397	/*
1221	serge	398	* R600 PCIE GART
1128	serge	399	*/
1221	serge	400	void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
		401	{
		402	unsigned i;
		403	u32 tmp;
1128	serge	404
1430	serge	405	/* flush hdp cache so updates hit vram */
1963	serge	406	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		407	!(rdev->flags & RADEON_IS_AGP)) {
2997	Serge	408	void __iomem ptr = (void )rdev->gart.ptr;
1963	serge	409	u32 tmp;
		410
		411	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		412	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
		413	* This seems to cause problems on some AGP cards. Just use the old
		414	* method for them.
		415	*/
		416	WREG32(HDP_DEBUG1, 0);
		417	tmp = readl((void __iomem *)ptr);
		418	} else
1430	serge	419	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		420
1221	serge	421	WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
		422	WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
		423	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		424	for (i = 0; i < rdev->usec_timeout; i++) {
		425	/* read MC_STATUS */
		426	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		427	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		428	if (tmp == 2) {
		429	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		430	return;
		431	}
		432	if (tmp) {
		433	return;
		434	}
		435	udelay(1);
1128	serge	436	}
1221	serge	437	}
1128	serge	438
1221	serge	439	int r600_pcie_gart_init(struct radeon_device *rdev)
		440	{
		441	int r;
		442
2997	Serge	443	if (rdev->gart.robj) {
1963	serge	444	WARN(1, "R600 PCIE GART already initialized\n");
1221	serge	445	return 0;
		446	}
		447	/* Initialize common gart structure */
		448	r = radeon_gart_init(rdev);
		449	if (r)
		450	return r;
		451	rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
		452	return radeon_gart_table_vram_alloc(rdev);
		453	}
		454
2997	Serge	455	static int r600_pcie_gart_enable(struct radeon_device *rdev)
1221	serge	456	{
		457	u32 tmp;
		458	int r, i;
		459
2997	Serge	460	if (rdev->gart.robj == NULL) {
1221	serge	461	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		462	return -EINVAL;
		463	}
		464	r = radeon_gart_table_vram_pin(rdev);
		465	if (r)
		466	return r;
1430	serge	467	radeon_gart_restore(rdev);
1221	serge	468
		469	/* Setup L2 cache */
		470	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		471	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		472	EFFECTIVE_L2_QUEUE_SIZE(7));
		473	WREG32(VM_L2_CNTL2, 0);
		474	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		475	/* Setup TLB control */
		476	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		477	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		478	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		479	ENABLE_WAIT_L2_QUERY;
		480	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		481	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		482	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		483	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		484	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		485	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		486	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		487	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		488	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		489	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		490	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		491	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		492	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		493	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		494	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		495	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		496	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		497	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		498	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		499	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		500	(u32)(rdev->dummy_page.addr >> 12));
		501	for (i = 1; i < 7; i++)
		502	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		503
		504	r600_pcie_gart_tlb_flush(rdev);
2997	Serge	505	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		506	(unsigned)(rdev->mc.gtt_size >> 20),
		507	(unsigned long long)rdev->gart.table_addr);
1221	serge	508	rdev->gart.ready = true;
1128	serge	509	return 0;
		510	}
		511
2997	Serge	512	static void r600_pcie_gart_disable(struct radeon_device *rdev)
1128	serge	513	{
1221	serge	514	u32 tmp;
2997	Serge	515	int i;
1221	serge	516
		517	/* Disable all tables */
		518	for (i = 0; i < 7; i++)
		519	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		520
		521	/* Disable L2 cache */
		522	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		523	EFFECTIVE_L2_QUEUE_SIZE(7));
		524	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		525	/* Setup L1 TLB control */
		526	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		527	ENABLE_WAIT_L2_QUERY;
		528	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		529	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		530	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		531	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		532	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		533	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		534	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		535	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		536	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
		537	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
		538	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		539	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		540	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
		541	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
2997	Serge	542	radeon_gart_table_vram_unpin(rdev);
1128	serge	543	}
		544
2997	Serge	545	static void r600_pcie_gart_fini(struct radeon_device *rdev)
1221	serge	546	{
1963	serge	547	radeon_gart_fini(rdev);
1221	serge	548	r600_pcie_gart_disable(rdev);
		549	radeon_gart_table_vram_free(rdev);
		550	}
1128	serge	551
2997	Serge	552	static void r600_agp_enable(struct radeon_device *rdev)
1128	serge	553	{
1221	serge	554	u32 tmp;
		555	int i;
		556
		557	/* Setup L2 cache */
		558	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		559	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		560	EFFECTIVE_L2_QUEUE_SIZE(7));
		561	WREG32(VM_L2_CNTL2, 0);
		562	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		563	/* Setup TLB control */
		564	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		565	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		566	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		567	ENABLE_WAIT_L2_QUERY;
		568	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		569	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		570	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		571	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		572	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		573	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		574	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		575	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		576	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		577	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		578	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		579	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		580	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		581	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		582	for (i = 0; i < 7; i++)
		583	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1128	serge	584	}
		585
		586	int r600_mc_wait_for_idle(struct radeon_device *rdev)
		587	{
1221	serge	588	unsigned i;
		589	u32 tmp;
		590
		591	for (i = 0; i < rdev->usec_timeout; i++) {
		592	/* read MC_STATUS */
		593	tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
		594	if (!tmp)
1128	serge	595	return 0;
1221	serge	596	udelay(1);
		597	}
		598	return -1;
1128	serge	599	}
		600
1221	serge	601	static void r600_mc_program(struct radeon_device *rdev)
1128	serge	602	{
1221	serge	603	struct rv515_mc_save save;
		604	u32 tmp;
		605	int i, j;
		606
		607	/* Initialize HDP */
		608	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		609	WREG32((0x2c14 + j), 0x00000000);
		610	WREG32((0x2c18 + j), 0x00000000);
		611	WREG32((0x2c1c + j), 0x00000000);
		612	WREG32((0x2c20 + j), 0x00000000);
		613	WREG32((0x2c24 + j), 0x00000000);
		614	}
		615	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		616
		617	rv515_mc_stop(rdev, &save);
		618	if (r600_mc_wait_for_idle(rdev)) {
		619	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		620	}
		621	/* Lockout access through VGA aperture (doesn't exist before R600) */
		622	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		623	/* Update configuration */
		624	if (rdev->flags & RADEON_IS_AGP) {
		625	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		626	/* VRAM before AGP */
		627	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		628	rdev->mc.vram_start >> 12);
		629	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		630	rdev->mc.gtt_end >> 12);
		631	} else {
		632	/* VRAM after AGP */
		633	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		634	rdev->mc.gtt_start >> 12);
		635	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		636	rdev->mc.vram_end >> 12);
		637	}
		638	} else {
		639	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
		640	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
		641	}
2997	Serge	642	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1221	serge	643	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		644	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		645	WREG32(MC_VM_FB_LOCATION, tmp);
		646	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
		647	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1963	serge	648	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1221	serge	649	if (rdev->flags & RADEON_IS_AGP) {
		650	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
		651	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
		652	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		653	} else {
		654	WREG32(MC_VM_AGP_BASE, 0);
		655	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		656	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		657	}
		658	if (r600_mc_wait_for_idle(rdev)) {
		659	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		660	}
		661	rv515_mc_resume(rdev, &save);
		662	/* we need to own VRAM, so turn off the VGA renderer here
		663	* to stop it overwriting our objects */
		664	rv515_vga_render_disable(rdev);
1128	serge	665	}
		666
1430	serge	667	/**
		668	* r600_vram_gtt_location - try to find VRAM & GTT location
		669	* @rdev: radeon device structure holding all necessary informations
		670	* @mc: memory controller structure holding memory informations
		671	*
		672	* Function will place try to place VRAM at same place as in CPU (PCI)
		673	* address space as some GPU seems to have issue when we reprogram at
		674	* different address space.
		675	*
		676	* If there is not enough space to fit the unvisible VRAM after the
		677	* aperture then we limit the VRAM size to the aperture.
		678	*
		679	* If we are using AGP then place VRAM adjacent to AGP aperture are we need
		680	* them to be in one from GPU point of view so that we can program GPU to
		681	* catch access outside them (weird GPU policy see ??).
		682	*
		683	* This function will never fails, worst case are limiting VRAM or GTT.
		684	*
		685	* Note: GTT start, end, size should be initialized before calling this
		686	* function on AGP platform.
		687	*/
1963	serge	688	static void r600_vram_gtt_location(struct radeon_device rdev, struct radeon_mc mc)
1430	serge	689	{
		690	u64 size_bf, size_af;
		691
		692	if (mc->mc_vram_size > 0xE0000000) {
		693	/* leave room for at least 512M GTT */
		694	dev_warn(rdev->dev, "limiting VRAM\n");
		695	mc->real_vram_size = 0xE0000000;
		696	mc->mc_vram_size = 0xE0000000;
		697	}
		698	if (rdev->flags & RADEON_IS_AGP) {
		699	size_bf = mc->gtt_start;
2997	Serge	700	size_af = 0xFFFFFFFF - mc->gtt_end;
1430	serge	701	if (size_bf > size_af) {
		702	if (mc->mc_vram_size > size_bf) {
		703	dev_warn(rdev->dev, "limiting VRAM\n");
		704	mc->real_vram_size = size_bf;
		705	mc->mc_vram_size = size_bf;
		706	}
		707	mc->vram_start = mc->gtt_start - mc->mc_vram_size;
		708	} else {
		709	if (mc->mc_vram_size > size_af) {
		710	dev_warn(rdev->dev, "limiting VRAM\n");
		711	mc->real_vram_size = size_af;
		712	mc->mc_vram_size = size_af;
		713	}
2997	Serge	714	mc->vram_start = mc->gtt_end + 1;
1430	serge	715	}
		716	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
		717	dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
		718	mc->mc_vram_size >> 20, mc->vram_start,
		719	mc->vram_end, mc->real_vram_size >> 20);
		720	} else {
		721	u64 base = 0;
1963	serge	722	if (rdev->flags & RADEON_IS_IGP) {
		723	base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
		724	base <<= 24;
		725	}
1430	serge	726	radeon_vram_location(rdev, &rdev->mc, base);
1963	serge	727	rdev->mc.gtt_base_align = 0;
1430	serge	728	radeon_gtt_location(rdev, mc);
		729	}
		730	}
		731
2997	Serge	732	static int r600_mc_init(struct radeon_device *rdev)
1128	serge	733	{
1221	serge	734	u32 tmp;
1268	serge	735	int chansize, numchan;
1128	serge	736
1221	serge	737	/* Get VRAM informations */
1128	serge	738	rdev->mc.vram_is_ddr = true;
1221	serge	739	tmp = RREG32(RAMCFG);
		740	if (tmp & CHANSIZE_OVERRIDE) {
1128	serge	741	chansize = 16;
1221	serge	742	} else if (tmp & CHANSIZE_MASK) {
1128	serge	743	chansize = 64;
		744	} else {
		745	chansize = 32;
		746	}
1268	serge	747	tmp = RREG32(CHMAP);
		748	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		749	case 0:
		750	default:
		751	numchan = 1;
		752	break;
		753	case 1:
		754	numchan = 2;
		755	break;
		756	case 2:
		757	numchan = 4;
		758	break;
		759	case 3:
		760	numchan = 8;
		761	break;
1128	serge	762	}
1268	serge	763	rdev->mc.vram_width = numchan * chansize;
1221	serge	764	/* Could aper size report 0 ? */
1963	serge	765	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		766	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1221	serge	767	/* Setup GPU memory space */
		768	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		769	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1430	serge	770	rdev->mc.visible_vram_size = rdev->mc.aper_size;
		771	r600_vram_gtt_location(rdev, &rdev->mc);
1963	serge	772
		773	if (rdev->flags & RADEON_IS_IGP) {
		774	rs690_pm_info(rdev);
1403	serge	775	rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1963	serge	776	}
		777	radeon_update_bandwidth_info(rdev);
1221	serge	778	return 0;
1128	serge	779	}
		780
2997	Serge	781	int r600_vram_scratch_init(struct radeon_device *rdev)
		782	{
		783	int r;
		784
		785	if (rdev->vram_scratch.robj == NULL) {
		786	r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
		787	PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
		788	NULL, &rdev->vram_scratch.robj);
		789	if (r) {
		790	return r;
		791	}
		792	}
		793
		794	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
		795	if (unlikely(r != 0))
		796	return r;
		797	r = radeon_bo_pin(rdev->vram_scratch.robj,
		798	RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
		799	if (r) {
		800	radeon_bo_unreserve(rdev->vram_scratch.robj);
		801	return r;
		802	}
		803	r = radeon_bo_kmap(rdev->vram_scratch.robj,
		804	(void **)&rdev->vram_scratch.ptr);
		805	if (r)
		806	radeon_bo_unpin(rdev->vram_scratch.robj);
		807	radeon_bo_unreserve(rdev->vram_scratch.robj);
		808
		809	return r;
		810	}
1221	serge	811	/* We doesn't check that the GPU really needs a reset we simply do the
		812	* reset, it's up to the caller to determine if the GPU needs one. We
		813	* might add an helper function to check that.
		814	*/
2997	Serge	815	static int r600_gpu_soft_reset(struct radeon_device *rdev)
1128	serge	816	{
1221	serge	817	struct rv515_mc_save save;
		818	u32 grbm_busy_mask = S_008010_VC_BUSY(1) \| S_008010_VGT_BUSY_NO_DMA(1) \|
		819	S_008010_VGT_BUSY(1) \| S_008010_TA03_BUSY(1) \|
		820	S_008010_TC_BUSY(1) \| S_008010_SX_BUSY(1) \|
		821	S_008010_SH_BUSY(1) \| S_008010_SPI03_BUSY(1) \|
		822	S_008010_SMX_BUSY(1) \| S_008010_SC_BUSY(1) \|
		823	S_008010_PA_BUSY(1) \| S_008010_DB03_BUSY(1) \|
		824	S_008010_CR_BUSY(1) \| S_008010_CB03_BUSY(1) \|
		825	S_008010_GUI_ACTIVE(1);
		826	u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) \| S_008014_SPI1_BUSY(1) \|
		827	S_008014_SPI2_BUSY(1) \| S_008014_SPI3_BUSY(1) \|
		828	S_008014_TA0_BUSY(1) \| S_008014_TA1_BUSY(1) \|
		829	S_008014_TA2_BUSY(1) \| S_008014_TA3_BUSY(1) \|
		830	S_008014_DB0_BUSY(1) \| S_008014_DB1_BUSY(1) \|
		831	S_008014_DB2_BUSY(1) \| S_008014_DB3_BUSY(1) \|
		832	S_008014_CB0_BUSY(1) \| S_008014_CB1_BUSY(1) \|
		833	S_008014_CB2_BUSY(1) \| S_008014_CB3_BUSY(1);
		834	u32 tmp;
1128	serge	835
1963	serge	836	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
		837	return 0;
		838
1221	serge	839	dev_info(rdev->dev, "GPU softreset \n");
		840	dev_info(rdev->dev, " R_008010_GRBM_STATUS=0x%08X\n",
		841	RREG32(R_008010_GRBM_STATUS));
		842	dev_info(rdev->dev, " R_008014_GRBM_STATUS2=0x%08X\n",
		843	RREG32(R_008014_GRBM_STATUS2));
		844	dev_info(rdev->dev, " R_000E50_SRBM_STATUS=0x%08X\n",
		845	RREG32(R_000E50_SRBM_STATUS));
2997	Serge	846	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		847	RREG32(CP_STALLED_STAT1));
		848	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		849	RREG32(CP_STALLED_STAT2));
		850	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		851	RREG32(CP_BUSY_STAT));
		852	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		853	RREG32(CP_STAT));
1221	serge	854	rv515_mc_stop(rdev, &save);
		855	if (r600_mc_wait_for_idle(rdev)) {
		856	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		857	}
		858	/* Disable CP parsing/prefetching */
1963	serge	859	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1221	serge	860	/* Check if any of the rendering block is busy and reset it */
		861	if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) \|\|
		862	(RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
		863	tmp = S_008020_SOFT_RESET_CR(1) \|
		864	S_008020_SOFT_RESET_DB(1) \|
		865	S_008020_SOFT_RESET_CB(1) \|
		866	S_008020_SOFT_RESET_PA(1) \|
		867	S_008020_SOFT_RESET_SC(1) \|
		868	S_008020_SOFT_RESET_SMX(1) \|
		869	S_008020_SOFT_RESET_SPI(1) \|
		870	S_008020_SOFT_RESET_SX(1) \|
		871	S_008020_SOFT_RESET_SH(1) \|
		872	S_008020_SOFT_RESET_TC(1) \|
		873	S_008020_SOFT_RESET_TA(1) \|
		874	S_008020_SOFT_RESET_VC(1) \|
		875	S_008020_SOFT_RESET_VGT(1);
		876	dev_info(rdev->dev, " R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		877	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1963	serge	878	RREG32(R_008020_GRBM_SOFT_RESET);
		879	mdelay(15);
1221	serge	880	WREG32(R_008020_GRBM_SOFT_RESET, 0);
		881	}
		882	/* Reset CP (we always reset CP) */
		883	tmp = S_008020_SOFT_RESET_CP(1);
		884	dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		885	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1963	serge	886	RREG32(R_008020_GRBM_SOFT_RESET);
		887	mdelay(15);
1221	serge	888	WREG32(R_008020_GRBM_SOFT_RESET, 0);
		889	/* Wait a little for things to settle down */
1963	serge	890	mdelay(1);
1221	serge	891	dev_info(rdev->dev, " R_008010_GRBM_STATUS=0x%08X\n",
		892	RREG32(R_008010_GRBM_STATUS));
		893	dev_info(rdev->dev, " R_008014_GRBM_STATUS2=0x%08X\n",
		894	RREG32(R_008014_GRBM_STATUS2));
		895	dev_info(rdev->dev, " R_000E50_SRBM_STATUS=0x%08X\n",
		896	RREG32(R_000E50_SRBM_STATUS));
2997	Serge	897	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		898	RREG32(CP_STALLED_STAT1));
		899	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		900	RREG32(CP_STALLED_STAT2));
		901	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		902	RREG32(CP_BUSY_STAT));
		903	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		904	RREG32(CP_STAT));
1221	serge	905	rv515_mc_resume(rdev, &save);
		906	return 0;
1128	serge	907	}
		908
2997	Serge	909	bool r600_gpu_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
1221	serge	910	{
1963	serge	911	u32 srbm_status;
		912	u32 grbm_status;
		913	u32 grbm_status2;
		914
		915	srbm_status = RREG32(R_000E50_SRBM_STATUS);
		916	grbm_status = RREG32(R_008010_GRBM_STATUS);
		917	grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
		918	if (!G_008010_GUI_ACTIVE(grbm_status)) {
2997	Serge	919	radeon_ring_lockup_update(ring);
1963	serge	920	return false;
		921	}
		922	/* force CP activities */
2997	Serge	923	radeon_ring_force_activity(rdev, ring);
		924	return radeon_ring_test_lockup(rdev, ring);
1963	serge	925	}
		926
		927	int r600_asic_reset(struct radeon_device *rdev)
		928	{
1221	serge	929	return r600_gpu_soft_reset(rdev);
		930	}
		931
2997	Serge	932	u32 r6xx_remap_render_backend(struct radeon_device *rdev,
		933	u32 tiling_pipe_num,
		934	u32 max_rb_num,
		935	u32 total_max_rb_num,
		936	u32 disabled_rb_mask)
1221	serge	937	{
2997	Serge	938	u32 rendering_pipe_num, rb_num_width, req_rb_num;
		939	u32 pipe_rb_ratio, pipe_rb_remain;
		940	u32 data = 0, mask = 1 << (max_rb_num - 1);
		941	unsigned i, j;
1221	serge	942
2997	Serge	943	/* mask out the RBs that don't exist on that asic */
		944	disabled_rb_mask \|= (0xff << max_rb_num) & 0xff;
1221	serge	945
2997	Serge	946	rendering_pipe_num = 1 << tiling_pipe_num;
		947	req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
		948	BUG_ON(rendering_pipe_num < req_rb_num);
1221	serge	949
2997	Serge	950	pipe_rb_ratio = rendering_pipe_num / req_rb_num;
		951	pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1221	serge	952
2997	Serge	953	if (rdev->family <= CHIP_RV740) {
		954	/* r6xx/r7xx */
		955	rb_num_width = 2;
		956	} else {
		957	/* eg+ */
		958	rb_num_width = 4;
		959	}
1221	serge	960
2997	Serge	961	for (i = 0; i < max_rb_num; i++) {
		962	if (!(mask & disabled_rb_mask)) {
		963	for (j = 0; j < pipe_rb_ratio; j++) {
		964	data <<= rb_num_width;
		965	data \|= max_rb_num - i - 1;
1221	serge	966	}
2997	Serge	967	if (pipe_rb_remain) {
		968	data <<= rb_num_width;
		969	data \|= max_rb_num - i - 1;
		970	pipe_rb_remain--;
1221	serge	971	}
2997	Serge	972	}
		973	mask >>= 1;
		974	}
1221	serge	975
2997	Serge	976	return data;
1221	serge	977	}
		978
		979	int r600_count_pipe_bits(uint32_t val)
		980	{
		981	int i, ret = 0;
		982
		983	for (i = 0; i < 32; i++) {
		984	ret += val & 1;
		985	val >>= 1;
		986	}
		987	return ret;
		988	}
		989
2997	Serge	990	static void r600_gpu_init(struct radeon_device *rdev)
1221	serge	991	{
		992	u32 tiling_config;
		993	u32 ramcfg;
1430	serge	994	u32 cc_rb_backend_disable;
		995	u32 cc_gc_shader_pipe_config;
1221	serge	996	u32 tmp;
		997	int i, j;
		998	u32 sq_config;
		999	u32 sq_gpr_resource_mgmt_1 = 0;
		1000	u32 sq_gpr_resource_mgmt_2 = 0;
		1001	u32 sq_thread_resource_mgmt = 0;
		1002	u32 sq_stack_resource_mgmt_1 = 0;
		1003	u32 sq_stack_resource_mgmt_2 = 0;
2997	Serge	1004	u32 disabled_rb_mask;
1221	serge	1005
2997	Serge	1006	rdev->config.r600.tiling_group_size = 256;
1221	serge	1007	switch (rdev->family) {
		1008	case CHIP_R600:
		1009	rdev->config.r600.max_pipes = 4;
		1010	rdev->config.r600.max_tile_pipes = 8;
		1011	rdev->config.r600.max_simds = 4;
		1012	rdev->config.r600.max_backends = 4;
		1013	rdev->config.r600.max_gprs = 256;
		1014	rdev->config.r600.max_threads = 192;
		1015	rdev->config.r600.max_stack_entries = 256;
		1016	rdev->config.r600.max_hw_contexts = 8;
		1017	rdev->config.r600.max_gs_threads = 16;
		1018	rdev->config.r600.sx_max_export_size = 128;
		1019	rdev->config.r600.sx_max_export_pos_size = 16;
		1020	rdev->config.r600.sx_max_export_smx_size = 128;
		1021	rdev->config.r600.sq_num_cf_insts = 2;
		1022	break;
		1023	case CHIP_RV630:
		1024	case CHIP_RV635:
		1025	rdev->config.r600.max_pipes = 2;
		1026	rdev->config.r600.max_tile_pipes = 2;
		1027	rdev->config.r600.max_simds = 3;
		1028	rdev->config.r600.max_backends = 1;
		1029	rdev->config.r600.max_gprs = 128;
		1030	rdev->config.r600.max_threads = 192;
		1031	rdev->config.r600.max_stack_entries = 128;
		1032	rdev->config.r600.max_hw_contexts = 8;
		1033	rdev->config.r600.max_gs_threads = 4;
		1034	rdev->config.r600.sx_max_export_size = 128;
		1035	rdev->config.r600.sx_max_export_pos_size = 16;
		1036	rdev->config.r600.sx_max_export_smx_size = 128;
		1037	rdev->config.r600.sq_num_cf_insts = 2;
		1038	break;
		1039	case CHIP_RV610:
		1040	case CHIP_RV620:
		1041	case CHIP_RS780:
		1042	case CHIP_RS880:
		1043	rdev->config.r600.max_pipes = 1;
		1044	rdev->config.r600.max_tile_pipes = 1;
		1045	rdev->config.r600.max_simds = 2;
		1046	rdev->config.r600.max_backends = 1;
		1047	rdev->config.r600.max_gprs = 128;
		1048	rdev->config.r600.max_threads = 192;
		1049	rdev->config.r600.max_stack_entries = 128;
		1050	rdev->config.r600.max_hw_contexts = 4;
		1051	rdev->config.r600.max_gs_threads = 4;
		1052	rdev->config.r600.sx_max_export_size = 128;
		1053	rdev->config.r600.sx_max_export_pos_size = 16;
		1054	rdev->config.r600.sx_max_export_smx_size = 128;
		1055	rdev->config.r600.sq_num_cf_insts = 1;
		1056	break;
		1057	case CHIP_RV670:
		1058	rdev->config.r600.max_pipes = 4;
		1059	rdev->config.r600.max_tile_pipes = 4;
		1060	rdev->config.r600.max_simds = 4;
		1061	rdev->config.r600.max_backends = 4;
		1062	rdev->config.r600.max_gprs = 192;
		1063	rdev->config.r600.max_threads = 192;
		1064	rdev->config.r600.max_stack_entries = 256;
		1065	rdev->config.r600.max_hw_contexts = 8;
		1066	rdev->config.r600.max_gs_threads = 16;
		1067	rdev->config.r600.sx_max_export_size = 128;
		1068	rdev->config.r600.sx_max_export_pos_size = 16;
		1069	rdev->config.r600.sx_max_export_smx_size = 128;
		1070	rdev->config.r600.sq_num_cf_insts = 2;
		1071	break;
		1072	default:
		1073	break;
		1074	}
		1075
		1076	/* Initialize HDP */
		1077	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1078	WREG32((0x2c14 + j), 0x00000000);
		1079	WREG32((0x2c18 + j), 0x00000000);
		1080	WREG32((0x2c1c + j), 0x00000000);
		1081	WREG32((0x2c20 + j), 0x00000000);
		1082	WREG32((0x2c24 + j), 0x00000000);
		1083	}
		1084
		1085	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1086
		1087	/* Setup tiling */
		1088	tiling_config = 0;
		1089	ramcfg = RREG32(RAMCFG);
		1090	switch (rdev->config.r600.max_tile_pipes) {
		1091	case 1:
		1092	tiling_config \|= PIPE_TILING(0);
		1093	break;
		1094	case 2:
		1095	tiling_config \|= PIPE_TILING(1);
		1096	break;
		1097	case 4:
		1098	tiling_config \|= PIPE_TILING(2);
		1099	break;
		1100	case 8:
		1101	tiling_config \|= PIPE_TILING(3);
		1102	break;
		1103	default:
		1104	break;
		1105	}
1430	serge	1106	rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
		1107	rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1221	serge	1108	tiling_config \|= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1963	serge	1109	tiling_config \|= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
2997	Serge	1110
1221	serge	1111	tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
		1112	if (tmp > 3) {
		1113	tiling_config \|= ROW_TILING(3);
		1114	tiling_config \|= SAMPLE_SPLIT(3);
		1115	} else {
		1116	tiling_config \|= ROW_TILING(tmp);
		1117	tiling_config \|= SAMPLE_SPLIT(tmp);
		1118	}
		1119	tiling_config \|= BANK_SWAPS(1);
1430	serge	1120
		1121	cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
2997	Serge	1122	tmp = R6XX_MAX_BACKENDS -
		1123	r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
		1124	if (tmp < rdev->config.r600.max_backends) {
		1125	rdev->config.r600.max_backends = tmp;
		1126	}
1430	serge	1127
2997	Serge	1128	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
		1129	tmp = R6XX_MAX_PIPES -
		1130	r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
		1131	if (tmp < rdev->config.r600.max_pipes) {
		1132	rdev->config.r600.max_pipes = tmp;
		1133	}
		1134	tmp = R6XX_MAX_SIMDS -
		1135	r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
		1136	if (tmp < rdev->config.r600.max_simds) {
		1137	rdev->config.r600.max_simds = tmp;
		1138	}
1430	serge	1139
2997	Serge	1140	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
		1141	tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
		1142	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
		1143	R6XX_MAX_BACKENDS, disabled_rb_mask);
		1144	tiling_config \|= tmp << 16;
		1145	rdev->config.r600.backend_map = tmp;
		1146
1963	serge	1147	rdev->config.r600.tile_config = tiling_config;
1221	serge	1148	WREG32(GB_TILING_CONFIG, tiling_config);
		1149	WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
		1150	WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
		1151
1430	serge	1152	tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1221	serge	1153	WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
		1154	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
		1155
		1156	/* Setup some CP states */
		1157	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \| ROQ_IB2_START(0x2b)));
		1158	WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) \| ROQ_END(0x40)));
		1159
		1160	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \| SYNC_GRADIENT \|
		1161	SYNC_WALKER \| SYNC_ALIGNER));
		1162	/* Setup various GPU states */
		1163	if (rdev->family == CHIP_RV670)
		1164	WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
		1165
		1166	tmp = RREG32(SX_DEBUG_1);
		1167	tmp \|= SMX_EVENT_RELEASE;
		1168	if ((rdev->family > CHIP_R600))
		1169	tmp \|= ENABLE_NEW_SMX_ADDRESS;
		1170	WREG32(SX_DEBUG_1, tmp);
		1171
		1172	if (((rdev->family) == CHIP_R600) \|\|
		1173	((rdev->family) == CHIP_RV630) \|\|
		1174	((rdev->family) == CHIP_RV610) \|\|
		1175	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1176	((rdev->family) == CHIP_RS780) \|\|
		1177	((rdev->family) == CHIP_RS880)) {
1221	serge	1178	WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
		1179	} else {
		1180	WREG32(DB_DEBUG, 0);
		1181	}
		1182	WREG32(DB_WATERMARKS, (DEPTH_FREE(4) \| DEPTH_CACHELINE_FREE(16) \|
		1183	DEPTH_FLUSH(16) \| DEPTH_PENDING_FREE(4)));
		1184
		1185	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1186	WREG32(VGT_NUM_INSTANCES, 0);
		1187
		1188	WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
		1189	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
		1190
		1191	tmp = RREG32(SQ_MS_FIFO_SIZES);
		1192	if (((rdev->family) == CHIP_RV610) \|\|
		1193	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1194	((rdev->family) == CHIP_RS780) \|\|
		1195	((rdev->family) == CHIP_RS880)) {
1221	serge	1196	tmp = (CACHE_FIFO_SIZE(0xa) \|
		1197	FETCH_FIFO_HIWATER(0xa) \|
		1198	DONE_FIFO_HIWATER(0xe0) \|
		1199	ALU_UPDATE_FIFO_HIWATER(0x8));
		1200	} else if (((rdev->family) == CHIP_R600) \|\|
		1201	((rdev->family) == CHIP_RV630)) {
		1202	tmp &= ~DONE_FIFO_HIWATER(0xff);
		1203	tmp \|= DONE_FIFO_HIWATER(0x4);
		1204	}
		1205	WREG32(SQ_MS_FIFO_SIZES, tmp);
		1206
		1207	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
		1208	* should be adjusted as needed by the 2D/3D drivers. This just sets default values
		1209	*/
		1210	sq_config = RREG32(SQ_CONFIG);
		1211	sq_config &= ~(PS_PRIO(3) \|
		1212	VS_PRIO(3) \|
		1213	GS_PRIO(3) \|
		1214	ES_PRIO(3));
		1215	sq_config \|= (DX9_CONSTS \|
		1216	VC_ENABLE \|
		1217	PS_PRIO(0) \|
		1218	VS_PRIO(1) \|
		1219	GS_PRIO(2) \|
		1220	ES_PRIO(3));
		1221
		1222	if ((rdev->family) == CHIP_R600) {
		1223	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) \|
		1224	NUM_VS_GPRS(124) \|
		1225	NUM_CLAUSE_TEMP_GPRS(4));
		1226	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) \|
		1227	NUM_ES_GPRS(0));
		1228	sq_thread_resource_mgmt = (NUM_PS_THREADS(136) \|
		1229	NUM_VS_THREADS(48) \|
		1230	NUM_GS_THREADS(4) \|
		1231	NUM_ES_THREADS(4));
		1232	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) \|
		1233	NUM_VS_STACK_ENTRIES(128));
		1234	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) \|
		1235	NUM_ES_STACK_ENTRIES(0));
		1236	} else if (((rdev->family) == CHIP_RV610) \|\|
		1237	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1238	((rdev->family) == CHIP_RS780) \|\|
		1239	((rdev->family) == CHIP_RS880)) {
1221	serge	1240	/* no vertex cache */
		1241	sq_config &= ~VC_ENABLE;
		1242
		1243	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1244	NUM_VS_GPRS(44) \|
		1245	NUM_CLAUSE_TEMP_GPRS(2));
		1246	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1247	NUM_ES_GPRS(17));
		1248	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1249	NUM_VS_THREADS(78) \|
		1250	NUM_GS_THREADS(4) \|
		1251	NUM_ES_THREADS(31));
		1252	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1253	NUM_VS_STACK_ENTRIES(40));
		1254	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1255	NUM_ES_STACK_ENTRIES(16));
		1256	} else if (((rdev->family) == CHIP_RV630) \|\|
		1257	((rdev->family) == CHIP_RV635)) {
		1258	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1259	NUM_VS_GPRS(44) \|
		1260	NUM_CLAUSE_TEMP_GPRS(2));
		1261	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) \|
		1262	NUM_ES_GPRS(18));
		1263	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1264	NUM_VS_THREADS(78) \|
		1265	NUM_GS_THREADS(4) \|
		1266	NUM_ES_THREADS(31));
		1267	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1268	NUM_VS_STACK_ENTRIES(40));
		1269	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1270	NUM_ES_STACK_ENTRIES(16));
		1271	} else if ((rdev->family) == CHIP_RV670) {
		1272	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1273	NUM_VS_GPRS(44) \|
		1274	NUM_CLAUSE_TEMP_GPRS(2));
		1275	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1276	NUM_ES_GPRS(17));
		1277	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1278	NUM_VS_THREADS(78) \|
		1279	NUM_GS_THREADS(4) \|
		1280	NUM_ES_THREADS(31));
		1281	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) \|
		1282	NUM_VS_STACK_ENTRIES(64));
		1283	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) \|
		1284	NUM_ES_STACK_ENTRIES(64));
		1285	}
		1286
		1287	WREG32(SQ_CONFIG, sq_config);
		1288	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		1289	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		1290	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		1291	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		1292	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		1293
		1294	if (((rdev->family) == CHIP_RV610) \|\|
		1295	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1296	((rdev->family) == CHIP_RS780) \|\|
		1297	((rdev->family) == CHIP_RS880)) {
1221	serge	1298	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
		1299	} else {
		1300	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
		1301	}
		1302
		1303	/* More default values. 2D/3D driver should adjust as needed */
		1304	WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) \| S0_Y(0x4) \|
		1305	S1_X(0x4) \| S1_Y(0xc)));
		1306	WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) \| S0_Y(0xe) \|
		1307	S1_X(0x2) \| S1_Y(0x2) \|
		1308	S2_X(0xa) \| S2_Y(0x6) \|
		1309	S3_X(0x6) \| S3_Y(0xa)));
		1310	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) \| S0_Y(0xb) \|
		1311	S1_X(0x4) \| S1_Y(0xc) \|
		1312	S2_X(0x1) \| S2_Y(0x6) \|
		1313	S3_X(0xa) \| S3_Y(0xe)));
		1314	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) \| S4_Y(0x1) \|
		1315	S5_X(0x0) \| S5_Y(0x0) \|
		1316	S6_X(0xb) \| S6_Y(0x4) \|
		1317	S7_X(0x7) \| S7_Y(0x8)));
		1318
		1319	WREG32(VGT_STRMOUT_EN, 0);
		1320	tmp = rdev->config.r600.max_pipes * 16;
		1321	switch (rdev->family) {
		1322	case CHIP_RV610:
1268	serge	1323	case CHIP_RV620:
1221	serge	1324	case CHIP_RS780:
1268	serge	1325	case CHIP_RS880:
1221	serge	1326	tmp += 32;
		1327	break;
		1328	case CHIP_RV670:
		1329	tmp += 128;
		1330	break;
		1331	default:
		1332	break;
		1333	}
		1334	if (tmp > 256) {
		1335	tmp = 256;
		1336	}
		1337	WREG32(VGT_ES_PER_GS, 128);
		1338	WREG32(VGT_GS_PER_ES, tmp);
		1339	WREG32(VGT_GS_PER_VS, 2);
		1340	WREG32(VGT_GS_VERTEX_REUSE, 16);
		1341
		1342	/* more default values. 2D/3D driver should adjust as needed */
		1343	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		1344	WREG32(VGT_STRMOUT_EN, 0);
		1345	WREG32(SX_MISC, 0);
		1346	WREG32(PA_SC_MODE_CNTL, 0);
		1347	WREG32(PA_SC_AA_CONFIG, 0);
		1348	WREG32(PA_SC_LINE_STIPPLE, 0);
		1349	WREG32(SPI_INPUT_Z, 0);
		1350	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
		1351	WREG32(CB_COLOR7_FRAG, 0);
		1352
		1353	/* Clear render buffer base addresses */
		1354	WREG32(CB_COLOR0_BASE, 0);
		1355	WREG32(CB_COLOR1_BASE, 0);
		1356	WREG32(CB_COLOR2_BASE, 0);
		1357	WREG32(CB_COLOR3_BASE, 0);
		1358	WREG32(CB_COLOR4_BASE, 0);
		1359	WREG32(CB_COLOR5_BASE, 0);
		1360	WREG32(CB_COLOR6_BASE, 0);
		1361	WREG32(CB_COLOR7_BASE, 0);
		1362	WREG32(CB_COLOR7_FRAG, 0);
		1363
		1364	switch (rdev->family) {
		1365	case CHIP_RV610:
1268	serge	1366	case CHIP_RV620:
1221	serge	1367	case CHIP_RS780:
1268	serge	1368	case CHIP_RS880:
1221	serge	1369	tmp = TC_L2_SIZE(8);
		1370	break;
		1371	case CHIP_RV630:
		1372	case CHIP_RV635:
		1373	tmp = TC_L2_SIZE(4);
		1374	break;
		1375	case CHIP_R600:
		1376	tmp = TC_L2_SIZE(0) \| L2_DISABLE_LATE_HIT;
		1377	break;
		1378	default:
		1379	tmp = TC_L2_SIZE(0);
		1380	break;
		1381	}
		1382	WREG32(TC_CNTL, tmp);
		1383
		1384	tmp = RREG32(HDP_HOST_PATH_CNTL);
		1385	WREG32(HDP_HOST_PATH_CNTL, tmp);
		1386
		1387	tmp = RREG32(ARB_POP);
		1388	tmp \|= ENABLE_TC128;
		1389	WREG32(ARB_POP, tmp);
		1390
		1391	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1392	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA \|
		1393	NUM_CLIP_SEQ(3)));
		1394	WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
2997	Serge	1395	WREG32(VC_ENHANCE, 0);
1221	serge	1396	}
		1397
		1398
1128	serge	1399	/*
		1400	* Indirect registers accessor
		1401	*/
1221	serge	1402	u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1128	serge	1403	{
1221	serge	1404	u32 r;
1128	serge	1405
1221	serge	1406	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1407	(void)RREG32(PCIE_PORT_INDEX);
		1408	r = RREG32(PCIE_PORT_DATA);
1128	serge	1409	return r;
		1410	}
		1411
1221	serge	1412	void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1128	serge	1413	{
1221	serge	1414	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1415	(void)RREG32(PCIE_PORT_INDEX);
		1416	WREG32(PCIE_PORT_DATA, (v));
		1417	(void)RREG32(PCIE_PORT_DATA);
1128	serge	1418	}
1221	serge	1419
		1420	/*
		1421	* CP & Ring
		1422	*/
		1423	void r600_cp_stop(struct radeon_device *rdev)
		1424	{
1963	serge	1425	// radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1221	serge	1426	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1963	serge	1427	WREG32(SCRATCH_UMSK, 0);
1221	serge	1428	}
1413	serge	1429
		1430	int r600_init_microcode(struct radeon_device *rdev)
		1431	{
		1432	struct platform_device *pdev;
		1433	const char *chip_name;
		1434	const char *rlc_chip_name;
		1435	size_t pfp_req_size, me_req_size, rlc_req_size;
		1436	char fw_name[30];
		1437	int err;
		1438
		1439	DRM_DEBUG("\n");
		1440
		1441	pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
		1442	err = IS_ERR(pdev);
		1443	if (err) {
		1444	printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
		1445	return -EINVAL;
		1446	}
		1447
		1448	switch (rdev->family) {
		1449	case CHIP_R600:
		1450	chip_name = "R600";
		1451	rlc_chip_name = "R600";
		1452	break;
		1453	case CHIP_RV610:
		1454	chip_name = "RV610";
		1455	rlc_chip_name = "R600";
		1456	break;
		1457	case CHIP_RV630:
		1458	chip_name = "RV630";
		1459	rlc_chip_name = "R600";
		1460	break;
		1461	case CHIP_RV620:
		1462	chip_name = "RV620";
		1463	rlc_chip_name = "R600";
		1464	break;
		1465	case CHIP_RV635:
		1466	chip_name = "RV635";
		1467	rlc_chip_name = "R600";
		1468	break;
		1469	case CHIP_RV670:
		1470	chip_name = "RV670";
		1471	rlc_chip_name = "R600";
		1472	break;
		1473	case CHIP_RS780:
		1474	case CHIP_RS880:
		1475	chip_name = "RS780";
		1476	rlc_chip_name = "R600";
		1477	break;
		1478	case CHIP_RV770:
		1479	chip_name = "RV770";
		1480	rlc_chip_name = "R700";
		1481	break;
		1482	case CHIP_RV730:
		1483	case CHIP_RV740:
		1484	chip_name = "RV730";
		1485	rlc_chip_name = "R700";
		1486	break;
		1487	case CHIP_RV710:
		1488	chip_name = "RV710";
		1489	rlc_chip_name = "R700";
		1490	break;
1963	serge	1491	case CHIP_CEDAR:
		1492	chip_name = "CEDAR";
		1493	rlc_chip_name = "CEDAR";
		1494	break;
		1495	case CHIP_REDWOOD:
		1496	chip_name = "REDWOOD";
		1497	rlc_chip_name = "REDWOOD";
		1498	break;
		1499	case CHIP_JUNIPER:
		1500	chip_name = "JUNIPER";
		1501	rlc_chip_name = "JUNIPER";
		1502	break;
		1503	case CHIP_CYPRESS:
		1504	case CHIP_HEMLOCK:
		1505	chip_name = "CYPRESS";
		1506	rlc_chip_name = "CYPRESS";
		1507	break;
		1508	case CHIP_PALM:
		1509	chip_name = "PALM";
		1510	rlc_chip_name = "SUMO";
		1511	break;
1986	serge	1512	case CHIP_SUMO:
		1513	chip_name = "SUMO";
		1514	rlc_chip_name = "SUMO";
		1515	break;
		1516	case CHIP_SUMO2:
		1517	chip_name = "SUMO2";
		1518	rlc_chip_name = "SUMO";
		1519	break;
1413	serge	1520	default: BUG();
		1521	}
		1522
1963	serge	1523	if (rdev->family >= CHIP_CEDAR) {
		1524	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		1525	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		1526	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		1527	} else if (rdev->family >= CHIP_RV770) {
1413	serge	1528	pfp_req_size = R700_PFP_UCODE_SIZE * 4;
		1529	me_req_size = R700_PM4_UCODE_SIZE * 4;
		1530	rlc_req_size = R700_RLC_UCODE_SIZE * 4;
		1531	} else {
		1532	pfp_req_size = PFP_UCODE_SIZE * 4;
		1533	me_req_size = PM4_UCODE_SIZE * 12;
		1534	rlc_req_size = RLC_UCODE_SIZE * 4;
		1535	}
		1536
		1537	DRM_INFO("Loading %s Microcode\n", chip_name);
		1538
		1539	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
		1540	err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
		1541	if (err)
		1542	goto out;
		1543	if (rdev->pfp_fw->size != pfp_req_size) {
		1544	printk(KERN_ERR
		1545	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1546	rdev->pfp_fw->size, fw_name);
		1547	err = -EINVAL;
		1548	goto out;
		1549	}
		1550
		1551	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
		1552	err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
		1553	if (err)
		1554	goto out;
		1555	if (rdev->me_fw->size != me_req_size) {
		1556	printk(KERN_ERR
		1557	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1558	rdev->me_fw->size, fw_name);
		1559	err = -EINVAL;
		1560	}
		1561
		1562	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
		1563	err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
		1564	if (err)
		1565	goto out;
		1566	if (rdev->rlc_fw->size != rlc_req_size) {
		1567	printk(KERN_ERR
		1568	"r600_rlc: Bogus length %zu in firmware \"%s\"\n",
		1569	rdev->rlc_fw->size, fw_name);
		1570	err = -EINVAL;
		1571	}
		1572
		1573	out:
		1574	platform_device_unregister(pdev);
		1575
		1576	if (err) {
		1577	if (err != -EINVAL)
		1578	printk(KERN_ERR
		1579	"r600_cp: Failed to load firmware \"%s\"\n",
		1580	fw_name);
		1581	release_firmware(rdev->pfp_fw);
		1582	rdev->pfp_fw = NULL;
		1583	release_firmware(rdev->me_fw);
		1584	rdev->me_fw = NULL;
		1585	release_firmware(rdev->rlc_fw);
		1586	rdev->rlc_fw = NULL;
		1587	}
		1588	return err;
		1589	}
		1590
		1591	static int r600_cp_load_microcode(struct radeon_device *rdev)
		1592	{
		1593	const __be32 *fw_data;
		1594	int i;
		1595
		1596	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1597	return -EINVAL;
		1598
		1599	r600_cp_stop(rdev);
		1600
1963	serge	1601	WREG32(CP_RB_CNTL,
		1602	#ifdef __BIG_ENDIAN
		1603	BUF_SWAP_32BIT \|
		1604	#endif
		1605	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
1413	serge	1606
		1607	/* Reset cp */
		1608	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1609	RREG32(GRBM_SOFT_RESET);
		1610	mdelay(15);
		1611	WREG32(GRBM_SOFT_RESET, 0);
		1612
		1613	WREG32(CP_ME_RAM_WADDR, 0);
		1614
		1615	fw_data = (const __be32 *)rdev->me_fw->data;
		1616	WREG32(CP_ME_RAM_WADDR, 0);
		1617	for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
		1618	WREG32(CP_ME_RAM_DATA,
		1619	be32_to_cpup(fw_data++));
		1620
		1621	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1622	WREG32(CP_PFP_UCODE_ADDR, 0);
		1623	for (i = 0; i < PFP_UCODE_SIZE; i++)
		1624	WREG32(CP_PFP_UCODE_DATA,
		1625	be32_to_cpup(fw_data++));
		1626
		1627	WREG32(CP_PFP_UCODE_ADDR, 0);
		1628	WREG32(CP_ME_RAM_WADDR, 0);
		1629	WREG32(CP_ME_RAM_RADDR, 0);
		1630	return 0;
		1631	}
		1632
1221	serge	1633	int r600_cp_start(struct radeon_device *rdev)
		1634	{
2997	Serge	1635	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1221	serge	1636	int r;
		1637	uint32_t cp_me;
		1638
2997	Serge	1639	r = radeon_ring_lock(rdev, ring, 7);
1221	serge	1640	if (r) {
		1641	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1642	return r;
		1643	}
2997	Serge	1644	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1645	radeon_ring_write(ring, 0x1);
1963	serge	1646	if (rdev->family >= CHIP_RV770) {
2997	Serge	1647	radeon_ring_write(ring, 0x0);
		1648	radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
1963	serge	1649	} else {
2997	Serge	1650	radeon_ring_write(ring, 0x3);
		1651	radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
1221	serge	1652	}
2997	Serge	1653	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1654	radeon_ring_write(ring, 0);
		1655	radeon_ring_write(ring, 0);
		1656	radeon_ring_unlock_commit(rdev, ring);
1221	serge	1657
		1658	cp_me = 0xff;
		1659	WREG32(R_0086D8_CP_ME_CNTL, cp_me);
		1660	return 0;
		1661	}
1413	serge	1662
		1663	int r600_cp_resume(struct radeon_device *rdev)
		1664	{
2997	Serge	1665	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1413	serge	1666	u32 tmp;
		1667	u32 rb_bufsz;
		1668	int r;
		1669
		1670	/* Reset cp */
		1671	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1672	RREG32(GRBM_SOFT_RESET);
		1673	mdelay(15);
		1674	WREG32(GRBM_SOFT_RESET, 0);
		1675
		1676	/* Set ring buffer size */
2997	Serge	1677	rb_bufsz = drm_order(ring->ring_size / 8);
1963	serge	1678	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
1413	serge	1679	#ifdef __BIG_ENDIAN
		1680	tmp \|= BUF_SWAP_32BIT;
		1681	#endif
		1682	WREG32(CP_RB_CNTL, tmp);
2997	Serge	1683	WREG32(CP_SEM_WAIT_TIMER, 0x0);
1413	serge	1684
		1685	/* Set the write pointer delay */
		1686	WREG32(CP_RB_WPTR_DELAY, 0);
		1687
		1688	/* Initialize the ring buffer's read and write pointers */
		1689	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1690	WREG32(CP_RB_RPTR_WR, 0);
2997	Serge	1691	ring->wptr = 0;
		1692	WREG32(CP_RB_WPTR, ring->wptr);
1963	serge	1693
		1694	/* set the wb address whether it's enabled or not */
		1695	WREG32(CP_RB_RPTR_ADDR,
		1696	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1697	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1698	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1699
		1700	if (rdev->wb.enabled)
		1701	WREG32(SCRATCH_UMSK, 0xff);
		1702	else {
		1703	tmp \|= RB_NO_UPDATE;
		1704	WREG32(SCRATCH_UMSK, 0);
		1705	}
		1706
1413	serge	1707	mdelay(1);
		1708	WREG32(CP_RB_CNTL, tmp);
		1709
2997	Serge	1710	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1413	serge	1711	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1712
2997	Serge	1713	ring->rptr = RREG32(CP_RB_RPTR);
1413	serge	1714
		1715	r600_cp_start(rdev);
2997	Serge	1716	ring->ready = true;
		1717	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
1413	serge	1718	if (r) {
2997	Serge	1719	ring->ready = false;
1413	serge	1720	return r;
		1721	}
		1722	return 0;
		1723	}
		1724
2997	Serge	1725	void r600_ring_init(struct radeon_device rdev, struct radeon_ring ring, unsigned ring_size)
1221	serge	1726	{
1233	serge	1727	u32 rb_bufsz;
2997	Serge	1728	int r;
1221	serge	1729
1233	serge	1730	/* Align ring size */
		1731	rb_bufsz = drm_order(ring_size / 8);
		1732	ring_size = (1 << (rb_bufsz + 1)) * 4;
2997	Serge	1733	ring->ring_size = ring_size;
		1734	ring->align_mask = 16 - 1;
		1735
		1736	if (radeon_ring_supports_scratch_reg(rdev, ring)) {
		1737	r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
		1738	if (r) {
		1739	DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
		1740	ring->rptr_save_reg = 0;
		1741	}
		1742	}
1233	serge	1743	}
		1744
1963	serge	1745	void r600_cp_fini(struct radeon_device *rdev)
		1746	{
2997	Serge	1747	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	1748	r600_cp_stop(rdev);
2997	Serge	1749	radeon_ring_fini(rdev, ring);
		1750	radeon_scratch_free(rdev, ring->rptr_save_reg);
1963	serge	1751	}
1233	serge	1752
1963	serge	1753
1233	serge	1754	/*
		1755	* GPU scratch registers helpers function.
		1756	*/
		1757	void r600_scratch_init(struct radeon_device *rdev)
		1758	{
		1759	int i;
		1760
		1761	rdev->scratch.num_reg = 7;
1963	serge	1762	rdev->scratch.reg_base = SCRATCH_REG0;
1233	serge	1763	for (i = 0; i < rdev->scratch.num_reg; i++) {
		1764	rdev->scratch.free[i] = true;
1963	serge	1765	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
1233	serge	1766	}
		1767	}
1413	serge	1768
2997	Serge	1769	int r600_ring_test(struct radeon_device rdev, struct radeon_ring ring)
1413	serge	1770	{
		1771	uint32_t scratch;
		1772	uint32_t tmp = 0;
		1773	unsigned i;
		1774	int r;
		1775
		1776	r = radeon_scratch_get(rdev, &scratch);
		1777	if (r) {
		1778	DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
		1779	return r;
		1780	}
		1781	WREG32(scratch, 0xCAFEDEAD);
2997	Serge	1782	r = radeon_ring_lock(rdev, ring, 3);
1413	serge	1783	if (r) {
2997	Serge	1784	DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
1413	serge	1785	radeon_scratch_free(rdev, scratch);
		1786	return r;
		1787	}
2997	Serge	1788	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1789	radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		1790	radeon_ring_write(ring, 0xDEADBEEF);
		1791	radeon_ring_unlock_commit(rdev, ring);
1413	serge	1792	for (i = 0; i < rdev->usec_timeout; i++) {
		1793	tmp = RREG32(scratch);
		1794	if (tmp == 0xDEADBEEF)
		1795	break;
		1796	DRM_UDELAY(1);
		1797	}
		1798	if (i < rdev->usec_timeout) {
2997	Serge	1799	DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
1413	serge	1800	} else {
2997	Serge	1801	DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
		1802	ring->idx, scratch, tmp);
1413	serge	1803	r = -EINVAL;
		1804	}
		1805	radeon_scratch_free(rdev, scratch);
		1806	return r;
		1807	}
1963	serge	1808
1413	serge	1809	void r600_fence_ring_emit(struct radeon_device *rdev,
		1810	struct radeon_fence *fence)
		1811	{
2997	Serge	1812	struct radeon_ring *ring = &rdev->ring[fence->ring];
		1813
1963	serge	1814	if (rdev->wb.use_event) {
2997	Serge	1815	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		1816	/* flush read cache over gart */
		1817	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		1818	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA \|
		1819	PACKET3_VC_ACTION_ENA \|
		1820	PACKET3_SH_ACTION_ENA);
		1821	radeon_ring_write(ring, 0xFFFFFFFF);
		1822	radeon_ring_write(ring, 0);
		1823	radeon_ring_write(ring, 10); /* poll interval */
1963	serge	1824	/* EVENT_WRITE_EOP - flush caches, send int */
2997	Serge	1825	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		1826	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) \| EVENT_INDEX(5));
		1827	radeon_ring_write(ring, addr & 0xffffffff);
		1828	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		1829	radeon_ring_write(ring, fence->seq);
		1830	radeon_ring_write(ring, 0);
1963	serge	1831	} else {
2997	Serge	1832	/* flush read cache over gart */
		1833	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		1834	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA \|
		1835	PACKET3_VC_ACTION_ENA \|
		1836	PACKET3_SH_ACTION_ENA);
		1837	radeon_ring_write(ring, 0xFFFFFFFF);
		1838	radeon_ring_write(ring, 0);
		1839	radeon_ring_write(ring, 10); /* poll interval */
		1840	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
		1841	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) \| EVENT_INDEX(0));
1430	serge	1842	/* wait for 3D idle clean */
2997	Serge	1843	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1844	radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		1845	radeon_ring_write(ring, WAIT_3D_IDLE_bit \| WAIT_3D_IDLECLEAN_bit);
1413	serge	1846	/* Emit fence sequence & fire IRQ */
2997	Serge	1847	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1848	radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		1849	radeon_ring_write(ring, fence->seq);
1413	serge	1850	/* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2997	Serge	1851	radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
		1852	radeon_ring_write(ring, RB_INT_STAT);
1963	serge	1853	}
1413	serge	1854	}
1963	serge	1855
2997	Serge	1856	void r600_semaphore_ring_emit(struct radeon_device *rdev,
		1857	struct radeon_ring *ring,
		1858	struct radeon_semaphore *semaphore,
		1859	bool emit_wait)
		1860	{
		1861	uint64_t addr = semaphore->gpu_addr;
		1862	unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
		1863
		1864	if (rdev->family < CHIP_CAYMAN)
		1865	sel \|= PACKET3_SEM_WAIT_ON_SIGNAL;
		1866
		1867	radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
		1868	radeon_ring_write(ring, addr & 0xffffffff);
		1869	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| sel);
		1870	}
		1871
2005	serge	1872	int r600_copy_blit(struct radeon_device *rdev,
2997	Serge	1873	uint64_t src_offset,
		1874	uint64_t dst_offset,
		1875	unsigned num_gpu_pages,
		1876	struct radeon_fence **fence)
2005	serge	1877	{
2997	Serge	1878	struct radeon_semaphore *sem = NULL;
		1879	struct radeon_sa_bo *vb = NULL;
2005	serge	1880	int r;
1963	serge	1881
2997	Serge	1882	r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem);
2005	serge	1883	if (r) {
		1884	return r;
		1885	}
2997	Serge	1886	r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb);
		1887	r600_blit_done_copy(rdev, fence, vb, sem);
2005	serge	1888	return 0;
		1889	}
		1890
1221	serge	1891	int r600_set_surface_reg(struct radeon_device *rdev, int reg,
		1892	uint32_t tiling_flags, uint32_t pitch,
		1893	uint32_t offset, uint32_t obj_size)
		1894	{
		1895	/* FIXME: implement */
		1896	return 0;
		1897	}
		1898
		1899	void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
		1900	{
		1901	/* FIXME: implement */
		1902	}
		1903
2997	Serge	1904	static int r600_startup(struct radeon_device *rdev)
1221	serge	1905	{
2997	Serge	1906	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1221	serge	1907	int r;
		1908
1963	serge	1909	/* enable pcie gen2 link */
		1910	r600_pcie_gen2_enable(rdev);
		1911
1413	serge	1912	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		1913	r = r600_init_microcode(rdev);
		1914	if (r) {
		1915	DRM_ERROR("Failed to load firmware!\n");
		1916	return r;
		1917	}
		1918	}
		1919
1221	serge	1920	r600_mc_program(rdev);
		1921	if (rdev->flags & RADEON_IS_AGP) {
		1922	r600_agp_enable(rdev);
		1923	} else {
		1924	r = r600_pcie_gart_enable(rdev);
		1925	if (r)
		1926	return r;
		1927	}
		1928	r600_gpu_init(rdev);
2005	serge	1929	r = r600_blit_init(rdev);
		1930	if (r) {
		1931	// r600_blit_fini(rdev);
2997	Serge	1932	rdev->asic->copy.copy = NULL;
2005	serge	1933	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		1934	}
1221	serge	1935
2005	serge	1936	/* allocate wb buffer */
		1937	r = radeon_wb_init(rdev);
		1938	if (r)
		1939	return r;
		1940
		1941	/* Enable IRQ */
		1942	r = r600_irq_init(rdev);
		1943	if (r) {
		1944	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		1945	// radeon_irq_kms_fini(rdev);
		1946	return r;
		1947	}
		1948	r600_irq_set(rdev);
		1949
2997	Serge	1950	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
		1951	R600_CP_RB_RPTR, R600_CP_RB_WPTR,
		1952	0, 0xfffff, RADEON_CP_PACKET2);
		1953
1413	serge	1954	if (r)
		1955	return r;
		1956	r = r600_cp_load_microcode(rdev);
		1957	if (r)
		1958	return r;
		1959	r = r600_cp_resume(rdev);
		1960	if (r)
		1961	return r;
1963	serge	1962
1221	serge	1963	return 0;
		1964	}
		1965
		1966	void r600_vga_set_state(struct radeon_device *rdev, bool state)
		1967	{
		1968	uint32_t temp;
		1969
		1970	temp = RREG32(CONFIG_CNTL);
		1971	if (state == false) {
		1972	temp &= ~(1<<0);
		1973	temp \|= (1<<1);
		1974	} else {
		1975	temp &= ~(1<<1);
		1976	}
		1977	WREG32(CONFIG_CNTL, temp);
		1978	}
		1979
		1980
		1981
		1982
		1983
		1984	/* Plan is to move initialization in that function and use
		1985	* helper function so that radeon_device_init pretty much
		1986	* do nothing more than calling asic specific function. This
		1987	* should also allow to remove a bunch of callback function
		1988	* like vram_info.
		1989	*/
		1990	int r600_init(struct radeon_device *rdev)
		1991	{
		1992	int r;
		1993
		1994	if (r600_debugfs_mc_info_init(rdev)) {
		1995	DRM_ERROR("Failed to register debugfs file for mc !\n");
		1996	}
		1997	/* Read BIOS */
		1998	if (!radeon_get_bios(rdev)) {
		1999	if (ASIC_IS_AVIVO(rdev))
		2000	return -EINVAL;
		2001	}
		2002	/* Must be an ATOMBIOS */
		2003	if (!rdev->is_atom_bios) {
		2004	dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
		2005	return -EINVAL;
		2006	}
		2007	r = radeon_atombios_init(rdev);
		2008	if (r)
		2009	return r;
		2010	/* Post card if necessary */
1963	serge	2011	if (!radeon_card_posted(rdev)) {
1321	serge	2012	if (!rdev->bios) {
		2013	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		2014	return -EINVAL;
		2015	}
1221	serge	2016	DRM_INFO("GPU not posted. posting now...\n");
		2017	atom_asic_init(rdev->mode_info.atom_context);
		2018	}
		2019	/* Initialize scratch registers */
		2020	r600_scratch_init(rdev);
		2021	/* Initialize surface registers */
		2022	radeon_surface_init(rdev);
1268	serge	2023	/* Initialize clocks */
1221	serge	2024	radeon_get_clock_info(rdev->ddev);
		2025	/* Fence driver */
2004	serge	2026	r = radeon_fence_driver_init(rdev);
		2027	if (r)
		2028	return r;
1403	serge	2029	if (rdev->flags & RADEON_IS_AGP) {
		2030	r = radeon_agp_init(rdev);
		2031	if (r)
		2032	radeon_agp_disable(rdev);
		2033	}
1221	serge	2034	r = r600_mc_init(rdev);
		2035	if (r)
		2036	return r;
		2037	/* Memory manager */
1321	serge	2038	r = radeon_bo_init(rdev);
1221	serge	2039	if (r)
		2040	return r;
1321	serge	2041
2004	serge	2042	r = radeon_irq_kms_init(rdev);
		2043	if (r)
		2044	return r;
1321	serge	2045
2997	Serge	2046	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
		2047	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1221	serge	2048
2004	serge	2049	rdev->ih.ring_obj = NULL;
		2050	r600_ih_ring_init(rdev, 64 * 1024);
1221	serge	2051
		2052	r = r600_pcie_gart_init(rdev);
		2053	if (r)
		2054	return r;
		2055
1321	serge	2056	rdev->accel_working = true;
1221	serge	2057	r = r600_startup(rdev);
		2058	if (r) {
1428	serge	2059	dev_err(rdev->dev, "disabling GPU acceleration\n");
1221	serge	2060	r600_pcie_gart_fini(rdev);
		2061	rdev->accel_working = false;
		2062	}
2005	serge	2063
1221	serge	2064	return 0;
		2065	}
		2066
2004	serge	2067	/*
		2068	* CS stuff
		2069	*/
		2070	void r600_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		2071	{
2997	Serge	2072	struct radeon_ring *ring = &rdev->ring[ib->ring];
		2073	u32 next_rptr;
		2074
		2075	if (ring->rptr_save_reg) {
		2076	next_rptr = ring->wptr + 3 + 4;
		2077	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2078	radeon_ring_write(ring, ((ring->rptr_save_reg -
		2079	PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		2080	radeon_ring_write(ring, next_rptr);
		2081	} else if (rdev->wb.enabled) {
		2082	next_rptr = ring->wptr + 5 + 4;
		2083	radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
		2084	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		2085	radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) \| (1 << 18));
		2086	radeon_ring_write(ring, next_rptr);
		2087	radeon_ring_write(ring, 0);
		2088	}
		2089
		2090	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		2091	radeon_ring_write(ring,
2004	serge	2092	#ifdef __BIG_ENDIAN
		2093	(2 << 0) \|
		2094	#endif
		2095	(ib->gpu_addr & 0xFFFFFFFC));
2997	Serge	2096	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
		2097	radeon_ring_write(ring, ib->length_dw);
2004	serge	2098	}
		2099
2997	Serge	2100	int r600_ib_test(struct radeon_device rdev, struct radeon_ring ring)
2004	serge	2101	{
2997	Serge	2102	struct radeon_ib ib;
2004	serge	2103	uint32_t scratch;
		2104	uint32_t tmp = 0;
		2105	unsigned i;
		2106	int r;
		2107
		2108	r = radeon_scratch_get(rdev, &scratch);
		2109	if (r) {
		2110	DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
		2111	return r;
		2112	}
		2113	WREG32(scratch, 0xCAFEDEAD);
2997	Serge	2114	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
2004	serge	2115	if (r) {
		2116	DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2997	Serge	2117	goto free_scratch;
2004	serge	2118	}
2997	Serge	2119	ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
		2120	ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		2121	ib.ptr[2] = 0xDEADBEEF;
		2122	ib.length_dw = 3;
		2123	r = radeon_ib_schedule(rdev, &ib, NULL);
2004	serge	2124	if (r) {
		2125	DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2997	Serge	2126	goto free_ib;
2004	serge	2127	}
2997	Serge	2128	r = radeon_fence_wait(ib.fence, false);
2004	serge	2129	if (r) {
		2130	DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2997	Serge	2131	goto free_ib;
2004	serge	2132	}
		2133	for (i = 0; i < rdev->usec_timeout; i++) {
		2134	tmp = RREG32(scratch);
		2135	if (tmp == 0xDEADBEEF)
		2136	break;
		2137	DRM_UDELAY(1);
		2138	}
		2139	if (i < rdev->usec_timeout) {
2997	Serge	2140	DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
2004	serge	2141	} else {
		2142	DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
		2143	scratch, tmp);
		2144	r = -EINVAL;
		2145	}
2997	Serge	2146	free_ib:
		2147	radeon_ib_free(rdev, &ib);
		2148	free_scratch:
2004	serge	2149	radeon_scratch_free(rdev, scratch);
		2150	return r;
		2151	}
		2152
		2153	/*
		2154	* Interrupts
		2155	*
		2156	* Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
		2157	* the same as the CP ring buffer, but in reverse. Rather than the CPU
		2158	* writing to the ring and the GPU consuming, the GPU writes to the ring
		2159	* and host consumes. As the host irq handler processes interrupts, it
		2160	* increments the rptr. When the rptr catches up with the wptr, all the
		2161	* current interrupts have been processed.
		2162	*/
		2163
		2164	void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
		2165	{
		2166	u32 rb_bufsz;
		2167
		2168	/* Align ring size */
		2169	rb_bufsz = drm_order(ring_size / 4);
		2170	ring_size = (1 << rb_bufsz) * 4;
		2171	rdev->ih.ring_size = ring_size;
		2172	rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
		2173	rdev->ih.rptr = 0;
		2174	}
		2175
2997	Serge	2176	int r600_ih_ring_alloc(struct radeon_device *rdev)
2004	serge	2177	{
		2178	int r;
		2179
		2180	/* Allocate ring buffer */
		2181	if (rdev->ih.ring_obj == NULL) {
		2182	r = radeon_bo_create(rdev, rdev->ih.ring_size,
		2183	PAGE_SIZE, true,
		2184	RADEON_GEM_DOMAIN_GTT,
2997	Serge	2185	NULL, &rdev->ih.ring_obj);
2004	serge	2186	if (r) {
		2187	DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
		2188	return r;
		2189	}
		2190	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		2191	if (unlikely(r != 0))
		2192	return r;
		2193	r = radeon_bo_pin(rdev->ih.ring_obj,
		2194	RADEON_GEM_DOMAIN_GTT,
		2195	&rdev->ih.gpu_addr);
		2196	if (r) {
		2197	radeon_bo_unreserve(rdev->ih.ring_obj);
		2198	DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
		2199	return r;
		2200	}
		2201	r = radeon_bo_kmap(rdev->ih.ring_obj,
		2202	(void **)&rdev->ih.ring);
		2203	radeon_bo_unreserve(rdev->ih.ring_obj);
		2204	if (r) {
		2205	DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
		2206	return r;
		2207	}
		2208	}
		2209	return 0;
		2210	}
		2211
2997	Serge	2212	void r600_ih_ring_fini(struct radeon_device *rdev)
2004	serge	2213	{
		2214	int r;
		2215	if (rdev->ih.ring_obj) {
		2216	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		2217	if (likely(r == 0)) {
		2218	radeon_bo_kunmap(rdev->ih.ring_obj);
		2219	radeon_bo_unpin(rdev->ih.ring_obj);
		2220	radeon_bo_unreserve(rdev->ih.ring_obj);
		2221	}
		2222	radeon_bo_unref(&rdev->ih.ring_obj);
		2223	rdev->ih.ring = NULL;
		2224	rdev->ih.ring_obj = NULL;
		2225	}
		2226	}
		2227
		2228	void r600_rlc_stop(struct radeon_device *rdev)
		2229	{
		2230
		2231	if ((rdev->family >= CHIP_RV770) &&
		2232	(rdev->family <= CHIP_RV740)) {
		2233	/* r7xx asics need to soft reset RLC before halting */
		2234	WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
		2235	RREG32(SRBM_SOFT_RESET);
2997	Serge	2236	mdelay(15);
2004	serge	2237	WREG32(SRBM_SOFT_RESET, 0);
		2238	RREG32(SRBM_SOFT_RESET);
		2239	}
		2240
		2241	WREG32(RLC_CNTL, 0);
		2242	}
		2243
		2244	static void r600_rlc_start(struct radeon_device *rdev)
		2245	{
		2246	WREG32(RLC_CNTL, RLC_ENABLE);
		2247	}
		2248
		2249	static int r600_rlc_init(struct radeon_device *rdev)
		2250	{
		2251	u32 i;
		2252	const __be32 *fw_data;
		2253
		2254	if (!rdev->rlc_fw)
		2255	return -EINVAL;
		2256
		2257	r600_rlc_stop(rdev);
		2258
2997	Serge	2259	WREG32(RLC_HB_CNTL, 0);
		2260
		2261	if (rdev->family == CHIP_ARUBA) {
		2262	WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		2263	WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		2264	}
		2265	if (rdev->family <= CHIP_CAYMAN) {
2004	serge	2266	WREG32(RLC_HB_BASE, 0);
		2267	WREG32(RLC_HB_RPTR, 0);
		2268	WREG32(RLC_HB_WPTR, 0);
2997	Serge	2269	}
2004	serge	2270	if (rdev->family <= CHIP_CAICOS) {
		2271	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
		2272	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
		2273	}
		2274	WREG32(RLC_MC_CNTL, 0);
		2275	WREG32(RLC_UCODE_CNTL, 0);
		2276
		2277	fw_data = (const __be32 *)rdev->rlc_fw->data;
2997	Serge	2278	if (rdev->family >= CHIP_ARUBA) {
		2279	for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
		2280	WREG32(RLC_UCODE_ADDR, i);
		2281	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2282	}
		2283	} else if (rdev->family >= CHIP_CAYMAN) {
2004	serge	2284	for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
		2285	WREG32(RLC_UCODE_ADDR, i);
		2286	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2287	}
		2288	} else if (rdev->family >= CHIP_CEDAR) {
		2289	for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
		2290	WREG32(RLC_UCODE_ADDR, i);
		2291	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2292	}
		2293	} else if (rdev->family >= CHIP_RV770) {
		2294	for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
		2295	WREG32(RLC_UCODE_ADDR, i);
		2296	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2297	}
		2298	} else {
		2299	for (i = 0; i < RLC_UCODE_SIZE; i++) {
		2300	WREG32(RLC_UCODE_ADDR, i);
		2301	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2302	}
		2303	}
		2304	WREG32(RLC_UCODE_ADDR, 0);
		2305
		2306	r600_rlc_start(rdev);
		2307
		2308	return 0;
		2309	}
		2310
		2311	static void r600_enable_interrupts(struct radeon_device *rdev)
		2312	{
		2313	u32 ih_cntl = RREG32(IH_CNTL);
		2314	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		2315
		2316	ih_cntl \|= ENABLE_INTR;
		2317	ih_rb_cntl \|= IH_RB_ENABLE;
		2318	WREG32(IH_CNTL, ih_cntl);
		2319	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2320	rdev->ih.enabled = true;
		2321	}
		2322
		2323	void r600_disable_interrupts(struct radeon_device *rdev)
		2324	{
		2325	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		2326	u32 ih_cntl = RREG32(IH_CNTL);
		2327
		2328	ih_rb_cntl &= ~IH_RB_ENABLE;
		2329	ih_cntl &= ~ENABLE_INTR;
		2330	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2331	WREG32(IH_CNTL, ih_cntl);
		2332	/* set rptr, wptr to 0 */
		2333	WREG32(IH_RB_RPTR, 0);
		2334	WREG32(IH_RB_WPTR, 0);
		2335	rdev->ih.enabled = false;
		2336	rdev->ih.rptr = 0;
		2337	}
		2338
1963	serge	2339	static void r600_disable_interrupt_state(struct radeon_device *rdev)
		2340	{
		2341	u32 tmp;
1221	serge	2342
1963	serge	2343	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		2344	WREG32(GRBM_INT_CNTL, 0);
		2345	WREG32(DxMODE_INT_MASK, 0);
		2346	WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
		2347	WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
		2348	if (ASIC_IS_DCE3(rdev)) {
		2349	WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
		2350	WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
		2351	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2352	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2353	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2354	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2355	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2356	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2357	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2358	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2359	if (ASIC_IS_DCE32(rdev)) {
		2360	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2361	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2362	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2363	WREG32(DC_HPD6_INT_CONTROL, tmp);
2997	Serge	2364	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2365	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
		2366	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2367	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
		2368	} else {
		2369	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2370	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
		2371	tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2372	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
1963	serge	2373	}
		2374	} else {
		2375	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		2376	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2377	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2378	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		2379	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2380	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		2381	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2382	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2997	Serge	2383	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2384	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
		2385	tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2386	WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
1963	serge	2387	}
		2388	}
1221	serge	2389
2004	serge	2390	int r600_irq_init(struct radeon_device *rdev)
		2391	{
		2392	int ret = 0;
		2393	int rb_bufsz;
		2394	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
1221	serge	2395
2004	serge	2396	/* allocate ring */
		2397	ret = r600_ih_ring_alloc(rdev);
		2398	if (ret)
		2399	return ret;
1221	serge	2400
2004	serge	2401	/* disable irqs */
		2402	r600_disable_interrupts(rdev);
1221	serge	2403
2004	serge	2404	/* init rlc */
		2405	ret = r600_rlc_init(rdev);
		2406	if (ret) {
		2407	r600_ih_ring_fini(rdev);
		2408	return ret;
		2409	}
1221	serge	2410
2004	serge	2411	/* setup interrupt control */
		2412	/* set dummy read address to ring address */
		2413	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
		2414	interrupt_cntl = RREG32(INTERRUPT_CNTL);
		2415	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
		2416	* IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
		2417	*/
		2418	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
		2419	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
		2420	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
		2421	WREG32(INTERRUPT_CNTL, interrupt_cntl);
1221	serge	2422
2004	serge	2423	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
		2424	rb_bufsz = drm_order(rdev->ih.ring_size / 4);
1221	serge	2425
2004	serge	2426	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE \|
		2427	IH_WPTR_OVERFLOW_CLEAR \|
		2428	(rb_bufsz << 1));
1963	serge	2429
2004	serge	2430	if (rdev->wb.enabled)
		2431	ih_rb_cntl \|= IH_WPTR_WRITEBACK_ENABLE;
		2432
		2433	/* set the writeback address whether it's enabled or not */
		2434	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
		2435	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
		2436
		2437	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2438
		2439	/* set rptr, wptr to 0 */
		2440	WREG32(IH_RB_RPTR, 0);
		2441	WREG32(IH_RB_WPTR, 0);
		2442
		2443	/* Default settings for IH_CNTL (disabled at first) */
		2444	ih_cntl = MC_WRREQ_CREDIT(0x10) \| MC_WR_CLEAN_CNT(0x10);
		2445	/* RPTR_REARM only works if msi's are enabled */
		2446	if (rdev->msi_enabled)
		2447	ih_cntl \|= RPTR_REARM;
		2448	WREG32(IH_CNTL, ih_cntl);
		2449
		2450	/* force the active interrupt state to all disabled */
		2451	if (rdev->family >= CHIP_CEDAR)
		2452	evergreen_disable_interrupt_state(rdev);
		2453	else
		2454	r600_disable_interrupt_state(rdev);
		2455
2997	Serge	2456	/* at this point everything should be setup correctly to enable master */
		2457	pci_set_master(rdev->pdev);
		2458
2004	serge	2459	/* enable irqs */
		2460	r600_enable_interrupts(rdev);
		2461
		2462	return ret;
		2463	}
		2464	int r600_irq_set(struct radeon_device *rdev)
		2465	{
		2466	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
		2467	u32 mode_int = 0;
		2468	u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
		2469	u32 grbm_int_cntl = 0;
2997	Serge	2470	u32 hdmi0, hdmi1;
2004	serge	2471	u32 d1grph = 0, d2grph = 0;
		2472
		2473	if (!rdev->irq.installed) {
		2474	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		2475	return -EINVAL;
		2476	}
		2477	/* don't enable anything if the ih is disabled */
		2478	if (!rdev->ih.enabled) {
		2479	r600_disable_interrupts(rdev);
		2480	/* force the active interrupt state to all disabled */
		2481	r600_disable_interrupt_state(rdev);
		2482	return 0;
		2483	}
		2484
		2485	if (ASIC_IS_DCE3(rdev)) {
		2486	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2487	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2488	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2489	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2490	if (ASIC_IS_DCE32(rdev)) {
		2491	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2492	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2997	Serge	2493	hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2494	hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		2495	} else {
		2496	hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2497	hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	2498	}
		2499	} else {
		2500	hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2501	hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2502	hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2997	Serge	2503	hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		2504	hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	2505	}
		2506
2997	Serge	2507	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
2004	serge	2508	DRM_DEBUG("r600_irq_set: sw int\n");
		2509	cp_int_cntl \|= RB_INT_ENABLE;
		2510	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		2511	}
		2512	if (rdev->irq.crtc_vblank_int[0] \|\|
2997	Serge	2513	atomic_read(&rdev->irq.pflip[0])) {
2004	serge	2514	DRM_DEBUG("r600_irq_set: vblank 0\n");
		2515	mode_int \|= D1MODE_VBLANK_INT_MASK;
		2516	}
		2517	if (rdev->irq.crtc_vblank_int[1] \|\|
2997	Serge	2518	atomic_read(&rdev->irq.pflip[1])) {
2004	serge	2519	DRM_DEBUG("r600_irq_set: vblank 1\n");
		2520	mode_int \|= D2MODE_VBLANK_INT_MASK;
		2521	}
		2522	if (rdev->irq.hpd[0]) {
		2523	DRM_DEBUG("r600_irq_set: hpd 1\n");
		2524	hpd1 \|= DC_HPDx_INT_EN;
		2525	}
		2526	if (rdev->irq.hpd[1]) {
		2527	DRM_DEBUG("r600_irq_set: hpd 2\n");
		2528	hpd2 \|= DC_HPDx_INT_EN;
		2529	}
		2530	if (rdev->irq.hpd[2]) {
		2531	DRM_DEBUG("r600_irq_set: hpd 3\n");
		2532	hpd3 \|= DC_HPDx_INT_EN;
		2533	}
		2534	if (rdev->irq.hpd[3]) {
		2535	DRM_DEBUG("r600_irq_set: hpd 4\n");
		2536	hpd4 \|= DC_HPDx_INT_EN;
		2537	}
		2538	if (rdev->irq.hpd[4]) {
		2539	DRM_DEBUG("r600_irq_set: hpd 5\n");
		2540	hpd5 \|= DC_HPDx_INT_EN;
		2541	}
		2542	if (rdev->irq.hpd[5]) {
		2543	DRM_DEBUG("r600_irq_set: hpd 6\n");
		2544	hpd6 \|= DC_HPDx_INT_EN;
		2545	}
2997	Serge	2546	if (rdev->irq.afmt[0]) {
		2547	DRM_DEBUG("r600_irq_set: hdmi 0\n");
		2548	hdmi0 \|= HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	2549	}
2997	Serge	2550	if (rdev->irq.afmt[1]) {
		2551	DRM_DEBUG("r600_irq_set: hdmi 0\n");
		2552	hdmi1 \|= HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	2553	}
		2554
		2555	WREG32(CP_INT_CNTL, cp_int_cntl);
		2556	WREG32(DxMODE_INT_MASK, mode_int);
		2557	WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
		2558	WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
		2559	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		2560	if (ASIC_IS_DCE3(rdev)) {
		2561	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		2562	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		2563	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		2564	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		2565	if (ASIC_IS_DCE32(rdev)) {
		2566	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		2567	WREG32(DC_HPD6_INT_CONTROL, hpd6);
2997	Serge	2568	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
		2569	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
		2570	} else {
		2571	WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
		2572	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
2004	serge	2573	}
		2574	} else {
		2575	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
		2576	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
		2577	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
2997	Serge	2578	WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
		2579	WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
2004	serge	2580	}
		2581
		2582	return 0;
		2583	}
		2584
2997	Serge	2585	static void r600_irq_ack(struct radeon_device *rdev)
2004	serge	2586	{
		2587	u32 tmp;
		2588
		2589	if (ASIC_IS_DCE3(rdev)) {
		2590	rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
		2591	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
		2592	rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
2997	Serge	2593	if (ASIC_IS_DCE32(rdev)) {
		2594	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
		2595	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
		2596	} else {
		2597	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
		2598	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
		2599	}
2004	serge	2600	} else {
		2601	rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		2602	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		2603	rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
2997	Serge	2604	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
		2605	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
2004	serge	2606	}
		2607	rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
		2608	rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
		2609
		2610	if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		2611	WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		2612	if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		2613	WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		2614	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
		2615	WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		2616	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
		2617	WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		2618	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
		2619	WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		2620	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
		2621	WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		2622	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		2623	if (ASIC_IS_DCE3(rdev)) {
		2624	tmp = RREG32(DC_HPD1_INT_CONTROL);
		2625	tmp \|= DC_HPDx_INT_ACK;
		2626	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2627	} else {
		2628	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		2629	tmp \|= DC_HPDx_INT_ACK;
		2630	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		2631	}
		2632	}
		2633	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		2634	if (ASIC_IS_DCE3(rdev)) {
		2635	tmp = RREG32(DC_HPD2_INT_CONTROL);
		2636	tmp \|= DC_HPDx_INT_ACK;
		2637	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2638	} else {
		2639	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		2640	tmp \|= DC_HPDx_INT_ACK;
		2641	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		2642	}
		2643	}
		2644	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		2645	if (ASIC_IS_DCE3(rdev)) {
		2646	tmp = RREG32(DC_HPD3_INT_CONTROL);
		2647	tmp \|= DC_HPDx_INT_ACK;
		2648	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2649	} else {
		2650	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		2651	tmp \|= DC_HPDx_INT_ACK;
		2652	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		2653	}
		2654	}
		2655	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		2656	tmp = RREG32(DC_HPD4_INT_CONTROL);
		2657	tmp \|= DC_HPDx_INT_ACK;
		2658	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2659	}
		2660	if (ASIC_IS_DCE32(rdev)) {
		2661	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		2662	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2663	tmp \|= DC_HPDx_INT_ACK;
		2664	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2665	}
		2666	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		2667	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2668	tmp \|= DC_HPDx_INT_ACK;
		2669	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2670	}
2997	Serge	2671	if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
		2672	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
		2673	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2674	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
		2675	}
		2676	if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
		2677	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
		2678	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		2679	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
2004	serge	2680	}
2997	Serge	2681	} else {
		2682	if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
		2683	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
		2684	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		2685	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
2004	serge	2686	}
2997	Serge	2687	if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
2004	serge	2688	if (ASIC_IS_DCE3(rdev)) {
2997	Serge	2689	tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
		2690	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		2691	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
		2692	} else {
		2693	tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
		2694	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		2695	WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
2004	serge	2696	}
		2697	}
		2698	}
		2699	}
		2700
2997	Serge	2701	static u32 r600_get_ih_wptr(struct radeon_device *rdev)
2004	serge	2702	{
		2703	u32 wptr, tmp;
		2704
		2705	if (rdev->wb.enabled)
		2706	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		2707	else
		2708	wptr = RREG32(IH_RB_WPTR);
		2709
		2710	if (wptr & RB_OVERFLOW) {
		2711	/* When a ring buffer overflow happen start parsing interrupt
		2712	* from the last not overwritten vector (wptr + 16). Hopefully
		2713	* this should allow us to catchup.
		2714	*/
		2715	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
		2716	wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
		2717	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		2718	tmp = RREG32(IH_RB_CNTL);
		2719	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		2720	WREG32(IH_RB_CNTL, tmp);
		2721	}
		2722	return (wptr & rdev->ih.ptr_mask);
		2723	}
		2724
		2725	/* r600 IV Ring
		2726	* Each IV ring entry is 128 bits:
		2727	* [7:0] - interrupt source id
		2728	* [31:8] - reserved
		2729	* [59:32] - interrupt source data
		2730	* [127:60] - reserved
		2731	*
		2732	* The basic interrupt vector entries
		2733	* are decoded as follows:
		2734	* src_id src_data description
		2735	* 1 0 D1 Vblank
		2736	* 1 1 D1 Vline
		2737	* 5 0 D2 Vblank
		2738	* 5 1 D2 Vline
		2739	* 19 0 FP Hot plug detection A
		2740	* 19 1 FP Hot plug detection B
		2741	* 19 2 DAC A auto-detection
		2742	* 19 3 DAC B auto-detection
		2743	* 21 4 HDMI block A
		2744	* 21 5 HDMI block B
		2745	* 176 - CP_INT RB
		2746	* 177 - CP_INT IB1
		2747	* 178 - CP_INT IB2
		2748	* 181 - EOP Interrupt
		2749	* 233 - GUI Idle
		2750	*
		2751	* Note, these are based on r600 and may need to be
		2752	* adjusted or added to on newer asics
		2753	*/
		2754
2160	serge	2755	#define DRM_DEBUG(...)
		2756
2004	serge	2757	int r600_irq_process(struct radeon_device *rdev)
		2758	{
		2759	u32 wptr;
		2760	u32 rptr;
		2761	u32 src_id, src_data;
		2762	u32 ring_index;
		2763	bool queue_hotplug = false;
2997	Serge	2764	bool queue_hdmi = false;
2004	serge	2765
		2766	if (!rdev->ih.enabled \|\| rdev->shutdown)
		2767	return IRQ_NONE;
		2768
2160	serge	2769	/* No MSIs, need a dummy read to flush PCI DMAs */
		2770	if (!rdev->msi_enabled)
		2771	RREG32(IH_RB_WPTR);
		2772
2004	serge	2773	wptr = r600_get_ih_wptr(rdev);
		2774
2997	Serge	2775	restart_ih:
		2776	/* is somebody else already processing irqs? */
		2777	if (atomic_xchg(&rdev->ih.lock, 1))
2004	serge	2778	return IRQ_NONE;
		2779
2997	Serge	2780	rptr = rdev->ih.rptr;
		2781	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		2782
2160	serge	2783	/* Order reading of wptr vs. reading of IH ring data */
		2784	rmb();
		2785
2004	serge	2786	/* display interrupts */
		2787	r600_irq_ack(rdev);
		2788
		2789	while (rptr != wptr) {
		2790	/* wptr/rptr are in bytes! */
		2791	ring_index = rptr / 4;
		2792	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		2793	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		2794
		2795	switch (src_id) {
		2796	case 1: /* D1 vblank/vline */
		2797	switch (src_data) {
		2798	case 0: /* D1 vblank */
		2799	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
		2800	if (rdev->irq.crtc_vblank_int[0]) {
		2801	// drm_handle_vblank(rdev->ddev, 0);
		2802	rdev->pm.vblank_sync = true;
		2803	// wake_up(&rdev->irq.vblank_queue);
		2804	}
		2805	// if (rdev->irq.pflip[0])
		2806	// radeon_crtc_handle_flip(rdev, 0);
		2807	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		2808	DRM_DEBUG("IH: D1 vblank\n");
		2809	}
		2810	break;
		2811	case 1: /* D1 vline */
		2812	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
		2813	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		2814	DRM_DEBUG("IH: D1 vline\n");
		2815	}
		2816	break;
		2817	default:
		2818	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2819	break;
		2820	}
		2821	break;
		2822	case 5: /* D2 vblank/vline */
		2823	switch (src_data) {
		2824	case 0: /* D2 vblank */
		2825	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
		2826	if (rdev->irq.crtc_vblank_int[1]) {
		2827	// drm_handle_vblank(rdev->ddev, 1);
		2828	rdev->pm.vblank_sync = true;
		2829	// wake_up(&rdev->irq.vblank_queue);
		2830	}
		2831	// if (rdev->irq.pflip[1])
		2832	// radeon_crtc_handle_flip(rdev, 1);
		2833	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
		2834	DRM_DEBUG("IH: D2 vblank\n");
		2835	}
		2836	break;
		2837	case 1: /* D1 vline */
		2838	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
		2839	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
		2840	DRM_DEBUG("IH: D2 vline\n");
		2841	}
		2842	break;
		2843	default:
		2844	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2845	break;
		2846	}
		2847	break;
		2848	case 19: /* HPD/DAC hotplug */
		2849	switch (src_data) {
		2850	case 0:
		2851	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		2852	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
		2853	queue_hotplug = true;
		2854	DRM_DEBUG("IH: HPD1\n");
		2855	}
		2856	break;
		2857	case 1:
		2858	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		2859	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
		2860	queue_hotplug = true;
		2861	DRM_DEBUG("IH: HPD2\n");
		2862	}
		2863	break;
		2864	case 4:
		2865	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		2866	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
		2867	queue_hotplug = true;
		2868	DRM_DEBUG("IH: HPD3\n");
		2869	}
		2870	break;
		2871	case 5:
		2872	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		2873	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
		2874	queue_hotplug = true;
		2875	DRM_DEBUG("IH: HPD4\n");
		2876	}
		2877	break;
		2878	case 10:
		2879	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		2880	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
		2881	queue_hotplug = true;
		2882	DRM_DEBUG("IH: HPD5\n");
		2883	}
		2884	break;
		2885	case 12:
		2886	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		2887	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
		2888	queue_hotplug = true;
		2889	DRM_DEBUG("IH: HPD6\n");
		2890	}
		2891	break;
		2892	default:
		2893	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2894	break;
		2895	}
		2896	break;
2997	Serge	2897	case 21: /* hdmi */
		2898	switch (src_data) {
		2899	case 4:
		2900	if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
		2901	rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
		2902	queue_hdmi = true;
		2903	DRM_DEBUG("IH: HDMI0\n");
		2904	}
		2905	break;
		2906	case 5:
		2907	if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
		2908	rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
		2909	queue_hdmi = true;
		2910	DRM_DEBUG("IH: HDMI1\n");
		2911	}
		2912	break;
		2913	default:
		2914	DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
		2915	break;
		2916	}
2004	serge	2917	break;
		2918	case 176: /* CP_INT in ring buffer */
		2919	case 177: /* CP_INT in IB1 */
		2920	case 178: /* CP_INT in IB2 */
		2921	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2997	Serge	2922	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2004	serge	2923	break;
		2924	case 181: /* CP EOP event */
		2925	DRM_DEBUG("IH: CP EOP\n");
2997	Serge	2926	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2004	serge	2927	break;
		2928	case 233: /* GUI IDLE */
		2929	DRM_DEBUG("IH: GUI idle\n");
		2930	break;
		2931	default:
		2932	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2933	break;
		2934	}
		2935
		2936	/* wptr/rptr are in bytes! */
		2937	rptr += 16;
		2938	rptr &= rdev->ih.ptr_mask;
		2939	}
2997	Serge	2940	rdev->ih.rptr = rptr;
		2941	WREG32(IH_RB_RPTR, rdev->ih.rptr);
		2942	atomic_set(&rdev->ih.lock, 0);
		2943
2004	serge	2944	/* make sure wptr hasn't changed while processing */
		2945	wptr = r600_get_ih_wptr(rdev);
2997	Serge	2946	if (wptr != rptr)
2004	serge	2947	goto restart_ih;
2997	Serge	2948
2004	serge	2949	return IRQ_HANDLED;
		2950	}
		2951
1221	serge	2952	/*
		2953	* Debugfs info
		2954	*/
		2955	#if defined(CONFIG_DEBUG_FS)
		2956
		2957	static int r600_debugfs_mc_info(struct seq_file m, void data)
		2958	{
		2959	struct drm_info_node node = (struct drm_info_node ) m->private;
		2960	struct drm_device *dev = node->minor->dev;
		2961	struct radeon_device *rdev = dev->dev_private;
		2962
		2963	DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
		2964	DREG32_SYS(m, rdev, VM_L2_STATUS);
		2965	return 0;
		2966	}
		2967
		2968	static struct drm_info_list r600_mc_info_list[] = {
		2969	{"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
		2970	};
		2971	#endif
		2972
		2973	int r600_debugfs_mc_info_init(struct radeon_device *rdev)
		2974	{
		2975	#if defined(CONFIG_DEBUG_FS)
		2976	return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
		2977	#else
		2978	return 0;
		2979	#endif
		2980	}
1404	serge	2981
		2982	/**
		2983	* r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
		2984	* rdev: radeon device structure
		2985	* bo: buffer object struct which userspace is waiting for idle
		2986	*
		2987	* Some R6XX/R7XX doesn't seems to take into account HDP flush performed
		2988	* through ring buffer, this leads to corruption in rendering, see
		2989	* http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
		2990	* directly perform HDP flush by writing register through MMIO.
		2991	*/
		2992	void r600_ioctl_wait_idle(struct radeon_device rdev, struct radeon_bo bo)
		2993	{
1963	serge	2994	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		2995	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
		2996	* This seems to cause problems on some AGP cards. Just use the old
		2997	* method for them.
		2998	*/
		2999	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		3000	rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
		3001	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		3002	u32 tmp;
		3003
		3004	WREG32(HDP_DEBUG1, 0);
		3005	tmp = readl((void __iomem *)ptr);
		3006	} else
1404	serge	3007	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		3008	}
1963	serge	3009
		3010	void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
		3011	{
		3012	u32 link_width_cntl, mask, target_reg;
		3013
		3014	if (rdev->flags & RADEON_IS_IGP)
		3015	return;
		3016
		3017	if (!(rdev->flags & RADEON_IS_PCIE))
		3018	return;
		3019
		3020	/* x2 cards have a special sequence */
		3021	if (ASIC_IS_X2(rdev))
		3022	return;
		3023
		3024	/* FIXME wait for idle */
		3025
		3026	switch (lanes) {
		3027	case 0:
		3028	mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
		3029	break;
		3030	case 1:
		3031	mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
		3032	break;
		3033	case 2:
		3034	mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
		3035	break;
		3036	case 4:
		3037	mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
		3038	break;
		3039	case 8:
		3040	mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
		3041	break;
		3042	case 12:
		3043	mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
		3044	break;
		3045	case 16:
		3046	default:
		3047	mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
		3048	break;
		3049	}
		3050
		3051	link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		3052
		3053	if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
		3054	(mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
		3055	return;
		3056
		3057	if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
		3058	return;
		3059
		3060	link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK \|
		3061	RADEON_PCIE_LC_RECONFIG_NOW \|
		3062	R600_PCIE_LC_RENEGOTIATE_EN \|
		3063	R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
		3064	link_width_cntl \|= mask;
		3065
		3066	WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3067
2005	serge	3068	/* some northbridges can renegotiate the link rather than requiring
		3069	* a complete re-config.
		3070	* e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)
1963	serge	3071	*/
		3072	if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
		3073	link_width_cntl \|= R600_PCIE_LC_RENEGOTIATE_EN \| R600_PCIE_LC_UPCONFIGURE_SUPPORT;
		3074	else
		3075	link_width_cntl \|= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
		3076
		3077	WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl \|
		3078	RADEON_PCIE_LC_RECONFIG_NOW));
		3079
		3080	if (rdev->family >= CHIP_RV770)
		3081	target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
		3082	else
		3083	target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
		3084
		3085	/* wait for lane set to complete */
		3086	link_width_cntl = RREG32(target_reg);
		3087	while (link_width_cntl == 0xffffffff)
		3088	link_width_cntl = RREG32(target_reg);
		3089
		3090	}
		3091
		3092	int r600_get_pcie_lanes(struct radeon_device *rdev)
		3093	{
		3094	u32 link_width_cntl;
		3095
		3096	if (rdev->flags & RADEON_IS_IGP)
		3097	return 0;
		3098
		3099	if (!(rdev->flags & RADEON_IS_PCIE))
		3100	return 0;
		3101
		3102	/* x2 cards have a special sequence */
		3103	if (ASIC_IS_X2(rdev))
		3104	return 0;
		3105
		3106	/* FIXME wait for idle */
		3107
		3108	link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		3109
		3110	switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
		3111	case RADEON_PCIE_LC_LINK_WIDTH_X0:
		3112	return 0;
		3113	case RADEON_PCIE_LC_LINK_WIDTH_X1:
		3114	return 1;
		3115	case RADEON_PCIE_LC_LINK_WIDTH_X2:
		3116	return 2;
		3117	case RADEON_PCIE_LC_LINK_WIDTH_X4:
		3118	return 4;
		3119	case RADEON_PCIE_LC_LINK_WIDTH_X8:
		3120	return 8;
		3121	case RADEON_PCIE_LC_LINK_WIDTH_X16:
		3122	default:
		3123	return 16;
		3124	}
		3125	}
		3126
		3127	static void r600_pcie_gen2_enable(struct radeon_device *rdev)
		3128	{
		3129	u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
		3130	u16 link_cntl2;
2997	Serge	3131	u32 mask;
		3132	int ret;
1963	serge	3133
		3134	if (radeon_pcie_gen2 == 0)
		3135	return;
		3136
		3137	if (rdev->flags & RADEON_IS_IGP)
		3138	return;
		3139
		3140	if (!(rdev->flags & RADEON_IS_PCIE))
		3141	return;
		3142
		3143	/* x2 cards have a special sequence */
		3144	if (ASIC_IS_X2(rdev))
		3145	return;
		3146
		3147	/* only RV6xx+ chips are supported */
		3148	if (rdev->family <= CHIP_R600)
		3149	return;
		3150
2997	Serge	3151	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
		3152	if (ret != 0)
		3153	return;
		3154
		3155	if (!(mask & DRM_PCIE_SPEED_50))
		3156	return;
		3157
		3158	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3159	if (speed_cntl & LC_CURRENT_DATA_RATE) {
		3160	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
		3161	return;
		3162	}
		3163
		3164	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
		3165
1963	serge	3166	/* 55 nm r6xx asics */
		3167	if ((rdev->family == CHIP_RV670) \|\|
		3168	(rdev->family == CHIP_RV620) \|\|
		3169	(rdev->family == CHIP_RV635)) {
		3170	/* advertise upconfig capability */
		3171	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3172	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3173	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3174	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3175	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
		3176	lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
		3177	link_width_cntl &= ~(LC_LINK_WIDTH_MASK \|
		3178	LC_RECONFIG_ARC_MISSING_ESCAPE);
		3179	link_width_cntl \|= lanes \| LC_RECONFIG_NOW \| LC_RENEGOTIATE_EN;
		3180	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3181	} else {
		3182	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3183	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3184	}
		3185	}
		3186
		3187	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3188	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
		3189	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		3190
		3191	/* 55 nm r6xx asics */
		3192	if ((rdev->family == CHIP_RV670) \|\|
		3193	(rdev->family == CHIP_RV620) \|\|
		3194	(rdev->family == CHIP_RV635)) {
		3195	WREG32(MM_CFGREGS_CNTL, 0x8);
		3196	link_cntl2 = RREG32(0x4088);
		3197	WREG32(MM_CFGREGS_CNTL, 0);
		3198	/* not supported yet */
		3199	if (link_cntl2 & SELECTABLE_DEEMPHASIS)
		3200	return;
		3201	}
		3202
		3203	speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
		3204	speed_cntl \|= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
		3205	speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
		3206	speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
		3207	speed_cntl \|= LC_FORCE_EN_HW_SPEED_CHANGE;
		3208	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3209
		3210	tmp = RREG32(0x541c);
		3211	WREG32(0x541c, tmp \| 0x8);
		3212	WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
		3213	link_cntl2 = RREG16(0x4088);
		3214	link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
		3215	link_cntl2 \|= 0x2;
		3216	WREG16(0x4088, link_cntl2);
		3217	WREG32(MM_CFGREGS_CNTL, 0);
		3218
		3219	if ((rdev->family == CHIP_RV670) \|\|
		3220	(rdev->family == CHIP_RV620) \|\|
		3221	(rdev->family == CHIP_RV635)) {
		3222	training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
		3223	training_cntl &= ~LC_POINT_7_PLUS_EN;
		3224	WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
		3225	} else {
		3226	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3227	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		3228	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3229	}
		3230
		3231	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3232	speed_cntl \|= LC_GEN2_EN_STRAP;
		3233	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3234
		3235	} else {
		3236	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3237	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		3238	if (1)
		3239	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3240	else
		3241	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3242	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3243	}
		3244	}
2997	Serge	3245
		3246	/**
		3247	* r600_get_gpu_clock - return GPU clock counter snapshot
		3248	*
		3249	* @rdev: radeon_device pointer
		3250	*
		3251	* Fetches a GPU clock counter snapshot (R6xx-cayman).
		3252	* Returns the 64 bit clock counter snapshot.
		3253	*/
		3254	uint64_t r600_get_gpu_clock(struct radeon_device *rdev)
		3255	{
		3256	uint64_t clock;
		3257
		3258	mutex_lock(&rdev->gpu_clock_mutex);
		3259	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
		3260	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) \|
		3261	((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
		3262	mutex_unlock(&rdev->gpu_clock_mutex);
		3263	return clock;
		3264	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/r600.c – Rev 2997