WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/r600.c

Rev	Author	Line No.	Line
1128	serge	1	/*
		2	* Copyright 2008 Advanced Micro Devices, Inc.
		3	* Copyright 2008 Red Hat Inc.
		4	* Copyright 2009 Jerome Glisse.
		5	*
		6	* Permission is hereby granted, free of charge, to any person obtaining a
		7	* copy of this software and associated documentation files (the "Software"),
		8	* to deal in the Software without restriction, including without limitation
		9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		10	* and/or sell copies of the Software, and to permit persons to whom the
		11	* Software is furnished to do so, subject to the following conditions:
		12	*
		13	* The above copyright notice and this permission notice shall be included in
		14	* all copies or substantial portions of the Software.
		15	*
		16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		22	* OTHER DEALINGS IN THE SOFTWARE.
		23	*
		24	* Authors: Dave Airlie
		25	* Alex Deucher
		26	* Jerome Glisse
		27	*/
1963	serge	28	#include
1233	serge	29	#include
1221	serge	30	#include
1128	serge	31	#include "drmP.h"
1221	serge	32	#include "radeon_drm.h"
1128	serge	33	#include "radeon.h"
1963	serge	34	#include "radeon_asic.h"
1221	serge	35	#include "radeon_mode.h"
		36	#include "r600d.h"
		37	#include "atom.h"
		38	#include "avivod.h"
1128	serge	39
1221	serge	40	#define PFP_UCODE_SIZE 576
		41	#define PM4_UCODE_SIZE 1792
1321	serge	42	#define RLC_UCODE_SIZE 768
1221	serge	43	#define R700_PFP_UCODE_SIZE 848
		44	#define R700_PM4_UCODE_SIZE 1360
1321	serge	45	#define R700_RLC_UCODE_SIZE 1024
1963	serge	46	#define EVERGREEN_PFP_UCODE_SIZE 1120
		47	#define EVERGREEN_PM4_UCODE_SIZE 1376
		48	#define EVERGREEN_RLC_UCODE_SIZE 768
		49	#define CAYMAN_RLC_UCODE_SIZE 1024
1128	serge	50
1221	serge	51	/* Firmware Names */
		52	MODULE_FIRMWARE("radeon/R600_pfp.bin");
		53	MODULE_FIRMWARE("radeon/R600_me.bin");
		54	MODULE_FIRMWARE("radeon/RV610_pfp.bin");
		55	MODULE_FIRMWARE("radeon/RV610_me.bin");
		56	MODULE_FIRMWARE("radeon/RV630_pfp.bin");
		57	MODULE_FIRMWARE("radeon/RV630_me.bin");
		58	MODULE_FIRMWARE("radeon/RV620_pfp.bin");
		59	MODULE_FIRMWARE("radeon/RV620_me.bin");
		60	MODULE_FIRMWARE("radeon/RV635_pfp.bin");
		61	MODULE_FIRMWARE("radeon/RV635_me.bin");
		62	MODULE_FIRMWARE("radeon/RV670_pfp.bin");
		63	MODULE_FIRMWARE("radeon/RV670_me.bin");
		64	MODULE_FIRMWARE("radeon/RS780_pfp.bin");
		65	MODULE_FIRMWARE("radeon/RS780_me.bin");
		66	MODULE_FIRMWARE("radeon/RV770_pfp.bin");
		67	MODULE_FIRMWARE("radeon/RV770_me.bin");
		68	MODULE_FIRMWARE("radeon/RV730_pfp.bin");
		69	MODULE_FIRMWARE("radeon/RV730_me.bin");
		70	MODULE_FIRMWARE("radeon/RV710_pfp.bin");
		71	MODULE_FIRMWARE("radeon/RV710_me.bin");
1321	serge	72	MODULE_FIRMWARE("radeon/R600_rlc.bin");
		73	MODULE_FIRMWARE("radeon/R700_rlc.bin");
1963	serge	74	MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
		75	MODULE_FIRMWARE("radeon/CEDAR_me.bin");
		76	MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
		77	MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
		78	MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
		79	MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
		80	MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
		81	MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
		82	MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
		83	MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
		84	MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
		85	MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
		86	MODULE_FIRMWARE("radeon/PALM_pfp.bin");
		87	MODULE_FIRMWARE("radeon/PALM_me.bin");
		88	MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
		89	MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
		90	MODULE_FIRMWARE("radeon/SUMO_me.bin");
		91	MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
		92	MODULE_FIRMWARE("radeon/SUMO2_me.bin");
1221	serge	93
		94	int r600_debugfs_mc_info_init(struct radeon_device *rdev);
		95
		96	/* r600,rv610,rv630,rv620,rv635,rv670 */
1128	serge	97	int r600_mc_wait_for_idle(struct radeon_device *rdev);
		98	void r600_gpu_init(struct radeon_device *rdev);
1221	serge	99	void r600_fini(struct radeon_device *rdev);
1963	serge	100	void r600_irq_disable(struct radeon_device *rdev);
		101	static void r600_pcie_gen2_enable(struct radeon_device *rdev);
1128	serge	102
1963	serge	103	/* get temperature in millidegrees */
		104	int rv6xx_get_temp(struct radeon_device *rdev)
		105	{
		106	u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
		107	ASIC_T_SHIFT;
		108	int actual_temp = temp & 0xff;
		109
		110	if (temp & 0x100)
		111	actual_temp -= 256;
		112
		113	return actual_temp * 1000;
		114	}
		115
		116
		117
		118
		119
		120
		121	bool r600_gui_idle(struct radeon_device *rdev)
		122	{
		123	if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
		124	return false;
		125	else
		126	return true;
		127	}
		128
1321	serge	129	/* hpd for digital panel detect/disconnect */
		130	bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		131	{
		132	bool connected = false;
		133
		134	if (ASIC_IS_DCE3(rdev)) {
		135	switch (hpd) {
		136	case RADEON_HPD_1:
		137	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		138	connected = true;
		139	break;
		140	case RADEON_HPD_2:
		141	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		142	connected = true;
		143	break;
		144	case RADEON_HPD_3:
		145	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		146	connected = true;
		147	break;
		148	case RADEON_HPD_4:
		149	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		150	connected = true;
		151	break;
		152	/* DCE 3.2 */
		153	case RADEON_HPD_5:
		154	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		155	connected = true;
		156	break;
		157	case RADEON_HPD_6:
		158	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		159	connected = true;
		160	break;
		161	default:
		162	break;
		163	}
		164	} else {
		165	switch (hpd) {
		166	case RADEON_HPD_1:
		167	if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		168	connected = true;
		169	break;
		170	case RADEON_HPD_2:
		171	if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		172	connected = true;
		173	break;
		174	case RADEON_HPD_3:
		175	if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		176	connected = true;
		177	break;
		178	default:
		179	break;
		180	}
		181	}
		182	return connected;
		183	}
		184
		185	void r600_hpd_set_polarity(struct radeon_device *rdev,
		186	enum radeon_hpd_id hpd)
		187	{
		188	u32 tmp;
		189	bool connected = r600_hpd_sense(rdev, hpd);
		190
		191	if (ASIC_IS_DCE3(rdev)) {
		192	switch (hpd) {
		193	case RADEON_HPD_1:
		194	tmp = RREG32(DC_HPD1_INT_CONTROL);
		195	if (connected)
		196	tmp &= ~DC_HPDx_INT_POLARITY;
		197	else
		198	tmp \|= DC_HPDx_INT_POLARITY;
		199	WREG32(DC_HPD1_INT_CONTROL, tmp);
		200	break;
		201	case RADEON_HPD_2:
		202	tmp = RREG32(DC_HPD2_INT_CONTROL);
		203	if (connected)
		204	tmp &= ~DC_HPDx_INT_POLARITY;
		205	else
		206	tmp \|= DC_HPDx_INT_POLARITY;
		207	WREG32(DC_HPD2_INT_CONTROL, tmp);
		208	break;
		209	case RADEON_HPD_3:
		210	tmp = RREG32(DC_HPD3_INT_CONTROL);
		211	if (connected)
		212	tmp &= ~DC_HPDx_INT_POLARITY;
		213	else
		214	tmp \|= DC_HPDx_INT_POLARITY;
		215	WREG32(DC_HPD3_INT_CONTROL, tmp);
		216	break;
		217	case RADEON_HPD_4:
		218	tmp = RREG32(DC_HPD4_INT_CONTROL);
		219	if (connected)
		220	tmp &= ~DC_HPDx_INT_POLARITY;
		221	else
		222	tmp \|= DC_HPDx_INT_POLARITY;
		223	WREG32(DC_HPD4_INT_CONTROL, tmp);
		224	break;
		225	case RADEON_HPD_5:
		226	tmp = RREG32(DC_HPD5_INT_CONTROL);
		227	if (connected)
		228	tmp &= ~DC_HPDx_INT_POLARITY;
		229	else
		230	tmp \|= DC_HPDx_INT_POLARITY;
		231	WREG32(DC_HPD5_INT_CONTROL, tmp);
		232	break;
		233	/* DCE 3.2 */
		234	case RADEON_HPD_6:
		235	tmp = RREG32(DC_HPD6_INT_CONTROL);
		236	if (connected)
		237	tmp &= ~DC_HPDx_INT_POLARITY;
		238	else
		239	tmp \|= DC_HPDx_INT_POLARITY;
		240	WREG32(DC_HPD6_INT_CONTROL, tmp);
		241	break;
		242	default:
		243	break;
		244	}
		245	} else {
		246	switch (hpd) {
		247	case RADEON_HPD_1:
		248	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		249	if (connected)
		250	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		251	else
		252	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		253	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		254	break;
		255	case RADEON_HPD_2:
		256	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		257	if (connected)
		258	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		259	else
		260	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		261	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		262	break;
		263	case RADEON_HPD_3:
		264	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		265	if (connected)
		266	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		267	else
		268	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		269	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		270	break;
		271	default:
		272	break;
		273	}
		274	}
		275	}
		276
		277	void r600_hpd_init(struct radeon_device *rdev)
		278	{
		279	struct drm_device *dev = rdev->ddev;
		280	struct drm_connector *connector;
		281
		282	if (ASIC_IS_DCE3(rdev)) {
		283	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \| DC_HPDx_RX_INT_TIMER(0xfa);
		284	if (ASIC_IS_DCE32(rdev))
		285	tmp \|= DC_HPDx_EN;
		286
		287	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		288	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		289	switch (radeon_connector->hpd.hpd) {
		290	case RADEON_HPD_1:
		291	WREG32(DC_HPD1_CONTROL, tmp);
2004	serge	292	rdev->irq.hpd[0] = true;
1321	serge	293	break;
		294	case RADEON_HPD_2:
		295	WREG32(DC_HPD2_CONTROL, tmp);
2004	serge	296	rdev->irq.hpd[1] = true;
1321	serge	297	break;
		298	case RADEON_HPD_3:
		299	WREG32(DC_HPD3_CONTROL, tmp);
2004	serge	300	rdev->irq.hpd[2] = true;
1321	serge	301	break;
		302	case RADEON_HPD_4:
		303	WREG32(DC_HPD4_CONTROL, tmp);
2004	serge	304	rdev->irq.hpd[3] = true;
1321	serge	305	break;
		306	/* DCE 3.2 */
		307	case RADEON_HPD_5:
		308	WREG32(DC_HPD5_CONTROL, tmp);
2004	serge	309	rdev->irq.hpd[4] = true;
1321	serge	310	break;
		311	case RADEON_HPD_6:
		312	WREG32(DC_HPD6_CONTROL, tmp);
2004	serge	313	rdev->irq.hpd[5] = true;
1321	serge	314	break;
		315	default:
		316	break;
		317	}
		318	}
		319	} else {
		320	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		321	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		322	switch (radeon_connector->hpd.hpd) {
		323	case RADEON_HPD_1:
		324	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
2004	serge	325	rdev->irq.hpd[0] = true;
1321	serge	326	break;
		327	case RADEON_HPD_2:
		328	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
2004	serge	329	rdev->irq.hpd[1] = true;
1321	serge	330	break;
		331	case RADEON_HPD_3:
		332	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
2004	serge	333	rdev->irq.hpd[2] = true;
1321	serge	334	break;
		335	default:
		336	break;
		337	}
		338	}
		339	}
2004	serge	340	if (rdev->irq.installed)
		341	r600_irq_set(rdev);
1321	serge	342	}
		343
		344	void r600_hpd_fini(struct radeon_device *rdev)
		345	{
		346	struct drm_device *dev = rdev->ddev;
		347	struct drm_connector *connector;
		348
		349	if (ASIC_IS_DCE3(rdev)) {
		350	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		351	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		352	switch (radeon_connector->hpd.hpd) {
		353	case RADEON_HPD_1:
		354	WREG32(DC_HPD1_CONTROL, 0);
2004	serge	355	rdev->irq.hpd[0] = false;
1321	serge	356	break;
		357	case RADEON_HPD_2:
		358	WREG32(DC_HPD2_CONTROL, 0);
2004	serge	359	rdev->irq.hpd[1] = false;
1321	serge	360	break;
		361	case RADEON_HPD_3:
		362	WREG32(DC_HPD3_CONTROL, 0);
2004	serge	363	rdev->irq.hpd[2] = false;
1321	serge	364	break;
		365	case RADEON_HPD_4:
		366	WREG32(DC_HPD4_CONTROL, 0);
2004	serge	367	rdev->irq.hpd[3] = false;
1321	serge	368	break;
		369	/* DCE 3.2 */
		370	case RADEON_HPD_5:
		371	WREG32(DC_HPD5_CONTROL, 0);
2004	serge	372	rdev->irq.hpd[4] = false;
1321	serge	373	break;
		374	case RADEON_HPD_6:
		375	WREG32(DC_HPD6_CONTROL, 0);
2004	serge	376	rdev->irq.hpd[5] = false;
1321	serge	377	break;
		378	default:
		379	break;
		380	}
		381	}
		382	} else {
		383	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		384	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		385	switch (radeon_connector->hpd.hpd) {
		386	case RADEON_HPD_1:
		387	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
2004	serge	388	rdev->irq.hpd[0] = false;
1321	serge	389	break;
		390	case RADEON_HPD_2:
		391	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
2004	serge	392	rdev->irq.hpd[1] = false;
1321	serge	393	break;
		394	case RADEON_HPD_3:
		395	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
2004	serge	396	rdev->irq.hpd[2] = false;
1321	serge	397	break;
		398	default:
		399	break;
		400	}
		401	}
		402	}
		403	}
		404
1128	serge	405	/*
1221	serge	406	* R600 PCIE GART
1128	serge	407	*/
1221	serge	408	void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
		409	{
		410	unsigned i;
		411	u32 tmp;
1128	serge	412
1430	serge	413	/* flush hdp cache so updates hit vram */
1963	serge	414	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		415	!(rdev->flags & RADEON_IS_AGP)) {
		416	void __iomem ptr = (void )rdev->gart.table.vram.ptr;
		417	u32 tmp;
		418
		419	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		420	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
		421	* This seems to cause problems on some AGP cards. Just use the old
		422	* method for them.
		423	*/
		424	WREG32(HDP_DEBUG1, 0);
		425	tmp = readl((void __iomem *)ptr);
		426	} else
1430	serge	427	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		428
1221	serge	429	WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
		430	WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
		431	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		432	for (i = 0; i < rdev->usec_timeout; i++) {
		433	/* read MC_STATUS */
		434	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		435	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		436	if (tmp == 2) {
		437	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		438	return;
		439	}
		440	if (tmp) {
		441	return;
		442	}
		443	udelay(1);
1128	serge	444	}
1221	serge	445	}
1128	serge	446
1221	serge	447	int r600_pcie_gart_init(struct radeon_device *rdev)
		448	{
		449	int r;
		450
		451	if (rdev->gart.table.vram.robj) {
1963	serge	452	WARN(1, "R600 PCIE GART already initialized\n");
1221	serge	453	return 0;
		454	}
		455	/* Initialize common gart structure */
		456	r = radeon_gart_init(rdev);
		457	if (r)
		458	return r;
		459	rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
		460	return radeon_gart_table_vram_alloc(rdev);
		461	}
		462
		463	int r600_pcie_gart_enable(struct radeon_device *rdev)
		464	{
		465	u32 tmp;
		466	int r, i;
		467
		468	if (rdev->gart.table.vram.robj == NULL) {
		469	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		470	return -EINVAL;
		471	}
		472	r = radeon_gart_table_vram_pin(rdev);
		473	if (r)
		474	return r;
1430	serge	475	radeon_gart_restore(rdev);
1221	serge	476
		477	/* Setup L2 cache */
		478	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		479	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		480	EFFECTIVE_L2_QUEUE_SIZE(7));
		481	WREG32(VM_L2_CNTL2, 0);
		482	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		483	/* Setup TLB control */
		484	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		485	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		486	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		487	ENABLE_WAIT_L2_QUERY;
		488	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		489	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		490	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		491	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		492	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		493	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		494	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		495	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		496	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		497	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		498	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		499	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		500	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		501	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		502	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		503	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		504	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		505	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		506	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		507	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		508	(u32)(rdev->dummy_page.addr >> 12));
		509	for (i = 1; i < 7; i++)
		510	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		511
		512	r600_pcie_gart_tlb_flush(rdev);
		513	rdev->gart.ready = true;
1128	serge	514	return 0;
		515	}
		516
1221	serge	517	void r600_pcie_gart_disable(struct radeon_device *rdev)
1128	serge	518	{
1221	serge	519	u32 tmp;
1321	serge	520	int i, r;
1221	serge	521
		522	/* Disable all tables */
		523	for (i = 0; i < 7; i++)
		524	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		525
		526	/* Disable L2 cache */
		527	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		528	EFFECTIVE_L2_QUEUE_SIZE(7));
		529	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		530	/* Setup L1 TLB control */
		531	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		532	ENABLE_WAIT_L2_QUERY;
		533	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		534	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		535	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		536	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		537	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		538	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		539	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		540	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		541	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
		542	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
		543	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		544	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		545	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
		546	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		547	if (rdev->gart.table.vram.robj) {
1403	serge	548	r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
		549	if (likely(r == 0)) {
		550	radeon_bo_kunmap(rdev->gart.table.vram.robj);
		551	radeon_bo_unpin(rdev->gart.table.vram.robj);
		552	radeon_bo_unreserve(rdev->gart.table.vram.robj);
		553	}
1221	serge	554	}
1128	serge	555	}
		556
1221	serge	557	void r600_pcie_gart_fini(struct radeon_device *rdev)
		558	{
1963	serge	559	radeon_gart_fini(rdev);
1221	serge	560	r600_pcie_gart_disable(rdev);
		561	radeon_gart_table_vram_free(rdev);
		562	}
1128	serge	563
1221	serge	564	void r600_agp_enable(struct radeon_device *rdev)
1128	serge	565	{
1221	serge	566	u32 tmp;
		567	int i;
		568
		569	/* Setup L2 cache */
		570	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		571	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		572	EFFECTIVE_L2_QUEUE_SIZE(7));
		573	WREG32(VM_L2_CNTL2, 0);
		574	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		575	/* Setup TLB control */
		576	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		577	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		578	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		579	ENABLE_WAIT_L2_QUERY;
		580	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		581	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		582	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		583	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		584	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		585	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		586	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		587	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		588	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		589	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		590	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		591	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		592	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		593	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		594	for (i = 0; i < 7; i++)
		595	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1128	serge	596	}
		597
		598	int r600_mc_wait_for_idle(struct radeon_device *rdev)
		599	{
1221	serge	600	unsigned i;
		601	u32 tmp;
		602
		603	for (i = 0; i < rdev->usec_timeout; i++) {
		604	/* read MC_STATUS */
		605	tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
		606	if (!tmp)
1128	serge	607	return 0;
1221	serge	608	udelay(1);
		609	}
		610	return -1;
1128	serge	611	}
		612
1221	serge	613	static void r600_mc_program(struct radeon_device *rdev)
1128	serge	614	{
1221	serge	615	struct rv515_mc_save save;
		616	u32 tmp;
		617	int i, j;
		618
		619	/* Initialize HDP */
		620	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		621	WREG32((0x2c14 + j), 0x00000000);
		622	WREG32((0x2c18 + j), 0x00000000);
		623	WREG32((0x2c1c + j), 0x00000000);
		624	WREG32((0x2c20 + j), 0x00000000);
		625	WREG32((0x2c24 + j), 0x00000000);
		626	}
		627	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		628
		629	rv515_mc_stop(rdev, &save);
		630	if (r600_mc_wait_for_idle(rdev)) {
		631	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		632	}
		633	/* Lockout access through VGA aperture (doesn't exist before R600) */
		634	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		635	/* Update configuration */
		636	if (rdev->flags & RADEON_IS_AGP) {
		637	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		638	/* VRAM before AGP */
		639	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		640	rdev->mc.vram_start >> 12);
		641	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		642	rdev->mc.gtt_end >> 12);
		643	} else {
		644	/* VRAM after AGP */
		645	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		646	rdev->mc.gtt_start >> 12);
		647	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		648	rdev->mc.vram_end >> 12);
		649	}
		650	} else {
		651	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
		652	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
		653	}
		654	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
		655	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		656	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		657	WREG32(MC_VM_FB_LOCATION, tmp);
		658	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
		659	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1963	serge	660	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1221	serge	661	if (rdev->flags & RADEON_IS_AGP) {
		662	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
		663	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
		664	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		665	} else {
		666	WREG32(MC_VM_AGP_BASE, 0);
		667	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		668	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		669	}
		670	if (r600_mc_wait_for_idle(rdev)) {
		671	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		672	}
		673	rv515_mc_resume(rdev, &save);
		674	/* we need to own VRAM, so turn off the VGA renderer here
		675	* to stop it overwriting our objects */
		676	rv515_vga_render_disable(rdev);
1128	serge	677	}
		678
1430	serge	679	/**
		680	* r600_vram_gtt_location - try to find VRAM & GTT location
		681	* @rdev: radeon device structure holding all necessary informations
		682	* @mc: memory controller structure holding memory informations
		683	*
		684	* Function will place try to place VRAM at same place as in CPU (PCI)
		685	* address space as some GPU seems to have issue when we reprogram at
		686	* different address space.
		687	*
		688	* If there is not enough space to fit the unvisible VRAM after the
		689	* aperture then we limit the VRAM size to the aperture.
		690	*
		691	* If we are using AGP then place VRAM adjacent to AGP aperture are we need
		692	* them to be in one from GPU point of view so that we can program GPU to
		693	* catch access outside them (weird GPU policy see ??).
		694	*
		695	* This function will never fails, worst case are limiting VRAM or GTT.
		696	*
		697	* Note: GTT start, end, size should be initialized before calling this
		698	* function on AGP platform.
		699	*/
1963	serge	700	static void r600_vram_gtt_location(struct radeon_device rdev, struct radeon_mc mc)
1430	serge	701	{
		702	u64 size_bf, size_af;
		703
		704	if (mc->mc_vram_size > 0xE0000000) {
		705	/* leave room for at least 512M GTT */
		706	dev_warn(rdev->dev, "limiting VRAM\n");
		707	mc->real_vram_size = 0xE0000000;
		708	mc->mc_vram_size = 0xE0000000;
		709	}
		710	if (rdev->flags & RADEON_IS_AGP) {
		711	size_bf = mc->gtt_start;
		712	size_af = 0xFFFFFFFF - mc->gtt_end + 1;
		713	if (size_bf > size_af) {
		714	if (mc->mc_vram_size > size_bf) {
		715	dev_warn(rdev->dev, "limiting VRAM\n");
		716	mc->real_vram_size = size_bf;
		717	mc->mc_vram_size = size_bf;
		718	}
		719	mc->vram_start = mc->gtt_start - mc->mc_vram_size;
		720	} else {
		721	if (mc->mc_vram_size > size_af) {
		722	dev_warn(rdev->dev, "limiting VRAM\n");
		723	mc->real_vram_size = size_af;
		724	mc->mc_vram_size = size_af;
		725	}
		726	mc->vram_start = mc->gtt_end;
		727	}
		728	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
		729	dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
		730	mc->mc_vram_size >> 20, mc->vram_start,
		731	mc->vram_end, mc->real_vram_size >> 20);
		732	} else {
		733	u64 base = 0;
1963	serge	734	if (rdev->flags & RADEON_IS_IGP) {
		735	base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
		736	base <<= 24;
		737	}
1430	serge	738	radeon_vram_location(rdev, &rdev->mc, base);
1963	serge	739	rdev->mc.gtt_base_align = 0;
1430	serge	740	radeon_gtt_location(rdev, mc);
		741	}
		742	}
		743
1221	serge	744	int r600_mc_init(struct radeon_device *rdev)
1128	serge	745	{
1221	serge	746	u32 tmp;
1268	serge	747	int chansize, numchan;
1128	serge	748
1221	serge	749	/* Get VRAM informations */
1128	serge	750	rdev->mc.vram_is_ddr = true;
1221	serge	751	tmp = RREG32(RAMCFG);
		752	if (tmp & CHANSIZE_OVERRIDE) {
1128	serge	753	chansize = 16;
1221	serge	754	} else if (tmp & CHANSIZE_MASK) {
1128	serge	755	chansize = 64;
		756	} else {
		757	chansize = 32;
		758	}
1268	serge	759	tmp = RREG32(CHMAP);
		760	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		761	case 0:
		762	default:
		763	numchan = 1;
		764	break;
		765	case 1:
		766	numchan = 2;
		767	break;
		768	case 2:
		769	numchan = 4;
		770	break;
		771	case 3:
		772	numchan = 8;
		773	break;
1128	serge	774	}
1268	serge	775	rdev->mc.vram_width = numchan * chansize;
1221	serge	776	/* Could aper size report 0 ? */
1963	serge	777	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		778	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1221	serge	779	/* Setup GPU memory space */
		780	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		781	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1430	serge	782	rdev->mc.visible_vram_size = rdev->mc.aper_size;
		783	r600_vram_gtt_location(rdev, &rdev->mc);
1963	serge	784
		785	if (rdev->flags & RADEON_IS_IGP) {
		786	rs690_pm_info(rdev);
1403	serge	787	rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1963	serge	788	}
		789	radeon_update_bandwidth_info(rdev);
1221	serge	790	return 0;
1128	serge	791	}
		792
1221	serge	793	/* We doesn't check that the GPU really needs a reset we simply do the
		794	* reset, it's up to the caller to determine if the GPU needs one. We
		795	* might add an helper function to check that.
		796	*/
		797	int r600_gpu_soft_reset(struct radeon_device *rdev)
1128	serge	798	{
1221	serge	799	struct rv515_mc_save save;
		800	u32 grbm_busy_mask = S_008010_VC_BUSY(1) \| S_008010_VGT_BUSY_NO_DMA(1) \|
		801	S_008010_VGT_BUSY(1) \| S_008010_TA03_BUSY(1) \|
		802	S_008010_TC_BUSY(1) \| S_008010_SX_BUSY(1) \|
		803	S_008010_SH_BUSY(1) \| S_008010_SPI03_BUSY(1) \|
		804	S_008010_SMX_BUSY(1) \| S_008010_SC_BUSY(1) \|
		805	S_008010_PA_BUSY(1) \| S_008010_DB03_BUSY(1) \|
		806	S_008010_CR_BUSY(1) \| S_008010_CB03_BUSY(1) \|
		807	S_008010_GUI_ACTIVE(1);
		808	u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) \| S_008014_SPI1_BUSY(1) \|
		809	S_008014_SPI2_BUSY(1) \| S_008014_SPI3_BUSY(1) \|
		810	S_008014_TA0_BUSY(1) \| S_008014_TA1_BUSY(1) \|
		811	S_008014_TA2_BUSY(1) \| S_008014_TA3_BUSY(1) \|
		812	S_008014_DB0_BUSY(1) \| S_008014_DB1_BUSY(1) \|
		813	S_008014_DB2_BUSY(1) \| S_008014_DB3_BUSY(1) \|
		814	S_008014_CB0_BUSY(1) \| S_008014_CB1_BUSY(1) \|
		815	S_008014_CB2_BUSY(1) \| S_008014_CB3_BUSY(1);
		816	u32 tmp;
1128	serge	817
1963	serge	818	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
		819	return 0;
		820
1221	serge	821	dev_info(rdev->dev, "GPU softreset \n");
		822	dev_info(rdev->dev, " R_008010_GRBM_STATUS=0x%08X\n",
		823	RREG32(R_008010_GRBM_STATUS));
		824	dev_info(rdev->dev, " R_008014_GRBM_STATUS2=0x%08X\n",
		825	RREG32(R_008014_GRBM_STATUS2));
		826	dev_info(rdev->dev, " R_000E50_SRBM_STATUS=0x%08X\n",
		827	RREG32(R_000E50_SRBM_STATUS));
		828	rv515_mc_stop(rdev, &save);
		829	if (r600_mc_wait_for_idle(rdev)) {
		830	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		831	}
		832	/* Disable CP parsing/prefetching */
1963	serge	833	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1221	serge	834	/* Check if any of the rendering block is busy and reset it */
		835	if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) \|\|
		836	(RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
		837	tmp = S_008020_SOFT_RESET_CR(1) \|
		838	S_008020_SOFT_RESET_DB(1) \|
		839	S_008020_SOFT_RESET_CB(1) \|
		840	S_008020_SOFT_RESET_PA(1) \|
		841	S_008020_SOFT_RESET_SC(1) \|
		842	S_008020_SOFT_RESET_SMX(1) \|
		843	S_008020_SOFT_RESET_SPI(1) \|
		844	S_008020_SOFT_RESET_SX(1) \|
		845	S_008020_SOFT_RESET_SH(1) \|
		846	S_008020_SOFT_RESET_TC(1) \|
		847	S_008020_SOFT_RESET_TA(1) \|
		848	S_008020_SOFT_RESET_VC(1) \|
		849	S_008020_SOFT_RESET_VGT(1);
		850	dev_info(rdev->dev, " R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		851	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1963	serge	852	RREG32(R_008020_GRBM_SOFT_RESET);
		853	mdelay(15);
1221	serge	854	WREG32(R_008020_GRBM_SOFT_RESET, 0);
		855	}
		856	/* Reset CP (we always reset CP) */
		857	tmp = S_008020_SOFT_RESET_CP(1);
		858	dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		859	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1963	serge	860	RREG32(R_008020_GRBM_SOFT_RESET);
		861	mdelay(15);
1221	serge	862	WREG32(R_008020_GRBM_SOFT_RESET, 0);
		863	/* Wait a little for things to settle down */
1963	serge	864	mdelay(1);
1221	serge	865	dev_info(rdev->dev, " R_008010_GRBM_STATUS=0x%08X\n",
		866	RREG32(R_008010_GRBM_STATUS));
		867	dev_info(rdev->dev, " R_008014_GRBM_STATUS2=0x%08X\n",
		868	RREG32(R_008014_GRBM_STATUS2));
		869	dev_info(rdev->dev, " R_000E50_SRBM_STATUS=0x%08X\n",
		870	RREG32(R_000E50_SRBM_STATUS));
		871	rv515_mc_resume(rdev, &save);
		872	return 0;
1128	serge	873	}
		874
1963	serge	875	bool r600_gpu_is_lockup(struct radeon_device *rdev)
1221	serge	876	{
1963	serge	877	u32 srbm_status;
		878	u32 grbm_status;
		879	u32 grbm_status2;
		880	struct r100_gpu_lockup *lockup;
		881	int r;
		882
		883	if (rdev->family >= CHIP_RV770)
		884	lockup = &rdev->config.rv770.lockup;
		885	else
		886	lockup = &rdev->config.r600.lockup;
		887
		888	srbm_status = RREG32(R_000E50_SRBM_STATUS);
		889	grbm_status = RREG32(R_008010_GRBM_STATUS);
		890	grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
		891	if (!G_008010_GUI_ACTIVE(grbm_status)) {
		892	r100_gpu_lockup_update(lockup, &rdev->cp);
		893	return false;
		894	}
		895	/* force CP activities */
		896	r = radeon_ring_lock(rdev, 2);
		897	if (!r) {
		898	/* PACKET2 NOP */
		899	radeon_ring_write(rdev, 0x80000000);
		900	radeon_ring_write(rdev, 0x80000000);
		901	radeon_ring_unlock_commit(rdev);
		902	}
		903	rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
		904	return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
		905	}
		906
		907	int r600_asic_reset(struct radeon_device *rdev)
		908	{
1221	serge	909	return r600_gpu_soft_reset(rdev);
		910	}
		911
		912	static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
		913	u32 num_backends,
		914	u32 backend_disable_mask)
		915	{
		916	u32 backend_map = 0;
		917	u32 enabled_backends_mask;
		918	u32 enabled_backends_count;
		919	u32 cur_pipe;
		920	u32 swizzle_pipe[R6XX_MAX_PIPES];
		921	u32 cur_backend;
		922	u32 i;
		923
		924	if (num_tile_pipes > R6XX_MAX_PIPES)
		925	num_tile_pipes = R6XX_MAX_PIPES;
		926	if (num_tile_pipes < 1)
		927	num_tile_pipes = 1;
		928	if (num_backends > R6XX_MAX_BACKENDS)
		929	num_backends = R6XX_MAX_BACKENDS;
		930	if (num_backends < 1)
		931	num_backends = 1;
		932
		933	enabled_backends_mask = 0;
		934	enabled_backends_count = 0;
		935	for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
		936	if (((backend_disable_mask >> i) & 1) == 0) {
		937	enabled_backends_mask \|= (1 << i);
		938	++enabled_backends_count;
		939	}
		940	if (enabled_backends_count == num_backends)
		941	break;
		942	}
		943
		944	if (enabled_backends_count == 0) {
		945	enabled_backends_mask = 1;
		946	enabled_backends_count = 1;
		947	}
		948
		949	if (enabled_backends_count != num_backends)
		950	num_backends = enabled_backends_count;
		951
		952	memset((uint8_t )&swizzle_pipe[0], 0, sizeof(u32) R6XX_MAX_PIPES);
		953	switch (num_tile_pipes) {
		954	case 1:
		955	swizzle_pipe[0] = 0;
		956	break;
		957	case 2:
		958	swizzle_pipe[0] = 0;
		959	swizzle_pipe[1] = 1;
		960	break;
		961	case 3:
		962	swizzle_pipe[0] = 0;
		963	swizzle_pipe[1] = 1;
		964	swizzle_pipe[2] = 2;
		965	break;
		966	case 4:
		967	swizzle_pipe[0] = 0;
		968	swizzle_pipe[1] = 1;
		969	swizzle_pipe[2] = 2;
		970	swizzle_pipe[3] = 3;
		971	break;
		972	case 5:
		973	swizzle_pipe[0] = 0;
		974	swizzle_pipe[1] = 1;
		975	swizzle_pipe[2] = 2;
		976	swizzle_pipe[3] = 3;
		977	swizzle_pipe[4] = 4;
		978	break;
		979	case 6:
		980	swizzle_pipe[0] = 0;
		981	swizzle_pipe[1] = 2;
		982	swizzle_pipe[2] = 4;
		983	swizzle_pipe[3] = 5;
		984	swizzle_pipe[4] = 1;
		985	swizzle_pipe[5] = 3;
		986	break;
		987	case 7:
		988	swizzle_pipe[0] = 0;
		989	swizzle_pipe[1] = 2;
		990	swizzle_pipe[2] = 4;
		991	swizzle_pipe[3] = 6;
		992	swizzle_pipe[4] = 1;
		993	swizzle_pipe[5] = 3;
		994	swizzle_pipe[6] = 5;
		995	break;
		996	case 8:
		997	swizzle_pipe[0] = 0;
		998	swizzle_pipe[1] = 2;
		999	swizzle_pipe[2] = 4;
		1000	swizzle_pipe[3] = 6;
		1001	swizzle_pipe[4] = 1;
		1002	swizzle_pipe[5] = 3;
		1003	swizzle_pipe[6] = 5;
		1004	swizzle_pipe[7] = 7;
		1005	break;
		1006	}
		1007
		1008	cur_backend = 0;
		1009	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
		1010	while (((1 << cur_backend) & enabled_backends_mask) == 0)
		1011	cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
		1012
		1013	backend_map \|= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
		1014
		1015	cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
		1016	}
		1017
		1018	return backend_map;
		1019	}
		1020
		1021	int r600_count_pipe_bits(uint32_t val)
		1022	{
		1023	int i, ret = 0;
		1024
		1025	for (i = 0; i < 32; i++) {
		1026	ret += val & 1;
		1027	val >>= 1;
		1028	}
		1029	return ret;
		1030	}
		1031
		1032	void r600_gpu_init(struct radeon_device *rdev)
		1033	{
		1034	u32 tiling_config;
		1035	u32 ramcfg;
1430	serge	1036	u32 backend_map;
		1037	u32 cc_rb_backend_disable;
		1038	u32 cc_gc_shader_pipe_config;
1221	serge	1039	u32 tmp;
		1040	int i, j;
		1041	u32 sq_config;
		1042	u32 sq_gpr_resource_mgmt_1 = 0;
		1043	u32 sq_gpr_resource_mgmt_2 = 0;
		1044	u32 sq_thread_resource_mgmt = 0;
		1045	u32 sq_stack_resource_mgmt_1 = 0;
		1046	u32 sq_stack_resource_mgmt_2 = 0;
		1047
		1048	/* FIXME: implement */
		1049	switch (rdev->family) {
		1050	case CHIP_R600:
		1051	rdev->config.r600.max_pipes = 4;
		1052	rdev->config.r600.max_tile_pipes = 8;
		1053	rdev->config.r600.max_simds = 4;
		1054	rdev->config.r600.max_backends = 4;
		1055	rdev->config.r600.max_gprs = 256;
		1056	rdev->config.r600.max_threads = 192;
		1057	rdev->config.r600.max_stack_entries = 256;
		1058	rdev->config.r600.max_hw_contexts = 8;
		1059	rdev->config.r600.max_gs_threads = 16;
		1060	rdev->config.r600.sx_max_export_size = 128;
		1061	rdev->config.r600.sx_max_export_pos_size = 16;
		1062	rdev->config.r600.sx_max_export_smx_size = 128;
		1063	rdev->config.r600.sq_num_cf_insts = 2;
		1064	break;
		1065	case CHIP_RV630:
		1066	case CHIP_RV635:
		1067	rdev->config.r600.max_pipes = 2;
		1068	rdev->config.r600.max_tile_pipes = 2;
		1069	rdev->config.r600.max_simds = 3;
		1070	rdev->config.r600.max_backends = 1;
		1071	rdev->config.r600.max_gprs = 128;
		1072	rdev->config.r600.max_threads = 192;
		1073	rdev->config.r600.max_stack_entries = 128;
		1074	rdev->config.r600.max_hw_contexts = 8;
		1075	rdev->config.r600.max_gs_threads = 4;
		1076	rdev->config.r600.sx_max_export_size = 128;
		1077	rdev->config.r600.sx_max_export_pos_size = 16;
		1078	rdev->config.r600.sx_max_export_smx_size = 128;
		1079	rdev->config.r600.sq_num_cf_insts = 2;
		1080	break;
		1081	case CHIP_RV610:
		1082	case CHIP_RV620:
		1083	case CHIP_RS780:
		1084	case CHIP_RS880:
		1085	rdev->config.r600.max_pipes = 1;
		1086	rdev->config.r600.max_tile_pipes = 1;
		1087	rdev->config.r600.max_simds = 2;
		1088	rdev->config.r600.max_backends = 1;
		1089	rdev->config.r600.max_gprs = 128;
		1090	rdev->config.r600.max_threads = 192;
		1091	rdev->config.r600.max_stack_entries = 128;
		1092	rdev->config.r600.max_hw_contexts = 4;
		1093	rdev->config.r600.max_gs_threads = 4;
		1094	rdev->config.r600.sx_max_export_size = 128;
		1095	rdev->config.r600.sx_max_export_pos_size = 16;
		1096	rdev->config.r600.sx_max_export_smx_size = 128;
		1097	rdev->config.r600.sq_num_cf_insts = 1;
		1098	break;
		1099	case CHIP_RV670:
		1100	rdev->config.r600.max_pipes = 4;
		1101	rdev->config.r600.max_tile_pipes = 4;
		1102	rdev->config.r600.max_simds = 4;
		1103	rdev->config.r600.max_backends = 4;
		1104	rdev->config.r600.max_gprs = 192;
		1105	rdev->config.r600.max_threads = 192;
		1106	rdev->config.r600.max_stack_entries = 256;
		1107	rdev->config.r600.max_hw_contexts = 8;
		1108	rdev->config.r600.max_gs_threads = 16;
		1109	rdev->config.r600.sx_max_export_size = 128;
		1110	rdev->config.r600.sx_max_export_pos_size = 16;
		1111	rdev->config.r600.sx_max_export_smx_size = 128;
		1112	rdev->config.r600.sq_num_cf_insts = 2;
		1113	break;
		1114	default:
		1115	break;
		1116	}
		1117
		1118	/* Initialize HDP */
		1119	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1120	WREG32((0x2c14 + j), 0x00000000);
		1121	WREG32((0x2c18 + j), 0x00000000);
		1122	WREG32((0x2c1c + j), 0x00000000);
		1123	WREG32((0x2c20 + j), 0x00000000);
		1124	WREG32((0x2c24 + j), 0x00000000);
		1125	}
		1126
		1127	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1128
		1129	/* Setup tiling */
		1130	tiling_config = 0;
		1131	ramcfg = RREG32(RAMCFG);
		1132	switch (rdev->config.r600.max_tile_pipes) {
		1133	case 1:
		1134	tiling_config \|= PIPE_TILING(0);
		1135	break;
		1136	case 2:
		1137	tiling_config \|= PIPE_TILING(1);
		1138	break;
		1139	case 4:
		1140	tiling_config \|= PIPE_TILING(2);
		1141	break;
		1142	case 8:
		1143	tiling_config \|= PIPE_TILING(3);
		1144	break;
		1145	default:
		1146	break;
		1147	}
1430	serge	1148	rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
		1149	rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1221	serge	1150	tiling_config \|= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1963	serge	1151	tiling_config \|= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
		1152	if ((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT)
		1153	rdev->config.r600.tiling_group_size = 512;
		1154	else
1430	serge	1155	rdev->config.r600.tiling_group_size = 256;
1221	serge	1156	tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
		1157	if (tmp > 3) {
		1158	tiling_config \|= ROW_TILING(3);
		1159	tiling_config \|= SAMPLE_SPLIT(3);
		1160	} else {
		1161	tiling_config \|= ROW_TILING(tmp);
		1162	tiling_config \|= SAMPLE_SPLIT(tmp);
		1163	}
		1164	tiling_config \|= BANK_SWAPS(1);
1430	serge	1165
		1166	cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
		1167	cc_rb_backend_disable \|=
		1168	BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << rdev->config.r600.max_backends) & R6XX_MAX_BACKENDS_MASK);
		1169
		1170	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
		1171	cc_gc_shader_pipe_config \|=
		1172	INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << rdev->config.r600.max_pipes) & R6XX_MAX_PIPES_MASK);
		1173	cc_gc_shader_pipe_config \|=
		1174	INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << rdev->config.r600.max_simds) & R6XX_MAX_SIMDS_MASK);
		1175
		1176	backend_map = r600_get_tile_pipe_to_backend_map(rdev->config.r600.max_tile_pipes,
		1177	(R6XX_MAX_BACKENDS -
		1178	r600_count_pipe_bits((cc_rb_backend_disable &
		1179	R6XX_MAX_BACKENDS_MASK) >> 16)),
		1180	(cc_rb_backend_disable >> 16));
1963	serge	1181	rdev->config.r600.tile_config = tiling_config;
1430	serge	1182	tiling_config \|= BACKEND_MAP(backend_map);
1221	serge	1183	WREG32(GB_TILING_CONFIG, tiling_config);
		1184	WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
		1185	WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
		1186
		1187	/* Setup pipes */
1430	serge	1188	WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
		1189	WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1963	serge	1190	WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1221	serge	1191
1430	serge	1192	tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1221	serge	1193	WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
		1194	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
		1195
		1196	/* Setup some CP states */
		1197	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \| ROQ_IB2_START(0x2b)));
		1198	WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) \| ROQ_END(0x40)));
		1199
		1200	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \| SYNC_GRADIENT \|
		1201	SYNC_WALKER \| SYNC_ALIGNER));
		1202	/* Setup various GPU states */
		1203	if (rdev->family == CHIP_RV670)
		1204	WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
		1205
		1206	tmp = RREG32(SX_DEBUG_1);
		1207	tmp \|= SMX_EVENT_RELEASE;
		1208	if ((rdev->family > CHIP_R600))
		1209	tmp \|= ENABLE_NEW_SMX_ADDRESS;
		1210	WREG32(SX_DEBUG_1, tmp);
		1211
		1212	if (((rdev->family) == CHIP_R600) \|\|
		1213	((rdev->family) == CHIP_RV630) \|\|
		1214	((rdev->family) == CHIP_RV610) \|\|
		1215	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1216	((rdev->family) == CHIP_RS780) \|\|
		1217	((rdev->family) == CHIP_RS880)) {
1221	serge	1218	WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
		1219	} else {
		1220	WREG32(DB_DEBUG, 0);
		1221	}
		1222	WREG32(DB_WATERMARKS, (DEPTH_FREE(4) \| DEPTH_CACHELINE_FREE(16) \|
		1223	DEPTH_FLUSH(16) \| DEPTH_PENDING_FREE(4)));
		1224
		1225	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1226	WREG32(VGT_NUM_INSTANCES, 0);
		1227
		1228	WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
		1229	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
		1230
		1231	tmp = RREG32(SQ_MS_FIFO_SIZES);
		1232	if (((rdev->family) == CHIP_RV610) \|\|
		1233	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1234	((rdev->family) == CHIP_RS780) \|\|
		1235	((rdev->family) == CHIP_RS880)) {
1221	serge	1236	tmp = (CACHE_FIFO_SIZE(0xa) \|
		1237	FETCH_FIFO_HIWATER(0xa) \|
		1238	DONE_FIFO_HIWATER(0xe0) \|
		1239	ALU_UPDATE_FIFO_HIWATER(0x8));
		1240	} else if (((rdev->family) == CHIP_R600) \|\|
		1241	((rdev->family) == CHIP_RV630)) {
		1242	tmp &= ~DONE_FIFO_HIWATER(0xff);
		1243	tmp \|= DONE_FIFO_HIWATER(0x4);
		1244	}
		1245	WREG32(SQ_MS_FIFO_SIZES, tmp);
		1246
		1247	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
		1248	* should be adjusted as needed by the 2D/3D drivers. This just sets default values
		1249	*/
		1250	sq_config = RREG32(SQ_CONFIG);
		1251	sq_config &= ~(PS_PRIO(3) \|
		1252	VS_PRIO(3) \|
		1253	GS_PRIO(3) \|
		1254	ES_PRIO(3));
		1255	sq_config \|= (DX9_CONSTS \|
		1256	VC_ENABLE \|
		1257	PS_PRIO(0) \|
		1258	VS_PRIO(1) \|
		1259	GS_PRIO(2) \|
		1260	ES_PRIO(3));
		1261
		1262	if ((rdev->family) == CHIP_R600) {
		1263	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) \|
		1264	NUM_VS_GPRS(124) \|
		1265	NUM_CLAUSE_TEMP_GPRS(4));
		1266	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) \|
		1267	NUM_ES_GPRS(0));
		1268	sq_thread_resource_mgmt = (NUM_PS_THREADS(136) \|
		1269	NUM_VS_THREADS(48) \|
		1270	NUM_GS_THREADS(4) \|
		1271	NUM_ES_THREADS(4));
		1272	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) \|
		1273	NUM_VS_STACK_ENTRIES(128));
		1274	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) \|
		1275	NUM_ES_STACK_ENTRIES(0));
		1276	} else if (((rdev->family) == CHIP_RV610) \|\|
		1277	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1278	((rdev->family) == CHIP_RS780) \|\|
		1279	((rdev->family) == CHIP_RS880)) {
1221	serge	1280	/* no vertex cache */
		1281	sq_config &= ~VC_ENABLE;
		1282
		1283	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1284	NUM_VS_GPRS(44) \|
		1285	NUM_CLAUSE_TEMP_GPRS(2));
		1286	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1287	NUM_ES_GPRS(17));
		1288	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1289	NUM_VS_THREADS(78) \|
		1290	NUM_GS_THREADS(4) \|
		1291	NUM_ES_THREADS(31));
		1292	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1293	NUM_VS_STACK_ENTRIES(40));
		1294	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1295	NUM_ES_STACK_ENTRIES(16));
		1296	} else if (((rdev->family) == CHIP_RV630) \|\|
		1297	((rdev->family) == CHIP_RV635)) {
		1298	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1299	NUM_VS_GPRS(44) \|
		1300	NUM_CLAUSE_TEMP_GPRS(2));
		1301	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) \|
		1302	NUM_ES_GPRS(18));
		1303	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1304	NUM_VS_THREADS(78) \|
		1305	NUM_GS_THREADS(4) \|
		1306	NUM_ES_THREADS(31));
		1307	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1308	NUM_VS_STACK_ENTRIES(40));
		1309	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1310	NUM_ES_STACK_ENTRIES(16));
		1311	} else if ((rdev->family) == CHIP_RV670) {
		1312	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1313	NUM_VS_GPRS(44) \|
		1314	NUM_CLAUSE_TEMP_GPRS(2));
		1315	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1316	NUM_ES_GPRS(17));
		1317	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1318	NUM_VS_THREADS(78) \|
		1319	NUM_GS_THREADS(4) \|
		1320	NUM_ES_THREADS(31));
		1321	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) \|
		1322	NUM_VS_STACK_ENTRIES(64));
		1323	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) \|
		1324	NUM_ES_STACK_ENTRIES(64));
		1325	}
		1326
		1327	WREG32(SQ_CONFIG, sq_config);
		1328	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		1329	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		1330	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		1331	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		1332	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		1333
		1334	if (((rdev->family) == CHIP_RV610) \|\|
		1335	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1336	((rdev->family) == CHIP_RS780) \|\|
		1337	((rdev->family) == CHIP_RS880)) {
1221	serge	1338	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
		1339	} else {
		1340	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
		1341	}
		1342
		1343	/* More default values. 2D/3D driver should adjust as needed */
		1344	WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) \| S0_Y(0x4) \|
		1345	S1_X(0x4) \| S1_Y(0xc)));
		1346	WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) \| S0_Y(0xe) \|
		1347	S1_X(0x2) \| S1_Y(0x2) \|
		1348	S2_X(0xa) \| S2_Y(0x6) \|
		1349	S3_X(0x6) \| S3_Y(0xa)));
		1350	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) \| S0_Y(0xb) \|
		1351	S1_X(0x4) \| S1_Y(0xc) \|
		1352	S2_X(0x1) \| S2_Y(0x6) \|
		1353	S3_X(0xa) \| S3_Y(0xe)));
		1354	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) \| S4_Y(0x1) \|
		1355	S5_X(0x0) \| S5_Y(0x0) \|
		1356	S6_X(0xb) \| S6_Y(0x4) \|
		1357	S7_X(0x7) \| S7_Y(0x8)));
		1358
		1359	WREG32(VGT_STRMOUT_EN, 0);
		1360	tmp = rdev->config.r600.max_pipes * 16;
		1361	switch (rdev->family) {
		1362	case CHIP_RV610:
1268	serge	1363	case CHIP_RV620:
1221	serge	1364	case CHIP_RS780:
1268	serge	1365	case CHIP_RS880:
1221	serge	1366	tmp += 32;
		1367	break;
		1368	case CHIP_RV670:
		1369	tmp += 128;
		1370	break;
		1371	default:
		1372	break;
		1373	}
		1374	if (tmp > 256) {
		1375	tmp = 256;
		1376	}
		1377	WREG32(VGT_ES_PER_GS, 128);
		1378	WREG32(VGT_GS_PER_ES, tmp);
		1379	WREG32(VGT_GS_PER_VS, 2);
		1380	WREG32(VGT_GS_VERTEX_REUSE, 16);
		1381
		1382	/* more default values. 2D/3D driver should adjust as needed */
		1383	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		1384	WREG32(VGT_STRMOUT_EN, 0);
		1385	WREG32(SX_MISC, 0);
		1386	WREG32(PA_SC_MODE_CNTL, 0);
		1387	WREG32(PA_SC_AA_CONFIG, 0);
		1388	WREG32(PA_SC_LINE_STIPPLE, 0);
		1389	WREG32(SPI_INPUT_Z, 0);
		1390	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
		1391	WREG32(CB_COLOR7_FRAG, 0);
		1392
		1393	/* Clear render buffer base addresses */
		1394	WREG32(CB_COLOR0_BASE, 0);
		1395	WREG32(CB_COLOR1_BASE, 0);
		1396	WREG32(CB_COLOR2_BASE, 0);
		1397	WREG32(CB_COLOR3_BASE, 0);
		1398	WREG32(CB_COLOR4_BASE, 0);
		1399	WREG32(CB_COLOR5_BASE, 0);
		1400	WREG32(CB_COLOR6_BASE, 0);
		1401	WREG32(CB_COLOR7_BASE, 0);
		1402	WREG32(CB_COLOR7_FRAG, 0);
		1403
		1404	switch (rdev->family) {
		1405	case CHIP_RV610:
1268	serge	1406	case CHIP_RV620:
1221	serge	1407	case CHIP_RS780:
1268	serge	1408	case CHIP_RS880:
1221	serge	1409	tmp = TC_L2_SIZE(8);
		1410	break;
		1411	case CHIP_RV630:
		1412	case CHIP_RV635:
		1413	tmp = TC_L2_SIZE(4);
		1414	break;
		1415	case CHIP_R600:
		1416	tmp = TC_L2_SIZE(0) \| L2_DISABLE_LATE_HIT;
		1417	break;
		1418	default:
		1419	tmp = TC_L2_SIZE(0);
		1420	break;
		1421	}
		1422	WREG32(TC_CNTL, tmp);
		1423
		1424	tmp = RREG32(HDP_HOST_PATH_CNTL);
		1425	WREG32(HDP_HOST_PATH_CNTL, tmp);
		1426
		1427	tmp = RREG32(ARB_POP);
		1428	tmp \|= ENABLE_TC128;
		1429	WREG32(ARB_POP, tmp);
		1430
		1431	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1432	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA \|
		1433	NUM_CLIP_SEQ(3)));
		1434	WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
		1435	}
		1436
		1437
1128	serge	1438	/*
		1439	* Indirect registers accessor
		1440	*/
1221	serge	1441	u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1128	serge	1442	{
1221	serge	1443	u32 r;
1128	serge	1444
1221	serge	1445	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1446	(void)RREG32(PCIE_PORT_INDEX);
		1447	r = RREG32(PCIE_PORT_DATA);
1128	serge	1448	return r;
		1449	}
		1450
1221	serge	1451	void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1128	serge	1452	{
1221	serge	1453	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1454	(void)RREG32(PCIE_PORT_INDEX);
		1455	WREG32(PCIE_PORT_DATA, (v));
		1456	(void)RREG32(PCIE_PORT_DATA);
1128	serge	1457	}
1221	serge	1458
		1459	/*
		1460	* CP & Ring
		1461	*/
		1462	void r600_cp_stop(struct radeon_device *rdev)
		1463	{
1963	serge	1464	// radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1221	serge	1465	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1963	serge	1466	WREG32(SCRATCH_UMSK, 0);
1221	serge	1467	}
1413	serge	1468
		1469	int r600_init_microcode(struct radeon_device *rdev)
		1470	{
		1471	struct platform_device *pdev;
		1472	const char *chip_name;
		1473	const char *rlc_chip_name;
		1474	size_t pfp_req_size, me_req_size, rlc_req_size;
		1475	char fw_name[30];
		1476	int err;
		1477
		1478	DRM_DEBUG("\n");
		1479
		1480	pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
		1481	err = IS_ERR(pdev);
		1482	if (err) {
		1483	printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
		1484	return -EINVAL;
		1485	}
		1486
		1487	switch (rdev->family) {
		1488	case CHIP_R600:
		1489	chip_name = "R600";
		1490	rlc_chip_name = "R600";
		1491	break;
		1492	case CHIP_RV610:
		1493	chip_name = "RV610";
		1494	rlc_chip_name = "R600";
		1495	break;
		1496	case CHIP_RV630:
		1497	chip_name = "RV630";
		1498	rlc_chip_name = "R600";
		1499	break;
		1500	case CHIP_RV620:
		1501	chip_name = "RV620";
		1502	rlc_chip_name = "R600";
		1503	break;
		1504	case CHIP_RV635:
		1505	chip_name = "RV635";
		1506	rlc_chip_name = "R600";
		1507	break;
		1508	case CHIP_RV670:
		1509	chip_name = "RV670";
		1510	rlc_chip_name = "R600";
		1511	break;
		1512	case CHIP_RS780:
		1513	case CHIP_RS880:
		1514	chip_name = "RS780";
		1515	rlc_chip_name = "R600";
		1516	break;
		1517	case CHIP_RV770:
		1518	chip_name = "RV770";
		1519	rlc_chip_name = "R700";
		1520	break;
		1521	case CHIP_RV730:
		1522	case CHIP_RV740:
		1523	chip_name = "RV730";
		1524	rlc_chip_name = "R700";
		1525	break;
		1526	case CHIP_RV710:
		1527	chip_name = "RV710";
		1528	rlc_chip_name = "R700";
		1529	break;
1963	serge	1530	case CHIP_CEDAR:
		1531	chip_name = "CEDAR";
		1532	rlc_chip_name = "CEDAR";
		1533	break;
		1534	case CHIP_REDWOOD:
		1535	chip_name = "REDWOOD";
		1536	rlc_chip_name = "REDWOOD";
		1537	break;
		1538	case CHIP_JUNIPER:
		1539	chip_name = "JUNIPER";
		1540	rlc_chip_name = "JUNIPER";
		1541	break;
		1542	case CHIP_CYPRESS:
		1543	case CHIP_HEMLOCK:
		1544	chip_name = "CYPRESS";
		1545	rlc_chip_name = "CYPRESS";
		1546	break;
		1547	case CHIP_PALM:
		1548	chip_name = "PALM";
		1549	rlc_chip_name = "SUMO";
		1550	break;
1986	serge	1551	case CHIP_SUMO:
		1552	chip_name = "SUMO";
		1553	rlc_chip_name = "SUMO";
		1554	break;
		1555	case CHIP_SUMO2:
		1556	chip_name = "SUMO2";
		1557	rlc_chip_name = "SUMO";
		1558	break;
1413	serge	1559	default: BUG();
		1560	}
		1561
1963	serge	1562	if (rdev->family >= CHIP_CEDAR) {
		1563	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		1564	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		1565	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		1566	} else if (rdev->family >= CHIP_RV770) {
1413	serge	1567	pfp_req_size = R700_PFP_UCODE_SIZE * 4;
		1568	me_req_size = R700_PM4_UCODE_SIZE * 4;
		1569	rlc_req_size = R700_RLC_UCODE_SIZE * 4;
		1570	} else {
		1571	pfp_req_size = PFP_UCODE_SIZE * 4;
		1572	me_req_size = PM4_UCODE_SIZE * 12;
		1573	rlc_req_size = RLC_UCODE_SIZE * 4;
		1574	}
		1575
		1576	DRM_INFO("Loading %s Microcode\n", chip_name);
		1577
		1578	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
		1579	err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
		1580	if (err)
		1581	goto out;
		1582	if (rdev->pfp_fw->size != pfp_req_size) {
		1583	printk(KERN_ERR
		1584	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1585	rdev->pfp_fw->size, fw_name);
		1586	err = -EINVAL;
		1587	goto out;
		1588	}
		1589
		1590	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
		1591	err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
		1592	if (err)
		1593	goto out;
		1594	if (rdev->me_fw->size != me_req_size) {
		1595	printk(KERN_ERR
		1596	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1597	rdev->me_fw->size, fw_name);
		1598	err = -EINVAL;
		1599	}
		1600
		1601	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
		1602	err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
		1603	if (err)
		1604	goto out;
		1605	if (rdev->rlc_fw->size != rlc_req_size) {
		1606	printk(KERN_ERR
		1607	"r600_rlc: Bogus length %zu in firmware \"%s\"\n",
		1608	rdev->rlc_fw->size, fw_name);
		1609	err = -EINVAL;
		1610	}
		1611
		1612	out:
		1613	platform_device_unregister(pdev);
		1614
		1615	if (err) {
		1616	if (err != -EINVAL)
		1617	printk(KERN_ERR
		1618	"r600_cp: Failed to load firmware \"%s\"\n",
		1619	fw_name);
		1620	release_firmware(rdev->pfp_fw);
		1621	rdev->pfp_fw = NULL;
		1622	release_firmware(rdev->me_fw);
		1623	rdev->me_fw = NULL;
		1624	release_firmware(rdev->rlc_fw);
		1625	rdev->rlc_fw = NULL;
		1626	}
		1627	return err;
		1628	}
		1629
		1630	static int r600_cp_load_microcode(struct radeon_device *rdev)
		1631	{
		1632	const __be32 *fw_data;
		1633	int i;
		1634
		1635	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1636	return -EINVAL;
		1637
		1638	r600_cp_stop(rdev);
		1639
1963	serge	1640	WREG32(CP_RB_CNTL,
		1641	#ifdef __BIG_ENDIAN
		1642	BUF_SWAP_32BIT \|
		1643	#endif
		1644	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
1413	serge	1645
		1646	/* Reset cp */
		1647	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1648	RREG32(GRBM_SOFT_RESET);
		1649	mdelay(15);
		1650	WREG32(GRBM_SOFT_RESET, 0);
		1651
		1652	WREG32(CP_ME_RAM_WADDR, 0);
		1653
		1654	fw_data = (const __be32 *)rdev->me_fw->data;
		1655	WREG32(CP_ME_RAM_WADDR, 0);
		1656	for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
		1657	WREG32(CP_ME_RAM_DATA,
		1658	be32_to_cpup(fw_data++));
		1659
		1660	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1661	WREG32(CP_PFP_UCODE_ADDR, 0);
		1662	for (i = 0; i < PFP_UCODE_SIZE; i++)
		1663	WREG32(CP_PFP_UCODE_DATA,
		1664	be32_to_cpup(fw_data++));
		1665
		1666	WREG32(CP_PFP_UCODE_ADDR, 0);
		1667	WREG32(CP_ME_RAM_WADDR, 0);
		1668	WREG32(CP_ME_RAM_RADDR, 0);
		1669	return 0;
		1670	}
		1671
1221	serge	1672	int r600_cp_start(struct radeon_device *rdev)
		1673	{
		1674	int r;
		1675	uint32_t cp_me;
		1676
		1677	r = radeon_ring_lock(rdev, 7);
		1678	if (r) {
		1679	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1680	return r;
		1681	}
		1682	radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1683	radeon_ring_write(rdev, 0x1);
1963	serge	1684	if (rdev->family >= CHIP_RV770) {
		1685	radeon_ring_write(rdev, 0x0);
		1686	radeon_ring_write(rdev, rdev->config.rv770.max_hw_contexts - 1);
		1687	} else {
1221	serge	1688	radeon_ring_write(rdev, 0x3);
		1689	radeon_ring_write(rdev, rdev->config.r600.max_hw_contexts - 1);
		1690	}
		1691	radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1692	radeon_ring_write(rdev, 0);
		1693	radeon_ring_write(rdev, 0);
		1694	radeon_ring_unlock_commit(rdev);
		1695
		1696	cp_me = 0xff;
		1697	WREG32(R_0086D8_CP_ME_CNTL, cp_me);
		1698	return 0;
		1699	}
1413	serge	1700
		1701	int r600_cp_resume(struct radeon_device *rdev)
		1702	{
		1703	u32 tmp;
		1704	u32 rb_bufsz;
		1705	int r;
		1706
		1707	/* Reset cp */
		1708	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1709	RREG32(GRBM_SOFT_RESET);
		1710	mdelay(15);
		1711	WREG32(GRBM_SOFT_RESET, 0);
		1712
		1713	/* Set ring buffer size */
		1714	rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1963	serge	1715	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
1413	serge	1716	#ifdef __BIG_ENDIAN
		1717	tmp \|= BUF_SWAP_32BIT;
		1718	#endif
		1719	WREG32(CP_RB_CNTL, tmp);
		1720	WREG32(CP_SEM_WAIT_TIMER, 0x4);
		1721
		1722	/* Set the write pointer delay */
		1723	WREG32(CP_RB_WPTR_DELAY, 0);
		1724
		1725	/* Initialize the ring buffer's read and write pointers */
		1726	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1727	WREG32(CP_RB_RPTR_WR, 0);
		1728	WREG32(CP_RB_WPTR, 0);
1963	serge	1729
		1730	/* set the wb address whether it's enabled or not */
		1731	WREG32(CP_RB_RPTR_ADDR,
		1732	#ifdef __BIG_ENDIAN
		1733	RB_RPTR_SWAP(2) \|
		1734	#endif
		1735	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1736	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1737	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1738
		1739	if (rdev->wb.enabled)
		1740	WREG32(SCRATCH_UMSK, 0xff);
		1741	else {
		1742	tmp \|= RB_NO_UPDATE;
		1743	WREG32(SCRATCH_UMSK, 0);
		1744	}
		1745
1413	serge	1746	mdelay(1);
		1747	WREG32(CP_RB_CNTL, tmp);
		1748
		1749	WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
		1750	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1751
		1752	rdev->cp.rptr = RREG32(CP_RB_RPTR);
		1753	rdev->cp.wptr = RREG32(CP_RB_WPTR);
		1754
		1755	r600_cp_start(rdev);
		1756	rdev->cp.ready = true;
		1757	r = radeon_ring_test(rdev);
		1758	if (r) {
		1759	rdev->cp.ready = false;
		1760	return r;
		1761	}
		1762	return 0;
		1763	}
		1764
1221	serge	1765	void r600_cp_commit(struct radeon_device *rdev)
		1766	{
		1767	WREG32(CP_RB_WPTR, rdev->cp.wptr);
		1768	(void)RREG32(CP_RB_WPTR);
		1769	}
		1770
1233	serge	1771	void r600_ring_init(struct radeon_device *rdev, unsigned ring_size)
		1772	{
		1773	u32 rb_bufsz;
1221	serge	1774
1233	serge	1775	/* Align ring size */
		1776	rb_bufsz = drm_order(ring_size / 8);
		1777	ring_size = (1 << (rb_bufsz + 1)) * 4;
		1778	rdev->cp.ring_size = ring_size;
		1779	rdev->cp.align_mask = 16 - 1;
		1780	}
		1781
1963	serge	1782	void r600_cp_fini(struct radeon_device *rdev)
		1783	{
		1784	r600_cp_stop(rdev);
		1785	radeon_ring_fini(rdev);
		1786	}
1233	serge	1787
1963	serge	1788
1233	serge	1789	/*
		1790	* GPU scratch registers helpers function.
		1791	*/
		1792	void r600_scratch_init(struct radeon_device *rdev)
		1793	{
		1794	int i;
		1795
		1796	rdev->scratch.num_reg = 7;
1963	serge	1797	rdev->scratch.reg_base = SCRATCH_REG0;
1233	serge	1798	for (i = 0; i < rdev->scratch.num_reg; i++) {
		1799	rdev->scratch.free[i] = true;
1963	serge	1800	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
1233	serge	1801	}
		1802	}
1413	serge	1803
		1804	int r600_ring_test(struct radeon_device *rdev)
		1805	{
		1806	uint32_t scratch;
		1807	uint32_t tmp = 0;
		1808	unsigned i;
		1809	int r;
		1810
		1811	r = radeon_scratch_get(rdev, &scratch);
		1812	if (r) {
		1813	DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
		1814	return r;
		1815	}
		1816	WREG32(scratch, 0xCAFEDEAD);
		1817	r = radeon_ring_lock(rdev, 3);
		1818	if (r) {
		1819	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1820	radeon_scratch_free(rdev, scratch);
		1821	return r;
		1822	}
		1823	radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1824	radeon_ring_write(rdev, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		1825	radeon_ring_write(rdev, 0xDEADBEEF);
		1826	radeon_ring_unlock_commit(rdev);
		1827	for (i = 0; i < rdev->usec_timeout; i++) {
		1828	tmp = RREG32(scratch);
		1829	if (tmp == 0xDEADBEEF)
		1830	break;
		1831	DRM_UDELAY(1);
		1832	}
		1833	if (i < rdev->usec_timeout) {
		1834	DRM_INFO("ring test succeeded in %d usecs\n", i);
		1835	} else {
		1836	DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n",
		1837	scratch, tmp);
		1838	r = -EINVAL;
		1839	}
		1840	radeon_scratch_free(rdev, scratch);
		1841	return r;
		1842	}
1963	serge	1843
1413	serge	1844	void r600_fence_ring_emit(struct radeon_device *rdev,
		1845	struct radeon_fence *fence)
		1846	{
1963	serge	1847	if (rdev->wb.use_event) {
		1848	u64 addr = rdev->wb.gpu_addr + R600_WB_EVENT_OFFSET +
		1849	(u64)(rdev->fence_drv.scratch_reg - rdev->scratch.reg_base);
		1850	/* EVENT_WRITE_EOP - flush caches, send int */
		1851	radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		1852	radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) \| EVENT_INDEX(5));
		1853	radeon_ring_write(rdev, addr & 0xffffffff);
		1854	radeon_ring_write(rdev, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		1855	radeon_ring_write(rdev, fence->seq);
		1856	radeon_ring_write(rdev, 0);
		1857	} else {
1430	serge	1858	radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
1963	serge	1859	radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) \| EVENT_INDEX(0));
1430	serge	1860	/* wait for 3D idle clean */
		1861	radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1862	radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		1863	radeon_ring_write(rdev, WAIT_3D_IDLE_bit \| WAIT_3D_IDLECLEAN_bit);
1413	serge	1864	/* Emit fence sequence & fire IRQ */
		1865	radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1866	radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		1867	radeon_ring_write(rdev, fence->seq);
		1868	/* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
		1869	radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
		1870	radeon_ring_write(rdev, RB_INT_STAT);
1963	serge	1871	}
1413	serge	1872	}
1963	serge	1873
		1874
1221	serge	1875	int r600_set_surface_reg(struct radeon_device *rdev, int reg,
		1876	uint32_t tiling_flags, uint32_t pitch,
		1877	uint32_t offset, uint32_t obj_size)
		1878	{
		1879	/* FIXME: implement */
		1880	return 0;
		1881	}
		1882
		1883	void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
		1884	{
		1885	/* FIXME: implement */
		1886	}
		1887
		1888	int r600_startup(struct radeon_device *rdev)
		1889	{
		1890	int r;
		1891
1963	serge	1892	/* enable pcie gen2 link */
		1893	r600_pcie_gen2_enable(rdev);
		1894
1413	serge	1895	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		1896	r = r600_init_microcode(rdev);
		1897	if (r) {
		1898	DRM_ERROR("Failed to load firmware!\n");
		1899	return r;
		1900	}
		1901	}
		1902
1221	serge	1903	r600_mc_program(rdev);
		1904	if (rdev->flags & RADEON_IS_AGP) {
		1905	r600_agp_enable(rdev);
		1906	} else {
		1907	r = r600_pcie_gart_enable(rdev);
		1908	if (r)
		1909	return r;
		1910	}
		1911	r600_gpu_init(rdev);
		1912
1413	serge	1913	r = radeon_ring_init(rdev, rdev->cp.ring_size);
		1914	if (r)
		1915	return r;
		1916	r = r600_cp_load_microcode(rdev);
		1917	if (r)
		1918	return r;
		1919	r = r600_cp_resume(rdev);
		1920	if (r)
		1921	return r;
1963	serge	1922
1221	serge	1923	return 0;
		1924	}
		1925
		1926	void r600_vga_set_state(struct radeon_device *rdev, bool state)
		1927	{
		1928	uint32_t temp;
		1929
		1930	temp = RREG32(CONFIG_CNTL);
		1931	if (state == false) {
		1932	temp &= ~(1<<0);
		1933	temp \|= (1<<1);
		1934	} else {
		1935	temp &= ~(1<<1);
		1936	}
		1937	WREG32(CONFIG_CNTL, temp);
		1938	}
		1939
		1940
		1941
		1942
		1943
		1944	/* Plan is to move initialization in that function and use
		1945	* helper function so that radeon_device_init pretty much
		1946	* do nothing more than calling asic specific function. This
		1947	* should also allow to remove a bunch of callback function
		1948	* like vram_info.
		1949	*/
		1950	int r600_init(struct radeon_device *rdev)
		1951	{
		1952	int r;
		1953
		1954	if (r600_debugfs_mc_info_init(rdev)) {
		1955	DRM_ERROR("Failed to register debugfs file for mc !\n");
		1956	}
		1957	/* This don't do much */
2004	serge	1958	r = radeon_gem_init(rdev);
		1959	if (r)
		1960	return r;
1221	serge	1961	/* Read BIOS */
		1962	if (!radeon_get_bios(rdev)) {
		1963	if (ASIC_IS_AVIVO(rdev))
		1964	return -EINVAL;
		1965	}
		1966	/* Must be an ATOMBIOS */
		1967	if (!rdev->is_atom_bios) {
		1968	dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
		1969	return -EINVAL;
		1970	}
		1971	r = radeon_atombios_init(rdev);
		1972	if (r)
		1973	return r;
		1974	/* Post card if necessary */
1963	serge	1975	if (!radeon_card_posted(rdev)) {
1321	serge	1976	if (!rdev->bios) {
		1977	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		1978	return -EINVAL;
		1979	}
1221	serge	1980	DRM_INFO("GPU not posted. posting now...\n");
		1981	atom_asic_init(rdev->mode_info.atom_context);
		1982	}
		1983	/* Initialize scratch registers */
		1984	r600_scratch_init(rdev);
		1985	/* Initialize surface registers */
		1986	radeon_surface_init(rdev);
1268	serge	1987	/* Initialize clocks */
1221	serge	1988	radeon_get_clock_info(rdev->ddev);
		1989	/* Fence driver */
2004	serge	1990	r = radeon_fence_driver_init(rdev);
		1991	if (r)
		1992	return r;
1403	serge	1993	if (rdev->flags & RADEON_IS_AGP) {
		1994	r = radeon_agp_init(rdev);
		1995	if (r)
		1996	radeon_agp_disable(rdev);
		1997	}
1221	serge	1998	r = r600_mc_init(rdev);
		1999	if (r)
		2000	return r;
		2001	/* Memory manager */
1321	serge	2002	r = radeon_bo_init(rdev);
1221	serge	2003	if (r)
		2004	return r;
1321	serge	2005
2004	serge	2006	r = radeon_irq_kms_init(rdev);
		2007	if (r)
		2008	return r;
1321	serge	2009
1413	serge	2010	rdev->cp.ring_obj = NULL;
		2011	r600_ring_init(rdev, 1024 * 1024);
1221	serge	2012
2004	serge	2013	rdev->ih.ring_obj = NULL;
		2014	r600_ih_ring_init(rdev, 64 * 1024);
1221	serge	2015
		2016	r = r600_pcie_gart_init(rdev);
		2017	if (r)
		2018	return r;
		2019
1321	serge	2020	rdev->accel_working = true;
1221	serge	2021	r = r600_startup(rdev);
		2022	if (r) {
1428	serge	2023	dev_err(rdev->dev, "disabling GPU acceleration\n");
1221	serge	2024	// r600_suspend(rdev);
		2025	// r600_wb_fini(rdev);
		2026	// radeon_ring_fini(rdev);
		2027	r600_pcie_gart_fini(rdev);
		2028	rdev->accel_working = false;
		2029	}
		2030	if (rdev->accel_working) {
		2031	// r = radeon_ib_pool_init(rdev);
		2032	// if (r) {
		2033	// DRM_ERROR("radeon: failled initializing IB pool (%d).\n", r);
		2034	// rdev->accel_working = false;
		2035	// }
		2036	// r = r600_ib_test(rdev);
		2037	// if (r) {
		2038	// DRM_ERROR("radeon: failled testing IB (%d).\n", r);
		2039	// rdev->accel_working = false;
		2040	// }
		2041	}
2004	serge	2042	if (r)
		2043	return r; /* TODO error handling */
1221	serge	2044	return 0;
		2045	}
		2046
2004	serge	2047	/*
		2048	* CS stuff
		2049	*/
		2050	void r600_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		2051	{
		2052	/* FIXME: implement */
		2053	radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		2054	radeon_ring_write(rdev,
		2055	#ifdef __BIG_ENDIAN
		2056	(2 << 0) \|
		2057	#endif
		2058	(ib->gpu_addr & 0xFFFFFFFC));
		2059	radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
		2060	radeon_ring_write(rdev, ib->length_dw);
		2061	}
		2062
		2063	int r600_ib_test(struct radeon_device *rdev)
		2064	{
		2065	struct radeon_ib *ib;
		2066	uint32_t scratch;
		2067	uint32_t tmp = 0;
		2068	unsigned i;
		2069	int r;
		2070
		2071	r = radeon_scratch_get(rdev, &scratch);
		2072	if (r) {
		2073	DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
		2074	return r;
		2075	}
		2076	WREG32(scratch, 0xCAFEDEAD);
		2077	r = radeon_ib_get(rdev, &ib);
		2078	if (r) {
		2079	DRM_ERROR("radeon: failed to get ib (%d).\n", r);
		2080	return r;
		2081	}
		2082	ib->ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
		2083	ib->ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		2084	ib->ptr[2] = 0xDEADBEEF;
		2085	ib->ptr[3] = PACKET2(0);
		2086	ib->ptr[4] = PACKET2(0);
		2087	ib->ptr[5] = PACKET2(0);
		2088	ib->ptr[6] = PACKET2(0);
		2089	ib->ptr[7] = PACKET2(0);
		2090	ib->ptr[8] = PACKET2(0);
		2091	ib->ptr[9] = PACKET2(0);
		2092	ib->ptr[10] = PACKET2(0);
		2093	ib->ptr[11] = PACKET2(0);
		2094	ib->ptr[12] = PACKET2(0);
		2095	ib->ptr[13] = PACKET2(0);
		2096	ib->ptr[14] = PACKET2(0);
		2097	ib->ptr[15] = PACKET2(0);
		2098	ib->length_dw = 16;
		2099	r = radeon_ib_schedule(rdev, ib);
		2100	if (r) {
		2101	radeon_scratch_free(rdev, scratch);
		2102	radeon_ib_free(rdev, &ib);
		2103	DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
		2104	return r;
		2105	}
		2106	r = radeon_fence_wait(ib->fence, false);
		2107	if (r) {
		2108	DRM_ERROR("radeon: fence wait failed (%d).\n", r);
		2109	return r;
		2110	}
		2111	for (i = 0; i < rdev->usec_timeout; i++) {
		2112	tmp = RREG32(scratch);
		2113	if (tmp == 0xDEADBEEF)
		2114	break;
		2115	DRM_UDELAY(1);
		2116	}
		2117	if (i < rdev->usec_timeout) {
		2118	DRM_INFO("ib test succeeded in %u usecs\n", i);
		2119	} else {
		2120	DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
		2121	scratch, tmp);
		2122	r = -EINVAL;
		2123	}
		2124	radeon_scratch_free(rdev, scratch);
		2125	radeon_ib_free(rdev, &ib);
		2126	return r;
		2127	}
		2128
		2129	/*
		2130	* Interrupts
		2131	*
		2132	* Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
		2133	* the same as the CP ring buffer, but in reverse. Rather than the CPU
		2134	* writing to the ring and the GPU consuming, the GPU writes to the ring
		2135	* and host consumes. As the host irq handler processes interrupts, it
		2136	* increments the rptr. When the rptr catches up with the wptr, all the
		2137	* current interrupts have been processed.
		2138	*/
		2139
		2140	void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
		2141	{
		2142	u32 rb_bufsz;
		2143
		2144	/* Align ring size */
		2145	rb_bufsz = drm_order(ring_size / 4);
		2146	ring_size = (1 << rb_bufsz) * 4;
		2147	rdev->ih.ring_size = ring_size;
		2148	rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
		2149	rdev->ih.rptr = 0;
		2150	}
		2151
		2152	static int r600_ih_ring_alloc(struct radeon_device *rdev)
		2153	{
		2154	int r;
		2155
		2156	/* Allocate ring buffer */
		2157	if (rdev->ih.ring_obj == NULL) {
		2158	r = radeon_bo_create(rdev, rdev->ih.ring_size,
		2159	PAGE_SIZE, true,
		2160	RADEON_GEM_DOMAIN_GTT,
		2161	&rdev->ih.ring_obj);
		2162	if (r) {
		2163	DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
		2164	return r;
		2165	}
		2166	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		2167	if (unlikely(r != 0))
		2168	return r;
		2169	r = radeon_bo_pin(rdev->ih.ring_obj,
		2170	RADEON_GEM_DOMAIN_GTT,
		2171	&rdev->ih.gpu_addr);
		2172	if (r) {
		2173	radeon_bo_unreserve(rdev->ih.ring_obj);
		2174	DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
		2175	return r;
		2176	}
		2177	r = radeon_bo_kmap(rdev->ih.ring_obj,
		2178	(void **)&rdev->ih.ring);
		2179	radeon_bo_unreserve(rdev->ih.ring_obj);
		2180	if (r) {
		2181	DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
		2182	return r;
		2183	}
		2184	}
		2185	return 0;
		2186	}
		2187
		2188	static void r600_ih_ring_fini(struct radeon_device *rdev)
		2189	{
		2190	int r;
		2191	if (rdev->ih.ring_obj) {
		2192	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		2193	if (likely(r == 0)) {
		2194	radeon_bo_kunmap(rdev->ih.ring_obj);
		2195	radeon_bo_unpin(rdev->ih.ring_obj);
		2196	radeon_bo_unreserve(rdev->ih.ring_obj);
		2197	}
		2198	radeon_bo_unref(&rdev->ih.ring_obj);
		2199	rdev->ih.ring = NULL;
		2200	rdev->ih.ring_obj = NULL;
		2201	}
		2202	}
		2203
		2204	void r600_rlc_stop(struct radeon_device *rdev)
		2205	{
		2206
		2207	if ((rdev->family >= CHIP_RV770) &&
		2208	(rdev->family <= CHIP_RV740)) {
		2209	/* r7xx asics need to soft reset RLC before halting */
		2210	WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
		2211	RREG32(SRBM_SOFT_RESET);
		2212	udelay(15000);
		2213	WREG32(SRBM_SOFT_RESET, 0);
		2214	RREG32(SRBM_SOFT_RESET);
		2215	}
		2216
		2217	WREG32(RLC_CNTL, 0);
		2218	}
		2219
		2220	static void r600_rlc_start(struct radeon_device *rdev)
		2221	{
		2222	WREG32(RLC_CNTL, RLC_ENABLE);
		2223	}
		2224
		2225	static int r600_rlc_init(struct radeon_device *rdev)
		2226	{
		2227	u32 i;
		2228	const __be32 *fw_data;
		2229
		2230	if (!rdev->rlc_fw)
		2231	return -EINVAL;
		2232
		2233	r600_rlc_stop(rdev);
		2234
		2235	WREG32(RLC_HB_BASE, 0);
		2236	WREG32(RLC_HB_CNTL, 0);
		2237	WREG32(RLC_HB_RPTR, 0);
		2238	WREG32(RLC_HB_WPTR, 0);
		2239	if (rdev->family <= CHIP_CAICOS) {
		2240	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
		2241	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
		2242	}
		2243	WREG32(RLC_MC_CNTL, 0);
		2244	WREG32(RLC_UCODE_CNTL, 0);
		2245
		2246	fw_data = (const __be32 *)rdev->rlc_fw->data;
		2247	if (rdev->family >= CHIP_CAYMAN) {
		2248	for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
		2249	WREG32(RLC_UCODE_ADDR, i);
		2250	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2251	}
		2252	} else if (rdev->family >= CHIP_CEDAR) {
		2253	for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
		2254	WREG32(RLC_UCODE_ADDR, i);
		2255	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2256	}
		2257	} else if (rdev->family >= CHIP_RV770) {
		2258	for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
		2259	WREG32(RLC_UCODE_ADDR, i);
		2260	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2261	}
		2262	} else {
		2263	for (i = 0; i < RLC_UCODE_SIZE; i++) {
		2264	WREG32(RLC_UCODE_ADDR, i);
		2265	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2266	}
		2267	}
		2268	WREG32(RLC_UCODE_ADDR, 0);
		2269
		2270	r600_rlc_start(rdev);
		2271
		2272	return 0;
		2273	}
		2274
		2275	static void r600_enable_interrupts(struct radeon_device *rdev)
		2276	{
		2277	u32 ih_cntl = RREG32(IH_CNTL);
		2278	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		2279
		2280	ih_cntl \|= ENABLE_INTR;
		2281	ih_rb_cntl \|= IH_RB_ENABLE;
		2282	WREG32(IH_CNTL, ih_cntl);
		2283	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2284	rdev->ih.enabled = true;
		2285	}
		2286
		2287	void r600_disable_interrupts(struct radeon_device *rdev)
		2288	{
		2289	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		2290	u32 ih_cntl = RREG32(IH_CNTL);
		2291
		2292	ih_rb_cntl &= ~IH_RB_ENABLE;
		2293	ih_cntl &= ~ENABLE_INTR;
		2294	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2295	WREG32(IH_CNTL, ih_cntl);
		2296	/* set rptr, wptr to 0 */
		2297	WREG32(IH_RB_RPTR, 0);
		2298	WREG32(IH_RB_WPTR, 0);
		2299	rdev->ih.enabled = false;
		2300	rdev->ih.wptr = 0;
		2301	rdev->ih.rptr = 0;
		2302	}
		2303
1963	serge	2304	static void r600_disable_interrupt_state(struct radeon_device *rdev)
		2305	{
		2306	u32 tmp;
1221	serge	2307
1963	serge	2308	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		2309	WREG32(GRBM_INT_CNTL, 0);
		2310	WREG32(DxMODE_INT_MASK, 0);
		2311	WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
		2312	WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
		2313	if (ASIC_IS_DCE3(rdev)) {
		2314	WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
		2315	WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
		2316	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2317	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2318	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2319	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2320	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2321	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2322	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2323	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2324	if (ASIC_IS_DCE32(rdev)) {
		2325	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2326	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2327	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2328	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2329	}
		2330	} else {
		2331	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		2332	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2333	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2334	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		2335	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2336	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		2337	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2338	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		2339	}
		2340	}
1221	serge	2341
2004	serge	2342	int r600_irq_init(struct radeon_device *rdev)
		2343	{
		2344	int ret = 0;
		2345	int rb_bufsz;
		2346	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
1221	serge	2347
2004	serge	2348	/* allocate ring */
		2349	ret = r600_ih_ring_alloc(rdev);
		2350	if (ret)
		2351	return ret;
1221	serge	2352
2004	serge	2353	/* disable irqs */
		2354	r600_disable_interrupts(rdev);
1221	serge	2355
2004	serge	2356	/* init rlc */
		2357	ret = r600_rlc_init(rdev);
		2358	if (ret) {
		2359	r600_ih_ring_fini(rdev);
		2360	return ret;
		2361	}
1221	serge	2362
2004	serge	2363	/* setup interrupt control */
		2364	/* set dummy read address to ring address */
		2365	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
		2366	interrupt_cntl = RREG32(INTERRUPT_CNTL);
		2367	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
		2368	* IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
		2369	*/
		2370	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
		2371	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
		2372	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
		2373	WREG32(INTERRUPT_CNTL, interrupt_cntl);
1221	serge	2374
2004	serge	2375	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
		2376	rb_bufsz = drm_order(rdev->ih.ring_size / 4);
1221	serge	2377
2004	serge	2378	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE \|
		2379	IH_WPTR_OVERFLOW_CLEAR \|
		2380	(rb_bufsz << 1));
1963	serge	2381
2004	serge	2382	if (rdev->wb.enabled)
		2383	ih_rb_cntl \|= IH_WPTR_WRITEBACK_ENABLE;
		2384
		2385	/* set the writeback address whether it's enabled or not */
		2386	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
		2387	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
		2388
		2389	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2390
		2391	/* set rptr, wptr to 0 */
		2392	WREG32(IH_RB_RPTR, 0);
		2393	WREG32(IH_RB_WPTR, 0);
		2394
		2395	/* Default settings for IH_CNTL (disabled at first) */
		2396	ih_cntl = MC_WRREQ_CREDIT(0x10) \| MC_WR_CLEAN_CNT(0x10);
		2397	/* RPTR_REARM only works if msi's are enabled */
		2398	if (rdev->msi_enabled)
		2399	ih_cntl \|= RPTR_REARM;
		2400
		2401	#ifdef __BIG_ENDIAN
		2402	ih_cntl \|= IH_MC_SWAP(IH_MC_SWAP_32BIT);
		2403	#endif
		2404	WREG32(IH_CNTL, ih_cntl);
		2405
		2406	/* force the active interrupt state to all disabled */
		2407	if (rdev->family >= CHIP_CEDAR)
		2408	evergreen_disable_interrupt_state(rdev);
		2409	else
		2410	r600_disable_interrupt_state(rdev);
		2411
		2412	/* enable irqs */
		2413	r600_enable_interrupts(rdev);
		2414
		2415	return ret;
		2416	}
		2417	int r600_irq_set(struct radeon_device *rdev)
		2418	{
		2419	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
		2420	u32 mode_int = 0;
		2421	u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
		2422	u32 grbm_int_cntl = 0;
		2423	u32 hdmi1, hdmi2;
		2424	u32 d1grph = 0, d2grph = 0;
		2425
		2426	ENTER();
		2427
		2428	if (!rdev->irq.installed) {
		2429	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		2430	return -EINVAL;
		2431	}
		2432	/* don't enable anything if the ih is disabled */
		2433	if (!rdev->ih.enabled) {
		2434	r600_disable_interrupts(rdev);
		2435	/* force the active interrupt state to all disabled */
		2436	r600_disable_interrupt_state(rdev);
		2437	return 0;
		2438	}
		2439
		2440	hdmi1 = RREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
		2441	if (ASIC_IS_DCE3(rdev)) {
		2442	hdmi2 = RREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
		2443	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2444	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2445	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2446	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2447	if (ASIC_IS_DCE32(rdev)) {
		2448	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2449	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2450	}
		2451	} else {
		2452	hdmi2 = RREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
		2453	hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2454	hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2455	hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2456	}
		2457
		2458	if (rdev->irq.sw_int) {
		2459	DRM_DEBUG("r600_irq_set: sw int\n");
		2460	cp_int_cntl \|= RB_INT_ENABLE;
		2461	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		2462	}
		2463	if (rdev->irq.crtc_vblank_int[0] \|\|
		2464	rdev->irq.pflip[0]) {
		2465	DRM_DEBUG("r600_irq_set: vblank 0\n");
		2466	mode_int \|= D1MODE_VBLANK_INT_MASK;
		2467	}
		2468	if (rdev->irq.crtc_vblank_int[1] \|\|
		2469	rdev->irq.pflip[1]) {
		2470	DRM_DEBUG("r600_irq_set: vblank 1\n");
		2471	mode_int \|= D2MODE_VBLANK_INT_MASK;
		2472	}
		2473	if (rdev->irq.hpd[0]) {
		2474	DRM_DEBUG("r600_irq_set: hpd 1\n");
		2475	hpd1 \|= DC_HPDx_INT_EN;
		2476	}
		2477	if (rdev->irq.hpd[1]) {
		2478	DRM_DEBUG("r600_irq_set: hpd 2\n");
		2479	hpd2 \|= DC_HPDx_INT_EN;
		2480	}
		2481	if (rdev->irq.hpd[2]) {
		2482	DRM_DEBUG("r600_irq_set: hpd 3\n");
		2483	hpd3 \|= DC_HPDx_INT_EN;
		2484	}
		2485	if (rdev->irq.hpd[3]) {
		2486	DRM_DEBUG("r600_irq_set: hpd 4\n");
		2487	hpd4 \|= DC_HPDx_INT_EN;
		2488	}
		2489	if (rdev->irq.hpd[4]) {
		2490	DRM_DEBUG("r600_irq_set: hpd 5\n");
		2491	hpd5 \|= DC_HPDx_INT_EN;
		2492	}
		2493	if (rdev->irq.hpd[5]) {
		2494	DRM_DEBUG("r600_irq_set: hpd 6\n");
		2495	hpd6 \|= DC_HPDx_INT_EN;
		2496	}
		2497	if (rdev->irq.hdmi[0]) {
		2498	DRM_DEBUG("r600_irq_set: hdmi 1\n");
		2499	hdmi1 \|= R600_HDMI_INT_EN;
		2500	}
		2501	if (rdev->irq.hdmi[1]) {
		2502	DRM_DEBUG("r600_irq_set: hdmi 2\n");
		2503	hdmi2 \|= R600_HDMI_INT_EN;
		2504	}
		2505	if (rdev->irq.gui_idle) {
		2506	DRM_DEBUG("gui idle\n");
		2507	grbm_int_cntl \|= GUI_IDLE_INT_ENABLE;
		2508	}
		2509
		2510	WREG32(CP_INT_CNTL, cp_int_cntl);
		2511	WREG32(DxMODE_INT_MASK, mode_int);
		2512	WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
		2513	WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
		2514	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		2515	WREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, hdmi1);
		2516	if (ASIC_IS_DCE3(rdev)) {
		2517	WREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, hdmi2);
		2518	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		2519	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		2520	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		2521	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		2522	if (ASIC_IS_DCE32(rdev)) {
		2523	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		2524	WREG32(DC_HPD6_INT_CONTROL, hpd6);
		2525	}
		2526	} else {
		2527	WREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, hdmi2);
		2528	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
		2529	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
		2530	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
		2531	}
		2532
		2533	LEAVE();
		2534
		2535	return 0;
		2536	}
		2537
		2538	static inline void r600_irq_ack(struct radeon_device *rdev)
		2539	{
		2540	u32 tmp;
		2541
		2542	if (ASIC_IS_DCE3(rdev)) {
		2543	rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
		2544	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
		2545	rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
		2546	} else {
		2547	rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		2548	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		2549	rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
		2550	}
		2551	rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
		2552	rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
		2553
		2554	if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		2555	WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		2556	if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		2557	WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		2558	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
		2559	WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		2560	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
		2561	WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		2562	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
		2563	WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		2564	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
		2565	WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		2566	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		2567	if (ASIC_IS_DCE3(rdev)) {
		2568	tmp = RREG32(DC_HPD1_INT_CONTROL);
		2569	tmp \|= DC_HPDx_INT_ACK;
		2570	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2571	} else {
		2572	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		2573	tmp \|= DC_HPDx_INT_ACK;
		2574	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		2575	}
		2576	}
		2577	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		2578	if (ASIC_IS_DCE3(rdev)) {
		2579	tmp = RREG32(DC_HPD2_INT_CONTROL);
		2580	tmp \|= DC_HPDx_INT_ACK;
		2581	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2582	} else {
		2583	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		2584	tmp \|= DC_HPDx_INT_ACK;
		2585	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		2586	}
		2587	}
		2588	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		2589	if (ASIC_IS_DCE3(rdev)) {
		2590	tmp = RREG32(DC_HPD3_INT_CONTROL);
		2591	tmp \|= DC_HPDx_INT_ACK;
		2592	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2593	} else {
		2594	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		2595	tmp \|= DC_HPDx_INT_ACK;
		2596	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		2597	}
		2598	}
		2599	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		2600	tmp = RREG32(DC_HPD4_INT_CONTROL);
		2601	tmp \|= DC_HPDx_INT_ACK;
		2602	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2603	}
		2604	if (ASIC_IS_DCE32(rdev)) {
		2605	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		2606	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2607	tmp \|= DC_HPDx_INT_ACK;
		2608	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2609	}
		2610	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		2611	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2612	tmp \|= DC_HPDx_INT_ACK;
		2613	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2614	}
		2615	}
		2616	if (RREG32(R600_HDMI_BLOCK1 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
		2617	WREG32_P(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
		2618	}
		2619	if (ASIC_IS_DCE3(rdev)) {
		2620	if (RREG32(R600_HDMI_BLOCK3 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
		2621	WREG32_P(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
		2622	}
		2623	} else {
		2624	if (RREG32(R600_HDMI_BLOCK2 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
		2625	WREG32_P(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
		2626	}
		2627	}
		2628	}
		2629
		2630	static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
		2631	{
		2632	u32 wptr, tmp;
		2633
		2634	if (rdev->wb.enabled)
		2635	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		2636	else
		2637	wptr = RREG32(IH_RB_WPTR);
		2638
		2639	if (wptr & RB_OVERFLOW) {
		2640	/* When a ring buffer overflow happen start parsing interrupt
		2641	* from the last not overwritten vector (wptr + 16). Hopefully
		2642	* this should allow us to catchup.
		2643	*/
		2644	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
		2645	wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
		2646	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		2647	tmp = RREG32(IH_RB_CNTL);
		2648	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		2649	WREG32(IH_RB_CNTL, tmp);
		2650	}
		2651	return (wptr & rdev->ih.ptr_mask);
		2652	}
		2653
		2654	/* r600 IV Ring
		2655	* Each IV ring entry is 128 bits:
		2656	* [7:0] - interrupt source id
		2657	* [31:8] - reserved
		2658	* [59:32] - interrupt source data
		2659	* [127:60] - reserved
		2660	*
		2661	* The basic interrupt vector entries
		2662	* are decoded as follows:
		2663	* src_id src_data description
		2664	* 1 0 D1 Vblank
		2665	* 1 1 D1 Vline
		2666	* 5 0 D2 Vblank
		2667	* 5 1 D2 Vline
		2668	* 19 0 FP Hot plug detection A
		2669	* 19 1 FP Hot plug detection B
		2670	* 19 2 DAC A auto-detection
		2671	* 19 3 DAC B auto-detection
		2672	* 21 4 HDMI block A
		2673	* 21 5 HDMI block B
		2674	* 176 - CP_INT RB
		2675	* 177 - CP_INT IB1
		2676	* 178 - CP_INT IB2
		2677	* 181 - EOP Interrupt
		2678	* 233 - GUI Idle
		2679	*
		2680	* Note, these are based on r600 and may need to be
		2681	* adjusted or added to on newer asics
		2682	*/
		2683
		2684	int r600_irq_process(struct radeon_device *rdev)
		2685	{
		2686	u32 wptr;
		2687	u32 rptr;
		2688	u32 src_id, src_data;
		2689	u32 ring_index;
		2690	unsigned long flags;
		2691	bool queue_hotplug = false;
		2692
		2693	if (!rdev->ih.enabled \|\| rdev->shutdown)
		2694	return IRQ_NONE;
		2695
		2696	wptr = r600_get_ih_wptr(rdev);
		2697	rptr = rdev->ih.rptr;
		2698	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		2699
		2700	spin_lock_irqsave(&rdev->ih.lock, flags);
		2701
		2702	if (rptr == wptr) {
		2703	spin_unlock_irqrestore(&rdev->ih.lock, flags);
		2704	return IRQ_NONE;
		2705	}
		2706
		2707	restart_ih:
		2708	/* display interrupts */
		2709	r600_irq_ack(rdev);
		2710
		2711	rdev->ih.wptr = wptr;
		2712	while (rptr != wptr) {
		2713	/* wptr/rptr are in bytes! */
		2714	ring_index = rptr / 4;
		2715	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		2716	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		2717
		2718	switch (src_id) {
		2719	case 1: /* D1 vblank/vline */
		2720	switch (src_data) {
		2721	case 0: /* D1 vblank */
		2722	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
		2723	if (rdev->irq.crtc_vblank_int[0]) {
		2724	// drm_handle_vblank(rdev->ddev, 0);
		2725	rdev->pm.vblank_sync = true;
		2726	// wake_up(&rdev->irq.vblank_queue);
		2727	}
		2728	// if (rdev->irq.pflip[0])
		2729	// radeon_crtc_handle_flip(rdev, 0);
		2730	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		2731	DRM_DEBUG("IH: D1 vblank\n");
		2732	}
		2733	break;
		2734	case 1: /* D1 vline */
		2735	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
		2736	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		2737	DRM_DEBUG("IH: D1 vline\n");
		2738	}
		2739	break;
		2740	default:
		2741	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2742	break;
		2743	}
		2744	break;
		2745	case 5: /* D2 vblank/vline */
		2746	switch (src_data) {
		2747	case 0: /* D2 vblank */
		2748	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
		2749	if (rdev->irq.crtc_vblank_int[1]) {
		2750	// drm_handle_vblank(rdev->ddev, 1);
		2751	rdev->pm.vblank_sync = true;
		2752	// wake_up(&rdev->irq.vblank_queue);
		2753	}
		2754	// if (rdev->irq.pflip[1])
		2755	// radeon_crtc_handle_flip(rdev, 1);
		2756	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
		2757	DRM_DEBUG("IH: D2 vblank\n");
		2758	}
		2759	break;
		2760	case 1: /* D1 vline */
		2761	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
		2762	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
		2763	DRM_DEBUG("IH: D2 vline\n");
		2764	}
		2765	break;
		2766	default:
		2767	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2768	break;
		2769	}
		2770	break;
		2771	case 19: /* HPD/DAC hotplug */
		2772	switch (src_data) {
		2773	case 0:
		2774	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		2775	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
		2776	queue_hotplug = true;
		2777	DRM_DEBUG("IH: HPD1\n");
		2778	}
		2779	break;
		2780	case 1:
		2781	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		2782	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
		2783	queue_hotplug = true;
		2784	DRM_DEBUG("IH: HPD2\n");
		2785	}
		2786	break;
		2787	case 4:
		2788	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		2789	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
		2790	queue_hotplug = true;
		2791	DRM_DEBUG("IH: HPD3\n");
		2792	}
		2793	break;
		2794	case 5:
		2795	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		2796	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
		2797	queue_hotplug = true;
		2798	DRM_DEBUG("IH: HPD4\n");
		2799	}
		2800	break;
		2801	case 10:
		2802	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		2803	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
		2804	queue_hotplug = true;
		2805	DRM_DEBUG("IH: HPD5\n");
		2806	}
		2807	break;
		2808	case 12:
		2809	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		2810	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
		2811	queue_hotplug = true;
		2812	DRM_DEBUG("IH: HPD6\n");
		2813	}
		2814	break;
		2815	default:
		2816	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2817	break;
		2818	}
		2819	break;
		2820	case 21: /* HDMI */
		2821	DRM_DEBUG("IH: HDMI: 0x%x\n", src_data);
		2822	// r600_audio_schedule_polling(rdev);
		2823	break;
		2824	case 176: /* CP_INT in ring buffer */
		2825	case 177: /* CP_INT in IB1 */
		2826	case 178: /* CP_INT in IB2 */
		2827	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
		2828	// radeon_fence_process(rdev);
		2829	break;
		2830	case 181: /* CP EOP event */
		2831	DRM_DEBUG("IH: CP EOP\n");
		2832	// radeon_fence_process(rdev);
		2833	break;
		2834	case 233: /* GUI IDLE */
		2835	DRM_DEBUG("IH: GUI idle\n");
		2836	rdev->pm.gui_idle = true;
		2837	// wake_up(&rdev->irq.idle_queue);
		2838	break;
		2839	default:
		2840	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2841	break;
		2842	}
		2843
		2844	/* wptr/rptr are in bytes! */
		2845	rptr += 16;
		2846	rptr &= rdev->ih.ptr_mask;
		2847	}
		2848	/* make sure wptr hasn't changed while processing */
		2849	wptr = r600_get_ih_wptr(rdev);
		2850	if (wptr != rdev->ih.wptr)
		2851	goto restart_ih;
		2852	// if (queue_hotplug)
		2853	// schedule_work(&rdev->hotplug_work);
		2854	rdev->ih.rptr = rptr;
		2855	WREG32(IH_RB_RPTR, rdev->ih.rptr);
		2856	spin_unlock_irqrestore(&rdev->ih.lock, flags);
		2857	return IRQ_HANDLED;
		2858	}
		2859
1221	serge	2860	/*
		2861	* Debugfs info
		2862	*/
		2863	#if defined(CONFIG_DEBUG_FS)
		2864
		2865	static int r600_debugfs_cp_ring_info(struct seq_file m, void data)
		2866	{
		2867	struct drm_info_node node = (struct drm_info_node ) m->private;
		2868	struct drm_device *dev = node->minor->dev;
		2869	struct radeon_device *rdev = dev->dev_private;
		2870	unsigned count, i, j;
		2871
		2872	radeon_ring_free_size(rdev);
1321	serge	2873	count = (rdev->cp.ring_size / 4) - rdev->cp.ring_free_dw;
1221	serge	2874	seq_printf(m, "CP_STAT 0x%08x\n", RREG32(CP_STAT));
1321	serge	2875	seq_printf(m, "CP_RB_WPTR 0x%08x\n", RREG32(CP_RB_WPTR));
		2876	seq_printf(m, "CP_RB_RPTR 0x%08x\n", RREG32(CP_RB_RPTR));
		2877	seq_printf(m, "driver's copy of the CP_RB_WPTR 0x%08x\n", rdev->cp.wptr);
		2878	seq_printf(m, "driver's copy of the CP_RB_RPTR 0x%08x\n", rdev->cp.rptr);
1221	serge	2879	seq_printf(m, "%u free dwords in ring\n", rdev->cp.ring_free_dw);
		2880	seq_printf(m, "%u dwords in ring\n", count);
1321	serge	2881	i = rdev->cp.rptr;
1221	serge	2882	for (j = 0; j <= count; j++) {
		2883	seq_printf(m, "r[%04d]=0x%08x\n", i, rdev->cp.ring[i]);
1321	serge	2884	i = (i + 1) & rdev->cp.ptr_mask;
1221	serge	2885	}
		2886	return 0;
		2887	}
		2888
		2889	static int r600_debugfs_mc_info(struct seq_file m, void data)
		2890	{
		2891	struct drm_info_node node = (struct drm_info_node ) m->private;
		2892	struct drm_device *dev = node->minor->dev;
		2893	struct radeon_device *rdev = dev->dev_private;
		2894
		2895	DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
		2896	DREG32_SYS(m, rdev, VM_L2_STATUS);
		2897	return 0;
		2898	}
		2899
		2900	static struct drm_info_list r600_mc_info_list[] = {
		2901	{"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
		2902	{"r600_ring_info", r600_debugfs_cp_ring_info, 0, NULL},
		2903	};
		2904	#endif
		2905
		2906	int r600_debugfs_mc_info_init(struct radeon_device *rdev)
		2907	{
		2908	#if defined(CONFIG_DEBUG_FS)
		2909	return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
		2910	#else
		2911	return 0;
		2912	#endif
		2913	}
1404	serge	2914
		2915	/**
		2916	* r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
		2917	* rdev: radeon device structure
		2918	* bo: buffer object struct which userspace is waiting for idle
		2919	*
		2920	* Some R6XX/R7XX doesn't seems to take into account HDP flush performed
		2921	* through ring buffer, this leads to corruption in rendering, see
		2922	* http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
		2923	* directly perform HDP flush by writing register through MMIO.
		2924	*/
		2925	void r600_ioctl_wait_idle(struct radeon_device rdev, struct radeon_bo bo)
		2926	{
1963	serge	2927	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		2928	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
		2929	* This seems to cause problems on some AGP cards. Just use the old
		2930	* method for them.
		2931	*/
		2932	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		2933	rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
		2934	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		2935	u32 tmp;
		2936
		2937	WREG32(HDP_DEBUG1, 0);
		2938	tmp = readl((void __iomem *)ptr);
		2939	} else
1404	serge	2940	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		2941	}
1963	serge	2942
		2943	void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
		2944	{
		2945	u32 link_width_cntl, mask, target_reg;
		2946
		2947	if (rdev->flags & RADEON_IS_IGP)
		2948	return;
		2949
		2950	if (!(rdev->flags & RADEON_IS_PCIE))
		2951	return;
		2952
		2953	/* x2 cards have a special sequence */
		2954	if (ASIC_IS_X2(rdev))
		2955	return;
		2956
		2957	/* FIXME wait for idle */
		2958
		2959	switch (lanes) {
		2960	case 0:
		2961	mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
		2962	break;
		2963	case 1:
		2964	mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
		2965	break;
		2966	case 2:
		2967	mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
		2968	break;
		2969	case 4:
		2970	mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
		2971	break;
		2972	case 8:
		2973	mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
		2974	break;
		2975	case 12:
		2976	mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
		2977	break;
		2978	case 16:
		2979	default:
		2980	mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
		2981	break;
		2982	}
		2983
		2984	link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		2985
		2986	if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
		2987	(mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
		2988	return;
		2989
		2990	if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
		2991	return;
		2992
		2993	link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK \|
		2994	RADEON_PCIE_LC_RECONFIG_NOW \|
		2995	R600_PCIE_LC_RENEGOTIATE_EN \|
		2996	R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
		2997	link_width_cntl \|= mask;
		2998
		2999	WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3000
		3001	/* some northbridges can renegotiate the link rather than requiring
		3002	* a complete re-config.
		3003	* e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)
		3004	*/
		3005	if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
		3006	link_width_cntl \|= R600_PCIE_LC_RENEGOTIATE_EN \| R600_PCIE_LC_UPCONFIGURE_SUPPORT;
		3007	else
		3008	link_width_cntl \|= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
		3009
		3010	WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl \|
		3011	RADEON_PCIE_LC_RECONFIG_NOW));
		3012
		3013	if (rdev->family >= CHIP_RV770)
		3014	target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
		3015	else
		3016	target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
		3017
		3018	/* wait for lane set to complete */
		3019	link_width_cntl = RREG32(target_reg);
		3020	while (link_width_cntl == 0xffffffff)
		3021	link_width_cntl = RREG32(target_reg);
		3022
		3023	}
		3024
		3025	int r600_get_pcie_lanes(struct radeon_device *rdev)
		3026	{
		3027	u32 link_width_cntl;
		3028
		3029	if (rdev->flags & RADEON_IS_IGP)
		3030	return 0;
		3031
		3032	if (!(rdev->flags & RADEON_IS_PCIE))
		3033	return 0;
		3034
		3035	/* x2 cards have a special sequence */
		3036	if (ASIC_IS_X2(rdev))
		3037	return 0;
		3038
		3039	/* FIXME wait for idle */
		3040
		3041	link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		3042
		3043	switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
		3044	case RADEON_PCIE_LC_LINK_WIDTH_X0:
		3045	return 0;
		3046	case RADEON_PCIE_LC_LINK_WIDTH_X1:
		3047	return 1;
		3048	case RADEON_PCIE_LC_LINK_WIDTH_X2:
		3049	return 2;
		3050	case RADEON_PCIE_LC_LINK_WIDTH_X4:
		3051	return 4;
		3052	case RADEON_PCIE_LC_LINK_WIDTH_X8:
		3053	return 8;
		3054	case RADEON_PCIE_LC_LINK_WIDTH_X16:
		3055	default:
		3056	return 16;
		3057	}
		3058	}
		3059
		3060	static void r600_pcie_gen2_enable(struct radeon_device *rdev)
		3061	{
		3062	u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
		3063	u16 link_cntl2;
		3064
		3065	if (radeon_pcie_gen2 == 0)
		3066	return;
		3067
		3068	if (rdev->flags & RADEON_IS_IGP)
		3069	return;
		3070
		3071	if (!(rdev->flags & RADEON_IS_PCIE))
		3072	return;
		3073
		3074	/* x2 cards have a special sequence */
		3075	if (ASIC_IS_X2(rdev))
		3076	return;
		3077
		3078	/* only RV6xx+ chips are supported */
		3079	if (rdev->family <= CHIP_R600)
		3080	return;
		3081
		3082	/* 55 nm r6xx asics */
		3083	if ((rdev->family == CHIP_RV670) \|\|
		3084	(rdev->family == CHIP_RV620) \|\|
		3085	(rdev->family == CHIP_RV635)) {
		3086	/* advertise upconfig capability */
		3087	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3088	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3089	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3090	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3091	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
		3092	lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
		3093	link_width_cntl &= ~(LC_LINK_WIDTH_MASK \|
		3094	LC_RECONFIG_ARC_MISSING_ESCAPE);
		3095	link_width_cntl \|= lanes \| LC_RECONFIG_NOW \| LC_RENEGOTIATE_EN;
		3096	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3097	} else {
		3098	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3099	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3100	}
		3101	}
		3102
		3103	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3104	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
		3105	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		3106
		3107	/* 55 nm r6xx asics */
		3108	if ((rdev->family == CHIP_RV670) \|\|
		3109	(rdev->family == CHIP_RV620) \|\|
		3110	(rdev->family == CHIP_RV635)) {
		3111	WREG32(MM_CFGREGS_CNTL, 0x8);
		3112	link_cntl2 = RREG32(0x4088);
		3113	WREG32(MM_CFGREGS_CNTL, 0);
		3114	/* not supported yet */
		3115	if (link_cntl2 & SELECTABLE_DEEMPHASIS)
		3116	return;
		3117	}
		3118
		3119	speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
		3120	speed_cntl \|= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
		3121	speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
		3122	speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
		3123	speed_cntl \|= LC_FORCE_EN_HW_SPEED_CHANGE;
		3124	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3125
		3126	tmp = RREG32(0x541c);
		3127	WREG32(0x541c, tmp \| 0x8);
		3128	WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
		3129	link_cntl2 = RREG16(0x4088);
		3130	link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
		3131	link_cntl2 \|= 0x2;
		3132	WREG16(0x4088, link_cntl2);
		3133	WREG32(MM_CFGREGS_CNTL, 0);
		3134
		3135	if ((rdev->family == CHIP_RV670) \|\|
		3136	(rdev->family == CHIP_RV620) \|\|
		3137	(rdev->family == CHIP_RV635)) {
		3138	training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
		3139	training_cntl &= ~LC_POINT_7_PLUS_EN;
		3140	WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
		3141	} else {
		3142	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3143	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		3144	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3145	}
		3146
		3147	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3148	speed_cntl \|= LC_GEN2_EN_STRAP;
		3149	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3150
		3151	} else {
		3152	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3153	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		3154	if (1)
		3155	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3156	else
		3157	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3158	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3159	}
		3160	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/r600.c – Rev 2004