WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/r600.c

Rev	Author	Line No.	Line
1128	serge	1	/*
		2	* Copyright 2008 Advanced Micro Devices, Inc.
		3	* Copyright 2008 Red Hat Inc.
		4	* Copyright 2009 Jerome Glisse.
		5	*
		6	* Permission is hereby granted, free of charge, to any person obtaining a
		7	* copy of this software and associated documentation files (the "Software"),
		8	* to deal in the Software without restriction, including without limitation
		9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		10	* and/or sell copies of the Software, and to permit persons to whom the
		11	* Software is furnished to do so, subject to the following conditions:
		12	*
		13	* The above copyright notice and this permission notice shall be included in
		14	* all copies or substantial portions of the Software.
		15	*
		16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		22	* OTHER DEALINGS IN THE SOFTWARE.
		23	*
		24	* Authors: Dave Airlie
		25	* Alex Deucher
		26	* Jerome Glisse
		27	*/
1963	serge	28	#include
1233	serge	29	#include
1221	serge	30	#include
1128	serge	31	#include "drmP.h"
1221	serge	32	#include "radeon_drm.h"
1128	serge	33	#include "radeon.h"
1963	serge	34	#include "radeon_asic.h"
1221	serge	35	#include "radeon_mode.h"
		36	#include "r600d.h"
		37	#include "atom.h"
		38	#include "avivod.h"
1128	serge	39
1221	serge	40	#define PFP_UCODE_SIZE 576
		41	#define PM4_UCODE_SIZE 1792
1321	serge	42	#define RLC_UCODE_SIZE 768
1221	serge	43	#define R700_PFP_UCODE_SIZE 848
		44	#define R700_PM4_UCODE_SIZE 1360
1321	serge	45	#define R700_RLC_UCODE_SIZE 1024
1963	serge	46	#define EVERGREEN_PFP_UCODE_SIZE 1120
		47	#define EVERGREEN_PM4_UCODE_SIZE 1376
		48	#define EVERGREEN_RLC_UCODE_SIZE 768
		49	#define CAYMAN_RLC_UCODE_SIZE 1024
1128	serge	50
1221	serge	51	/* Firmware Names */
		52	MODULE_FIRMWARE("radeon/R600_pfp.bin");
		53	MODULE_FIRMWARE("radeon/R600_me.bin");
		54	MODULE_FIRMWARE("radeon/RV610_pfp.bin");
		55	MODULE_FIRMWARE("radeon/RV610_me.bin");
		56	MODULE_FIRMWARE("radeon/RV630_pfp.bin");
		57	MODULE_FIRMWARE("radeon/RV630_me.bin");
		58	MODULE_FIRMWARE("radeon/RV620_pfp.bin");
		59	MODULE_FIRMWARE("radeon/RV620_me.bin");
		60	MODULE_FIRMWARE("radeon/RV635_pfp.bin");
		61	MODULE_FIRMWARE("radeon/RV635_me.bin");
		62	MODULE_FIRMWARE("radeon/RV670_pfp.bin");
		63	MODULE_FIRMWARE("radeon/RV670_me.bin");
		64	MODULE_FIRMWARE("radeon/RS780_pfp.bin");
		65	MODULE_FIRMWARE("radeon/RS780_me.bin");
		66	MODULE_FIRMWARE("radeon/RV770_pfp.bin");
		67	MODULE_FIRMWARE("radeon/RV770_me.bin");
		68	MODULE_FIRMWARE("radeon/RV730_pfp.bin");
		69	MODULE_FIRMWARE("radeon/RV730_me.bin");
		70	MODULE_FIRMWARE("radeon/RV710_pfp.bin");
		71	MODULE_FIRMWARE("radeon/RV710_me.bin");
1321	serge	72	MODULE_FIRMWARE("radeon/R600_rlc.bin");
		73	MODULE_FIRMWARE("radeon/R700_rlc.bin");
1963	serge	74	MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
		75	MODULE_FIRMWARE("radeon/CEDAR_me.bin");
		76	MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
		77	MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
		78	MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
		79	MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
		80	MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
		81	MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
		82	MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
		83	MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
		84	MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
		85	MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
		86	MODULE_FIRMWARE("radeon/PALM_pfp.bin");
		87	MODULE_FIRMWARE("radeon/PALM_me.bin");
		88	MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
		89	MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
		90	MODULE_FIRMWARE("radeon/SUMO_me.bin");
		91	MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
		92	MODULE_FIRMWARE("radeon/SUMO2_me.bin");
1221	serge	93
		94	int r600_debugfs_mc_info_init(struct radeon_device *rdev);
		95
		96	/* r600,rv610,rv630,rv620,rv635,rv670 */
1128	serge	97	int r600_mc_wait_for_idle(struct radeon_device *rdev);
		98	void r600_gpu_init(struct radeon_device *rdev);
1221	serge	99	void r600_fini(struct radeon_device *rdev);
1963	serge	100	void r600_irq_disable(struct radeon_device *rdev);
		101	static void r600_pcie_gen2_enable(struct radeon_device *rdev);
1128	serge	102
1963	serge	103	/* get temperature in millidegrees */
		104	int rv6xx_get_temp(struct radeon_device *rdev)
		105	{
		106	u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
		107	ASIC_T_SHIFT;
		108	int actual_temp = temp & 0xff;
		109
		110	if (temp & 0x100)
		111	actual_temp -= 256;
		112
		113	return actual_temp * 1000;
		114	}
		115
		116
		117
		118
		119
		120
		121	bool r600_gui_idle(struct radeon_device *rdev)
		122	{
		123	if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
		124	return false;
		125	else
		126	return true;
		127	}
		128
1321	serge	129	/* hpd for digital panel detect/disconnect */
		130	bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		131	{
		132	bool connected = false;
		133
		134	if (ASIC_IS_DCE3(rdev)) {
		135	switch (hpd) {
		136	case RADEON_HPD_1:
		137	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		138	connected = true;
		139	break;
		140	case RADEON_HPD_2:
		141	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		142	connected = true;
		143	break;
		144	case RADEON_HPD_3:
		145	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		146	connected = true;
		147	break;
		148	case RADEON_HPD_4:
		149	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		150	connected = true;
		151	break;
		152	/* DCE 3.2 */
		153	case RADEON_HPD_5:
		154	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		155	connected = true;
		156	break;
		157	case RADEON_HPD_6:
		158	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		159	connected = true;
		160	break;
		161	default:
		162	break;
		163	}
		164	} else {
		165	switch (hpd) {
		166	case RADEON_HPD_1:
		167	if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		168	connected = true;
		169	break;
		170	case RADEON_HPD_2:
		171	if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		172	connected = true;
		173	break;
		174	case RADEON_HPD_3:
		175	if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		176	connected = true;
		177	break;
		178	default:
		179	break;
		180	}
		181	}
		182	return connected;
		183	}
		184
		185	void r600_hpd_set_polarity(struct radeon_device *rdev,
		186	enum radeon_hpd_id hpd)
		187	{
		188	u32 tmp;
		189	bool connected = r600_hpd_sense(rdev, hpd);
		190
		191	if (ASIC_IS_DCE3(rdev)) {
		192	switch (hpd) {
		193	case RADEON_HPD_1:
		194	tmp = RREG32(DC_HPD1_INT_CONTROL);
		195	if (connected)
		196	tmp &= ~DC_HPDx_INT_POLARITY;
		197	else
		198	tmp \|= DC_HPDx_INT_POLARITY;
		199	WREG32(DC_HPD1_INT_CONTROL, tmp);
		200	break;
		201	case RADEON_HPD_2:
		202	tmp = RREG32(DC_HPD2_INT_CONTROL);
		203	if (connected)
		204	tmp &= ~DC_HPDx_INT_POLARITY;
		205	else
		206	tmp \|= DC_HPDx_INT_POLARITY;
		207	WREG32(DC_HPD2_INT_CONTROL, tmp);
		208	break;
		209	case RADEON_HPD_3:
		210	tmp = RREG32(DC_HPD3_INT_CONTROL);
		211	if (connected)
		212	tmp &= ~DC_HPDx_INT_POLARITY;
		213	else
		214	tmp \|= DC_HPDx_INT_POLARITY;
		215	WREG32(DC_HPD3_INT_CONTROL, tmp);
		216	break;
		217	case RADEON_HPD_4:
		218	tmp = RREG32(DC_HPD4_INT_CONTROL);
		219	if (connected)
		220	tmp &= ~DC_HPDx_INT_POLARITY;
		221	else
		222	tmp \|= DC_HPDx_INT_POLARITY;
		223	WREG32(DC_HPD4_INT_CONTROL, tmp);
		224	break;
		225	case RADEON_HPD_5:
		226	tmp = RREG32(DC_HPD5_INT_CONTROL);
		227	if (connected)
		228	tmp &= ~DC_HPDx_INT_POLARITY;
		229	else
		230	tmp \|= DC_HPDx_INT_POLARITY;
		231	WREG32(DC_HPD5_INT_CONTROL, tmp);
		232	break;
		233	/* DCE 3.2 */
		234	case RADEON_HPD_6:
		235	tmp = RREG32(DC_HPD6_INT_CONTROL);
		236	if (connected)
		237	tmp &= ~DC_HPDx_INT_POLARITY;
		238	else
		239	tmp \|= DC_HPDx_INT_POLARITY;
		240	WREG32(DC_HPD6_INT_CONTROL, tmp);
		241	break;
		242	default:
		243	break;
		244	}
		245	} else {
		246	switch (hpd) {
		247	case RADEON_HPD_1:
		248	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		249	if (connected)
		250	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		251	else
		252	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		253	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		254	break;
		255	case RADEON_HPD_2:
		256	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		257	if (connected)
		258	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		259	else
		260	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		261	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		262	break;
		263	case RADEON_HPD_3:
		264	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		265	if (connected)
		266	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		267	else
		268	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		269	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		270	break;
		271	default:
		272	break;
		273	}
		274	}
		275	}
		276
		277	void r600_hpd_init(struct radeon_device *rdev)
		278	{
		279	struct drm_device *dev = rdev->ddev;
		280	struct drm_connector *connector;
		281
		282	if (ASIC_IS_DCE3(rdev)) {
		283	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \| DC_HPDx_RX_INT_TIMER(0xfa);
		284	if (ASIC_IS_DCE32(rdev))
		285	tmp \|= DC_HPDx_EN;
		286
		287	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		288	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		289	switch (radeon_connector->hpd.hpd) {
		290	case RADEON_HPD_1:
		291	WREG32(DC_HPD1_CONTROL, tmp);
2004	serge	292	rdev->irq.hpd[0] = true;
1321	serge	293	break;
		294	case RADEON_HPD_2:
		295	WREG32(DC_HPD2_CONTROL, tmp);
2004	serge	296	rdev->irq.hpd[1] = true;
1321	serge	297	break;
		298	case RADEON_HPD_3:
		299	WREG32(DC_HPD3_CONTROL, tmp);
2004	serge	300	rdev->irq.hpd[2] = true;
1321	serge	301	break;
		302	case RADEON_HPD_4:
		303	WREG32(DC_HPD4_CONTROL, tmp);
2004	serge	304	rdev->irq.hpd[3] = true;
1321	serge	305	break;
		306	/* DCE 3.2 */
		307	case RADEON_HPD_5:
		308	WREG32(DC_HPD5_CONTROL, tmp);
2004	serge	309	rdev->irq.hpd[4] = true;
1321	serge	310	break;
		311	case RADEON_HPD_6:
		312	WREG32(DC_HPD6_CONTROL, tmp);
2004	serge	313	rdev->irq.hpd[5] = true;
1321	serge	314	break;
		315	default:
		316	break;
		317	}
		318	}
		319	} else {
		320	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		321	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		322	switch (radeon_connector->hpd.hpd) {
		323	case RADEON_HPD_1:
		324	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
2004	serge	325	rdev->irq.hpd[0] = true;
1321	serge	326	break;
		327	case RADEON_HPD_2:
		328	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
2004	serge	329	rdev->irq.hpd[1] = true;
1321	serge	330	break;
		331	case RADEON_HPD_3:
		332	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
2004	serge	333	rdev->irq.hpd[2] = true;
1321	serge	334	break;
		335	default:
		336	break;
		337	}
		338	}
		339	}
2004	serge	340	if (rdev->irq.installed)
		341	r600_irq_set(rdev);
1321	serge	342	}
		343
		344	void r600_hpd_fini(struct radeon_device *rdev)
		345	{
		346	struct drm_device *dev = rdev->ddev;
		347	struct drm_connector *connector;
		348
		349	if (ASIC_IS_DCE3(rdev)) {
		350	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		351	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		352	switch (radeon_connector->hpd.hpd) {
		353	case RADEON_HPD_1:
		354	WREG32(DC_HPD1_CONTROL, 0);
2004	serge	355	rdev->irq.hpd[0] = false;
1321	serge	356	break;
		357	case RADEON_HPD_2:
		358	WREG32(DC_HPD2_CONTROL, 0);
2004	serge	359	rdev->irq.hpd[1] = false;
1321	serge	360	break;
		361	case RADEON_HPD_3:
		362	WREG32(DC_HPD3_CONTROL, 0);
2004	serge	363	rdev->irq.hpd[2] = false;
1321	serge	364	break;
		365	case RADEON_HPD_4:
		366	WREG32(DC_HPD4_CONTROL, 0);
2004	serge	367	rdev->irq.hpd[3] = false;
1321	serge	368	break;
		369	/* DCE 3.2 */
		370	case RADEON_HPD_5:
		371	WREG32(DC_HPD5_CONTROL, 0);
2004	serge	372	rdev->irq.hpd[4] = false;
1321	serge	373	break;
		374	case RADEON_HPD_6:
		375	WREG32(DC_HPD6_CONTROL, 0);
2004	serge	376	rdev->irq.hpd[5] = false;
1321	serge	377	break;
		378	default:
		379	break;
		380	}
		381	}
		382	} else {
		383	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		384	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		385	switch (radeon_connector->hpd.hpd) {
		386	case RADEON_HPD_1:
		387	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
2004	serge	388	rdev->irq.hpd[0] = false;
1321	serge	389	break;
		390	case RADEON_HPD_2:
		391	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
2004	serge	392	rdev->irq.hpd[1] = false;
1321	serge	393	break;
		394	case RADEON_HPD_3:
		395	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
2004	serge	396	rdev->irq.hpd[2] = false;
1321	serge	397	break;
		398	default:
		399	break;
		400	}
		401	}
		402	}
		403	}
		404
1128	serge	405	/*
1221	serge	406	* R600 PCIE GART
1128	serge	407	*/
1221	serge	408	void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
		409	{
		410	unsigned i;
		411	u32 tmp;
1128	serge	412
1430	serge	413	/* flush hdp cache so updates hit vram */
1963	serge	414	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		415	!(rdev->flags & RADEON_IS_AGP)) {
		416	void __iomem ptr = (void )rdev->gart.table.vram.ptr;
		417	u32 tmp;
		418
		419	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		420	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
		421	* This seems to cause problems on some AGP cards. Just use the old
		422	* method for them.
		423	*/
		424	WREG32(HDP_DEBUG1, 0);
		425	tmp = readl((void __iomem *)ptr);
		426	} else
1430	serge	427	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		428
1221	serge	429	WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
		430	WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
		431	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		432	for (i = 0; i < rdev->usec_timeout; i++) {
		433	/* read MC_STATUS */
		434	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		435	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		436	if (tmp == 2) {
		437	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		438	return;
		439	}
		440	if (tmp) {
		441	return;
		442	}
		443	udelay(1);
1128	serge	444	}
1221	serge	445	}
1128	serge	446
1221	serge	447	int r600_pcie_gart_init(struct radeon_device *rdev)
		448	{
		449	int r;
		450
		451	if (rdev->gart.table.vram.robj) {
1963	serge	452	WARN(1, "R600 PCIE GART already initialized\n");
1221	serge	453	return 0;
		454	}
		455	/* Initialize common gart structure */
		456	r = radeon_gart_init(rdev);
		457	if (r)
		458	return r;
		459	rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
		460	return radeon_gart_table_vram_alloc(rdev);
		461	}
		462
		463	int r600_pcie_gart_enable(struct radeon_device *rdev)
		464	{
		465	u32 tmp;
		466	int r, i;
		467
		468	if (rdev->gart.table.vram.robj == NULL) {
		469	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		470	return -EINVAL;
		471	}
		472	r = radeon_gart_table_vram_pin(rdev);
		473	if (r)
		474	return r;
1430	serge	475	radeon_gart_restore(rdev);
1221	serge	476
		477	/* Setup L2 cache */
		478	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		479	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		480	EFFECTIVE_L2_QUEUE_SIZE(7));
		481	WREG32(VM_L2_CNTL2, 0);
		482	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		483	/* Setup TLB control */
		484	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		485	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		486	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		487	ENABLE_WAIT_L2_QUERY;
		488	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		489	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		490	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		491	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		492	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		493	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		494	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		495	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		496	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		497	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		498	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		499	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		500	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		501	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		502	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		503	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		504	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		505	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		506	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		507	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		508	(u32)(rdev->dummy_page.addr >> 12));
		509	for (i = 1; i < 7; i++)
		510	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		511
		512	r600_pcie_gart_tlb_flush(rdev);
		513	rdev->gart.ready = true;
1128	serge	514	return 0;
		515	}
		516
1221	serge	517	void r600_pcie_gart_disable(struct radeon_device *rdev)
1128	serge	518	{
1221	serge	519	u32 tmp;
1321	serge	520	int i, r;
1221	serge	521
		522	/* Disable all tables */
		523	for (i = 0; i < 7; i++)
		524	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		525
		526	/* Disable L2 cache */
		527	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		528	EFFECTIVE_L2_QUEUE_SIZE(7));
		529	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		530	/* Setup L1 TLB control */
		531	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		532	ENABLE_WAIT_L2_QUERY;
		533	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		534	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		535	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		536	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		537	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		538	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		539	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		540	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		541	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
		542	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
		543	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		544	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		545	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
		546	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		547	if (rdev->gart.table.vram.robj) {
1403	serge	548	r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
		549	if (likely(r == 0)) {
		550	radeon_bo_kunmap(rdev->gart.table.vram.robj);
		551	radeon_bo_unpin(rdev->gart.table.vram.robj);
		552	radeon_bo_unreserve(rdev->gart.table.vram.robj);
		553	}
1221	serge	554	}
1128	serge	555	}
		556
1221	serge	557	void r600_pcie_gart_fini(struct radeon_device *rdev)
		558	{
1963	serge	559	radeon_gart_fini(rdev);
1221	serge	560	r600_pcie_gart_disable(rdev);
		561	radeon_gart_table_vram_free(rdev);
		562	}
1128	serge	563
1221	serge	564	void r600_agp_enable(struct radeon_device *rdev)
1128	serge	565	{
1221	serge	566	u32 tmp;
		567	int i;
		568
		569	/* Setup L2 cache */
		570	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		571	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		572	EFFECTIVE_L2_QUEUE_SIZE(7));
		573	WREG32(VM_L2_CNTL2, 0);
		574	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		575	/* Setup TLB control */
		576	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		577	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		578	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		579	ENABLE_WAIT_L2_QUERY;
		580	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		581	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		582	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		583	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		584	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		585	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		586	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		587	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		588	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		589	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		590	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		591	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		592	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		593	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		594	for (i = 0; i < 7; i++)
		595	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1128	serge	596	}
		597
		598	int r600_mc_wait_for_idle(struct radeon_device *rdev)
		599	{
1221	serge	600	unsigned i;
		601	u32 tmp;
		602
		603	for (i = 0; i < rdev->usec_timeout; i++) {
		604	/* read MC_STATUS */
		605	tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
		606	if (!tmp)
1128	serge	607	return 0;
1221	serge	608	udelay(1);
		609	}
		610	return -1;
1128	serge	611	}
		612
1221	serge	613	static void r600_mc_program(struct radeon_device *rdev)
1128	serge	614	{
1221	serge	615	struct rv515_mc_save save;
		616	u32 tmp;
		617	int i, j;
		618
		619	/* Initialize HDP */
		620	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		621	WREG32((0x2c14 + j), 0x00000000);
		622	WREG32((0x2c18 + j), 0x00000000);
		623	WREG32((0x2c1c + j), 0x00000000);
		624	WREG32((0x2c20 + j), 0x00000000);
		625	WREG32((0x2c24 + j), 0x00000000);
		626	}
		627	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		628
		629	rv515_mc_stop(rdev, &save);
		630	if (r600_mc_wait_for_idle(rdev)) {
		631	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		632	}
		633	/* Lockout access through VGA aperture (doesn't exist before R600) */
		634	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		635	/* Update configuration */
		636	if (rdev->flags & RADEON_IS_AGP) {
		637	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		638	/* VRAM before AGP */
		639	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		640	rdev->mc.vram_start >> 12);
		641	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		642	rdev->mc.gtt_end >> 12);
		643	} else {
		644	/* VRAM after AGP */
		645	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		646	rdev->mc.gtt_start >> 12);
		647	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		648	rdev->mc.vram_end >> 12);
		649	}
		650	} else {
		651	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
		652	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
		653	}
		654	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
		655	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		656	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		657	WREG32(MC_VM_FB_LOCATION, tmp);
		658	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
		659	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1963	serge	660	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1221	serge	661	if (rdev->flags & RADEON_IS_AGP) {
		662	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
		663	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
		664	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		665	} else {
		666	WREG32(MC_VM_AGP_BASE, 0);
		667	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		668	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		669	}
		670	if (r600_mc_wait_for_idle(rdev)) {
		671	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		672	}
		673	rv515_mc_resume(rdev, &save);
		674	/* we need to own VRAM, so turn off the VGA renderer here
		675	* to stop it overwriting our objects */
		676	rv515_vga_render_disable(rdev);
1128	serge	677	}
		678
1430	serge	679	/**
		680	* r600_vram_gtt_location - try to find VRAM & GTT location
		681	* @rdev: radeon device structure holding all necessary informations
		682	* @mc: memory controller structure holding memory informations
		683	*
		684	* Function will place try to place VRAM at same place as in CPU (PCI)
		685	* address space as some GPU seems to have issue when we reprogram at
		686	* different address space.
		687	*
		688	* If there is not enough space to fit the unvisible VRAM after the
		689	* aperture then we limit the VRAM size to the aperture.
		690	*
		691	* If we are using AGP then place VRAM adjacent to AGP aperture are we need
		692	* them to be in one from GPU point of view so that we can program GPU to
		693	* catch access outside them (weird GPU policy see ??).
		694	*
		695	* This function will never fails, worst case are limiting VRAM or GTT.
		696	*
		697	* Note: GTT start, end, size should be initialized before calling this
		698	* function on AGP platform.
		699	*/
1963	serge	700	static void r600_vram_gtt_location(struct radeon_device rdev, struct radeon_mc mc)
1430	serge	701	{
		702	u64 size_bf, size_af;
		703
		704	if (mc->mc_vram_size > 0xE0000000) {
		705	/* leave room for at least 512M GTT */
		706	dev_warn(rdev->dev, "limiting VRAM\n");
		707	mc->real_vram_size = 0xE0000000;
		708	mc->mc_vram_size = 0xE0000000;
		709	}
		710	if (rdev->flags & RADEON_IS_AGP) {
		711	size_bf = mc->gtt_start;
		712	size_af = 0xFFFFFFFF - mc->gtt_end + 1;
		713	if (size_bf > size_af) {
		714	if (mc->mc_vram_size > size_bf) {
		715	dev_warn(rdev->dev, "limiting VRAM\n");
		716	mc->real_vram_size = size_bf;
		717	mc->mc_vram_size = size_bf;
		718	}
		719	mc->vram_start = mc->gtt_start - mc->mc_vram_size;
		720	} else {
		721	if (mc->mc_vram_size > size_af) {
		722	dev_warn(rdev->dev, "limiting VRAM\n");
		723	mc->real_vram_size = size_af;
		724	mc->mc_vram_size = size_af;
		725	}
		726	mc->vram_start = mc->gtt_end;
		727	}
		728	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
		729	dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
		730	mc->mc_vram_size >> 20, mc->vram_start,
		731	mc->vram_end, mc->real_vram_size >> 20);
		732	} else {
		733	u64 base = 0;
1963	serge	734	if (rdev->flags & RADEON_IS_IGP) {
		735	base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
		736	base <<= 24;
		737	}
1430	serge	738	radeon_vram_location(rdev, &rdev->mc, base);
1963	serge	739	rdev->mc.gtt_base_align = 0;
1430	serge	740	radeon_gtt_location(rdev, mc);
		741	}
		742	}
		743
1221	serge	744	int r600_mc_init(struct radeon_device *rdev)
1128	serge	745	{
1221	serge	746	u32 tmp;
1268	serge	747	int chansize, numchan;
1128	serge	748
1221	serge	749	/* Get VRAM informations */
1128	serge	750	rdev->mc.vram_is_ddr = true;
1221	serge	751	tmp = RREG32(RAMCFG);
		752	if (tmp & CHANSIZE_OVERRIDE) {
1128	serge	753	chansize = 16;
1221	serge	754	} else if (tmp & CHANSIZE_MASK) {
1128	serge	755	chansize = 64;
		756	} else {
		757	chansize = 32;
		758	}
1268	serge	759	tmp = RREG32(CHMAP);
		760	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		761	case 0:
		762	default:
		763	numchan = 1;
		764	break;
		765	case 1:
		766	numchan = 2;
		767	break;
		768	case 2:
		769	numchan = 4;
		770	break;
		771	case 3:
		772	numchan = 8;
		773	break;
1128	serge	774	}
1268	serge	775	rdev->mc.vram_width = numchan * chansize;
1221	serge	776	/* Could aper size report 0 ? */
1963	serge	777	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		778	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1221	serge	779	/* Setup GPU memory space */
		780	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		781	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1430	serge	782	rdev->mc.visible_vram_size = rdev->mc.aper_size;
		783	r600_vram_gtt_location(rdev, &rdev->mc);
1963	serge	784
		785	if (rdev->flags & RADEON_IS_IGP) {
		786	rs690_pm_info(rdev);
1403	serge	787	rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1963	serge	788	}
		789	radeon_update_bandwidth_info(rdev);
1221	serge	790	return 0;
1128	serge	791	}
		792
1221	serge	793	/* We doesn't check that the GPU really needs a reset we simply do the
		794	* reset, it's up to the caller to determine if the GPU needs one. We
		795	* might add an helper function to check that.
		796	*/
		797	int r600_gpu_soft_reset(struct radeon_device *rdev)
1128	serge	798	{
1221	serge	799	struct rv515_mc_save save;
		800	u32 grbm_busy_mask = S_008010_VC_BUSY(1) \| S_008010_VGT_BUSY_NO_DMA(1) \|
		801	S_008010_VGT_BUSY(1) \| S_008010_TA03_BUSY(1) \|
		802	S_008010_TC_BUSY(1) \| S_008010_SX_BUSY(1) \|
		803	S_008010_SH_BUSY(1) \| S_008010_SPI03_BUSY(1) \|
		804	S_008010_SMX_BUSY(1) \| S_008010_SC_BUSY(1) \|
		805	S_008010_PA_BUSY(1) \| S_008010_DB03_BUSY(1) \|
		806	S_008010_CR_BUSY(1) \| S_008010_CB03_BUSY(1) \|
		807	S_008010_GUI_ACTIVE(1);
		808	u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) \| S_008014_SPI1_BUSY(1) \|
		809	S_008014_SPI2_BUSY(1) \| S_008014_SPI3_BUSY(1) \|
		810	S_008014_TA0_BUSY(1) \| S_008014_TA1_BUSY(1) \|
		811	S_008014_TA2_BUSY(1) \| S_008014_TA3_BUSY(1) \|
		812	S_008014_DB0_BUSY(1) \| S_008014_DB1_BUSY(1) \|
		813	S_008014_DB2_BUSY(1) \| S_008014_DB3_BUSY(1) \|
		814	S_008014_CB0_BUSY(1) \| S_008014_CB1_BUSY(1) \|
		815	S_008014_CB2_BUSY(1) \| S_008014_CB3_BUSY(1);
		816	u32 tmp;
1128	serge	817
1963	serge	818	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
		819	return 0;
		820
1221	serge	821	dev_info(rdev->dev, "GPU softreset \n");
		822	dev_info(rdev->dev, " R_008010_GRBM_STATUS=0x%08X\n",
		823	RREG32(R_008010_GRBM_STATUS));
		824	dev_info(rdev->dev, " R_008014_GRBM_STATUS2=0x%08X\n",
		825	RREG32(R_008014_GRBM_STATUS2));
		826	dev_info(rdev->dev, " R_000E50_SRBM_STATUS=0x%08X\n",
		827	RREG32(R_000E50_SRBM_STATUS));
		828	rv515_mc_stop(rdev, &save);
		829	if (r600_mc_wait_for_idle(rdev)) {
		830	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		831	}
		832	/* Disable CP parsing/prefetching */
1963	serge	833	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1221	serge	834	/* Check if any of the rendering block is busy and reset it */
		835	if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) \|\|
		836	(RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
		837	tmp = S_008020_SOFT_RESET_CR(1) \|
		838	S_008020_SOFT_RESET_DB(1) \|
		839	S_008020_SOFT_RESET_CB(1) \|
		840	S_008020_SOFT_RESET_PA(1) \|
		841	S_008020_SOFT_RESET_SC(1) \|
		842	S_008020_SOFT_RESET_SMX(1) \|
		843	S_008020_SOFT_RESET_SPI(1) \|
		844	S_008020_SOFT_RESET_SX(1) \|
		845	S_008020_SOFT_RESET_SH(1) \|
		846	S_008020_SOFT_RESET_TC(1) \|
		847	S_008020_SOFT_RESET_TA(1) \|
		848	S_008020_SOFT_RESET_VC(1) \|
		849	S_008020_SOFT_RESET_VGT(1);
		850	dev_info(rdev->dev, " R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		851	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1963	serge	852	RREG32(R_008020_GRBM_SOFT_RESET);
		853	mdelay(15);
1221	serge	854	WREG32(R_008020_GRBM_SOFT_RESET, 0);
		855	}
		856	/* Reset CP (we always reset CP) */
		857	tmp = S_008020_SOFT_RESET_CP(1);
		858	dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		859	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1963	serge	860	RREG32(R_008020_GRBM_SOFT_RESET);
		861	mdelay(15);
1221	serge	862	WREG32(R_008020_GRBM_SOFT_RESET, 0);
		863	/* Wait a little for things to settle down */
1963	serge	864	mdelay(1);
1221	serge	865	dev_info(rdev->dev, " R_008010_GRBM_STATUS=0x%08X\n",
		866	RREG32(R_008010_GRBM_STATUS));
		867	dev_info(rdev->dev, " R_008014_GRBM_STATUS2=0x%08X\n",
		868	RREG32(R_008014_GRBM_STATUS2));
		869	dev_info(rdev->dev, " R_000E50_SRBM_STATUS=0x%08X\n",
		870	RREG32(R_000E50_SRBM_STATUS));
		871	rv515_mc_resume(rdev, &save);
		872	return 0;
1128	serge	873	}
		874
1963	serge	875	bool r600_gpu_is_lockup(struct radeon_device *rdev)
1221	serge	876	{
1963	serge	877	u32 srbm_status;
		878	u32 grbm_status;
		879	u32 grbm_status2;
		880	struct r100_gpu_lockup *lockup;
		881	int r;
		882
		883	if (rdev->family >= CHIP_RV770)
		884	lockup = &rdev->config.rv770.lockup;
		885	else
		886	lockup = &rdev->config.r600.lockup;
		887
		888	srbm_status = RREG32(R_000E50_SRBM_STATUS);
		889	grbm_status = RREG32(R_008010_GRBM_STATUS);
		890	grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
		891	if (!G_008010_GUI_ACTIVE(grbm_status)) {
		892	r100_gpu_lockup_update(lockup, &rdev->cp);
		893	return false;
		894	}
		895	/* force CP activities */
		896	r = radeon_ring_lock(rdev, 2);
		897	if (!r) {
		898	/* PACKET2 NOP */
		899	radeon_ring_write(rdev, 0x80000000);
		900	radeon_ring_write(rdev, 0x80000000);
		901	radeon_ring_unlock_commit(rdev);
		902	}
		903	rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
		904	return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
		905	}
		906
		907	int r600_asic_reset(struct radeon_device *rdev)
		908	{
1221	serge	909	return r600_gpu_soft_reset(rdev);
		910	}
		911
		912	static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
		913	u32 num_backends,
		914	u32 backend_disable_mask)
		915	{
		916	u32 backend_map = 0;
		917	u32 enabled_backends_mask;
		918	u32 enabled_backends_count;
		919	u32 cur_pipe;
		920	u32 swizzle_pipe[R6XX_MAX_PIPES];
		921	u32 cur_backend;
		922	u32 i;
		923
		924	if (num_tile_pipes > R6XX_MAX_PIPES)
		925	num_tile_pipes = R6XX_MAX_PIPES;
		926	if (num_tile_pipes < 1)
		927	num_tile_pipes = 1;
		928	if (num_backends > R6XX_MAX_BACKENDS)
		929	num_backends = R6XX_MAX_BACKENDS;
		930	if (num_backends < 1)
		931	num_backends = 1;
		932
		933	enabled_backends_mask = 0;
		934	enabled_backends_count = 0;
		935	for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
		936	if (((backend_disable_mask >> i) & 1) == 0) {
		937	enabled_backends_mask \|= (1 << i);
		938	++enabled_backends_count;
		939	}
		940	if (enabled_backends_count == num_backends)
		941	break;
		942	}
		943
		944	if (enabled_backends_count == 0) {
		945	enabled_backends_mask = 1;
		946	enabled_backends_count = 1;
		947	}
		948
		949	if (enabled_backends_count != num_backends)
		950	num_backends = enabled_backends_count;
		951
		952	memset((uint8_t )&swizzle_pipe[0], 0, sizeof(u32) R6XX_MAX_PIPES);
		953	switch (num_tile_pipes) {
		954	case 1:
		955	swizzle_pipe[0] = 0;
		956	break;
		957	case 2:
		958	swizzle_pipe[0] = 0;
		959	swizzle_pipe[1] = 1;
		960	break;
		961	case 3:
		962	swizzle_pipe[0] = 0;
		963	swizzle_pipe[1] = 1;
		964	swizzle_pipe[2] = 2;
		965	break;
		966	case 4:
		967	swizzle_pipe[0] = 0;
		968	swizzle_pipe[1] = 1;
		969	swizzle_pipe[2] = 2;
		970	swizzle_pipe[3] = 3;
		971	break;
		972	case 5:
		973	swizzle_pipe[0] = 0;
		974	swizzle_pipe[1] = 1;
		975	swizzle_pipe[2] = 2;
		976	swizzle_pipe[3] = 3;
		977	swizzle_pipe[4] = 4;
		978	break;
		979	case 6:
		980	swizzle_pipe[0] = 0;
		981	swizzle_pipe[1] = 2;
		982	swizzle_pipe[2] = 4;
		983	swizzle_pipe[3] = 5;
		984	swizzle_pipe[4] = 1;
		985	swizzle_pipe[5] = 3;
		986	break;
		987	case 7:
		988	swizzle_pipe[0] = 0;
		989	swizzle_pipe[1] = 2;
		990	swizzle_pipe[2] = 4;
		991	swizzle_pipe[3] = 6;
		992	swizzle_pipe[4] = 1;
		993	swizzle_pipe[5] = 3;
		994	swizzle_pipe[6] = 5;
		995	break;
		996	case 8:
		997	swizzle_pipe[0] = 0;
		998	swizzle_pipe[1] = 2;
		999	swizzle_pipe[2] = 4;
		1000	swizzle_pipe[3] = 6;
		1001	swizzle_pipe[4] = 1;
		1002	swizzle_pipe[5] = 3;
		1003	swizzle_pipe[6] = 5;
		1004	swizzle_pipe[7] = 7;
		1005	break;
		1006	}
		1007
		1008	cur_backend = 0;
		1009	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
		1010	while (((1 << cur_backend) & enabled_backends_mask) == 0)
		1011	cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
		1012
		1013	backend_map \|= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
		1014
		1015	cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
		1016	}
		1017
		1018	return backend_map;
		1019	}
		1020
		1021	int r600_count_pipe_bits(uint32_t val)
		1022	{
		1023	int i, ret = 0;
		1024
		1025	for (i = 0; i < 32; i++) {
		1026	ret += val & 1;
		1027	val >>= 1;
		1028	}
		1029	return ret;
		1030	}
		1031
		1032	void r600_gpu_init(struct radeon_device *rdev)
		1033	{
		1034	u32 tiling_config;
		1035	u32 ramcfg;
1430	serge	1036	u32 backend_map;
		1037	u32 cc_rb_backend_disable;
		1038	u32 cc_gc_shader_pipe_config;
1221	serge	1039	u32 tmp;
		1040	int i, j;
		1041	u32 sq_config;
		1042	u32 sq_gpr_resource_mgmt_1 = 0;
		1043	u32 sq_gpr_resource_mgmt_2 = 0;
		1044	u32 sq_thread_resource_mgmt = 0;
		1045	u32 sq_stack_resource_mgmt_1 = 0;
		1046	u32 sq_stack_resource_mgmt_2 = 0;
		1047
		1048	/* FIXME: implement */
		1049	switch (rdev->family) {
		1050	case CHIP_R600:
		1051	rdev->config.r600.max_pipes = 4;
		1052	rdev->config.r600.max_tile_pipes = 8;
		1053	rdev->config.r600.max_simds = 4;
		1054	rdev->config.r600.max_backends = 4;
		1055	rdev->config.r600.max_gprs = 256;
		1056	rdev->config.r600.max_threads = 192;
		1057	rdev->config.r600.max_stack_entries = 256;
		1058	rdev->config.r600.max_hw_contexts = 8;
		1059	rdev->config.r600.max_gs_threads = 16;
		1060	rdev->config.r600.sx_max_export_size = 128;
		1061	rdev->config.r600.sx_max_export_pos_size = 16;
		1062	rdev->config.r600.sx_max_export_smx_size = 128;
		1063	rdev->config.r600.sq_num_cf_insts = 2;
		1064	break;
		1065	case CHIP_RV630:
		1066	case CHIP_RV635:
		1067	rdev->config.r600.max_pipes = 2;
		1068	rdev->config.r600.max_tile_pipes = 2;
		1069	rdev->config.r600.max_simds = 3;
		1070	rdev->config.r600.max_backends = 1;
		1071	rdev->config.r600.max_gprs = 128;
		1072	rdev->config.r600.max_threads = 192;
		1073	rdev->config.r600.max_stack_entries = 128;
		1074	rdev->config.r600.max_hw_contexts = 8;
		1075	rdev->config.r600.max_gs_threads = 4;
		1076	rdev->config.r600.sx_max_export_size = 128;
		1077	rdev->config.r600.sx_max_export_pos_size = 16;
		1078	rdev->config.r600.sx_max_export_smx_size = 128;
		1079	rdev->config.r600.sq_num_cf_insts = 2;
		1080	break;
		1081	case CHIP_RV610:
		1082	case CHIP_RV620:
		1083	case CHIP_RS780:
		1084	case CHIP_RS880:
		1085	rdev->config.r600.max_pipes = 1;
		1086	rdev->config.r600.max_tile_pipes = 1;
		1087	rdev->config.r600.max_simds = 2;
		1088	rdev->config.r600.max_backends = 1;
		1089	rdev->config.r600.max_gprs = 128;
		1090	rdev->config.r600.max_threads = 192;
		1091	rdev->config.r600.max_stack_entries = 128;
		1092	rdev->config.r600.max_hw_contexts = 4;
		1093	rdev->config.r600.max_gs_threads = 4;
		1094	rdev->config.r600.sx_max_export_size = 128;
		1095	rdev->config.r600.sx_max_export_pos_size = 16;
		1096	rdev->config.r600.sx_max_export_smx_size = 128;
		1097	rdev->config.r600.sq_num_cf_insts = 1;
		1098	break;
		1099	case CHIP_RV670:
		1100	rdev->config.r600.max_pipes = 4;
		1101	rdev->config.r600.max_tile_pipes = 4;
		1102	rdev->config.r600.max_simds = 4;
		1103	rdev->config.r600.max_backends = 4;
		1104	rdev->config.r600.max_gprs = 192;
		1105	rdev->config.r600.max_threads = 192;
		1106	rdev->config.r600.max_stack_entries = 256;
		1107	rdev->config.r600.max_hw_contexts = 8;
		1108	rdev->config.r600.max_gs_threads = 16;
		1109	rdev->config.r600.sx_max_export_size = 128;
		1110	rdev->config.r600.sx_max_export_pos_size = 16;
		1111	rdev->config.r600.sx_max_export_smx_size = 128;
		1112	rdev->config.r600.sq_num_cf_insts = 2;
		1113	break;
		1114	default:
		1115	break;
		1116	}
		1117
		1118	/* Initialize HDP */
		1119	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1120	WREG32((0x2c14 + j), 0x00000000);
		1121	WREG32((0x2c18 + j), 0x00000000);
		1122	WREG32((0x2c1c + j), 0x00000000);
		1123	WREG32((0x2c20 + j), 0x00000000);
		1124	WREG32((0x2c24 + j), 0x00000000);
		1125	}
		1126
		1127	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1128
		1129	/* Setup tiling */
		1130	tiling_config = 0;
		1131	ramcfg = RREG32(RAMCFG);
		1132	switch (rdev->config.r600.max_tile_pipes) {
		1133	case 1:
		1134	tiling_config \|= PIPE_TILING(0);
		1135	break;
		1136	case 2:
		1137	tiling_config \|= PIPE_TILING(1);
		1138	break;
		1139	case 4:
		1140	tiling_config \|= PIPE_TILING(2);
		1141	break;
		1142	case 8:
		1143	tiling_config \|= PIPE_TILING(3);
		1144	break;
		1145	default:
		1146	break;
		1147	}
1430	serge	1148	rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
		1149	rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1221	serge	1150	tiling_config \|= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1963	serge	1151	tiling_config \|= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
		1152	if ((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT)
		1153	rdev->config.r600.tiling_group_size = 512;
		1154	else
1430	serge	1155	rdev->config.r600.tiling_group_size = 256;
1221	serge	1156	tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
		1157	if (tmp > 3) {
		1158	tiling_config \|= ROW_TILING(3);
		1159	tiling_config \|= SAMPLE_SPLIT(3);
		1160	} else {
		1161	tiling_config \|= ROW_TILING(tmp);
		1162	tiling_config \|= SAMPLE_SPLIT(tmp);
		1163	}
		1164	tiling_config \|= BANK_SWAPS(1);
1430	serge	1165
		1166	cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
		1167	cc_rb_backend_disable \|=
		1168	BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << rdev->config.r600.max_backends) & R6XX_MAX_BACKENDS_MASK);
		1169
		1170	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
		1171	cc_gc_shader_pipe_config \|=
		1172	INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << rdev->config.r600.max_pipes) & R6XX_MAX_PIPES_MASK);
		1173	cc_gc_shader_pipe_config \|=
		1174	INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << rdev->config.r600.max_simds) & R6XX_MAX_SIMDS_MASK);
		1175
		1176	backend_map = r600_get_tile_pipe_to_backend_map(rdev->config.r600.max_tile_pipes,
		1177	(R6XX_MAX_BACKENDS -
		1178	r600_count_pipe_bits((cc_rb_backend_disable &
		1179	R6XX_MAX_BACKENDS_MASK) >> 16)),
		1180	(cc_rb_backend_disable >> 16));
1963	serge	1181	rdev->config.r600.tile_config = tiling_config;
1430	serge	1182	tiling_config \|= BACKEND_MAP(backend_map);
1221	serge	1183	WREG32(GB_TILING_CONFIG, tiling_config);
		1184	WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
		1185	WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
		1186
		1187	/* Setup pipes */
1430	serge	1188	WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
		1189	WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1963	serge	1190	WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1221	serge	1191
1430	serge	1192	tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1221	serge	1193	WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
		1194	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
		1195
		1196	/* Setup some CP states */
		1197	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \| ROQ_IB2_START(0x2b)));
		1198	WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) \| ROQ_END(0x40)));
		1199
		1200	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \| SYNC_GRADIENT \|
		1201	SYNC_WALKER \| SYNC_ALIGNER));
		1202	/* Setup various GPU states */
		1203	if (rdev->family == CHIP_RV670)
		1204	WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
		1205
		1206	tmp = RREG32(SX_DEBUG_1);
		1207	tmp \|= SMX_EVENT_RELEASE;
		1208	if ((rdev->family > CHIP_R600))
		1209	tmp \|= ENABLE_NEW_SMX_ADDRESS;
		1210	WREG32(SX_DEBUG_1, tmp);
		1211
		1212	if (((rdev->family) == CHIP_R600) \|\|
		1213	((rdev->family) == CHIP_RV630) \|\|
		1214	((rdev->family) == CHIP_RV610) \|\|
		1215	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1216	((rdev->family) == CHIP_RS780) \|\|
		1217	((rdev->family) == CHIP_RS880)) {
1221	serge	1218	WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
		1219	} else {
		1220	WREG32(DB_DEBUG, 0);
		1221	}
		1222	WREG32(DB_WATERMARKS, (DEPTH_FREE(4) \| DEPTH_CACHELINE_FREE(16) \|
		1223	DEPTH_FLUSH(16) \| DEPTH_PENDING_FREE(4)));
		1224
		1225	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1226	WREG32(VGT_NUM_INSTANCES, 0);
		1227
		1228	WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
		1229	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
		1230
		1231	tmp = RREG32(SQ_MS_FIFO_SIZES);
		1232	if (((rdev->family) == CHIP_RV610) \|\|
		1233	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1234	((rdev->family) == CHIP_RS780) \|\|
		1235	((rdev->family) == CHIP_RS880)) {
1221	serge	1236	tmp = (CACHE_FIFO_SIZE(0xa) \|
		1237	FETCH_FIFO_HIWATER(0xa) \|
		1238	DONE_FIFO_HIWATER(0xe0) \|
		1239	ALU_UPDATE_FIFO_HIWATER(0x8));
		1240	} else if (((rdev->family) == CHIP_R600) \|\|
		1241	((rdev->family) == CHIP_RV630)) {
		1242	tmp &= ~DONE_FIFO_HIWATER(0xff);
		1243	tmp \|= DONE_FIFO_HIWATER(0x4);
		1244	}
		1245	WREG32(SQ_MS_FIFO_SIZES, tmp);
		1246
		1247	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
		1248	* should be adjusted as needed by the 2D/3D drivers. This just sets default values
		1249	*/
		1250	sq_config = RREG32(SQ_CONFIG);
		1251	sq_config &= ~(PS_PRIO(3) \|
		1252	VS_PRIO(3) \|
		1253	GS_PRIO(3) \|
		1254	ES_PRIO(3));
		1255	sq_config \|= (DX9_CONSTS \|
		1256	VC_ENABLE \|
		1257	PS_PRIO(0) \|
		1258	VS_PRIO(1) \|
		1259	GS_PRIO(2) \|
		1260	ES_PRIO(3));
		1261
		1262	if ((rdev->family) == CHIP_R600) {
		1263	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) \|
		1264	NUM_VS_GPRS(124) \|
		1265	NUM_CLAUSE_TEMP_GPRS(4));
		1266	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) \|
		1267	NUM_ES_GPRS(0));
		1268	sq_thread_resource_mgmt = (NUM_PS_THREADS(136) \|
		1269	NUM_VS_THREADS(48) \|
		1270	NUM_GS_THREADS(4) \|
		1271	NUM_ES_THREADS(4));
		1272	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) \|
		1273	NUM_VS_STACK_ENTRIES(128));
		1274	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) \|
		1275	NUM_ES_STACK_ENTRIES(0));
		1276	} else if (((rdev->family) == CHIP_RV610) \|\|
		1277	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1278	((rdev->family) == CHIP_RS780) \|\|
		1279	((rdev->family) == CHIP_RS880)) {
1221	serge	1280	/* no vertex cache */
		1281	sq_config &= ~VC_ENABLE;
		1282
		1283	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1284	NUM_VS_GPRS(44) \|
		1285	NUM_CLAUSE_TEMP_GPRS(2));
		1286	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1287	NUM_ES_GPRS(17));
		1288	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1289	NUM_VS_THREADS(78) \|
		1290	NUM_GS_THREADS(4) \|
		1291	NUM_ES_THREADS(31));
		1292	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1293	NUM_VS_STACK_ENTRIES(40));
		1294	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1295	NUM_ES_STACK_ENTRIES(16));
		1296	} else if (((rdev->family) == CHIP_RV630) \|\|
		1297	((rdev->family) == CHIP_RV635)) {
		1298	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1299	NUM_VS_GPRS(44) \|
		1300	NUM_CLAUSE_TEMP_GPRS(2));
		1301	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) \|
		1302	NUM_ES_GPRS(18));
		1303	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1304	NUM_VS_THREADS(78) \|
		1305	NUM_GS_THREADS(4) \|
		1306	NUM_ES_THREADS(31));
		1307	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1308	NUM_VS_STACK_ENTRIES(40));
		1309	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1310	NUM_ES_STACK_ENTRIES(16));
		1311	} else if ((rdev->family) == CHIP_RV670) {
		1312	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1313	NUM_VS_GPRS(44) \|
		1314	NUM_CLAUSE_TEMP_GPRS(2));
		1315	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1316	NUM_ES_GPRS(17));
		1317	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1318	NUM_VS_THREADS(78) \|
		1319	NUM_GS_THREADS(4) \|
		1320	NUM_ES_THREADS(31));
		1321	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) \|
		1322	NUM_VS_STACK_ENTRIES(64));
		1323	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) \|
		1324	NUM_ES_STACK_ENTRIES(64));
		1325	}
		1326
		1327	WREG32(SQ_CONFIG, sq_config);
		1328	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		1329	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		1330	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		1331	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		1332	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		1333
		1334	if (((rdev->family) == CHIP_RV610) \|\|
		1335	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1336	((rdev->family) == CHIP_RS780) \|\|
		1337	((rdev->family) == CHIP_RS880)) {
1221	serge	1338	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
		1339	} else {
		1340	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
		1341	}
		1342
		1343	/* More default values. 2D/3D driver should adjust as needed */
		1344	WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) \| S0_Y(0x4) \|
		1345	S1_X(0x4) \| S1_Y(0xc)));
		1346	WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) \| S0_Y(0xe) \|
		1347	S1_X(0x2) \| S1_Y(0x2) \|
		1348	S2_X(0xa) \| S2_Y(0x6) \|
		1349	S3_X(0x6) \| S3_Y(0xa)));
		1350	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) \| S0_Y(0xb) \|
		1351	S1_X(0x4) \| S1_Y(0xc) \|
		1352	S2_X(0x1) \| S2_Y(0x6) \|
		1353	S3_X(0xa) \| S3_Y(0xe)));
		1354	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) \| S4_Y(0x1) \|
		1355	S5_X(0x0) \| S5_Y(0x0) \|
		1356	S6_X(0xb) \| S6_Y(0x4) \|
		1357	S7_X(0x7) \| S7_Y(0x8)));
		1358
		1359	WREG32(VGT_STRMOUT_EN, 0);
		1360	tmp = rdev->config.r600.max_pipes * 16;
		1361	switch (rdev->family) {
		1362	case CHIP_RV610:
1268	serge	1363	case CHIP_RV620:
1221	serge	1364	case CHIP_RS780:
1268	serge	1365	case CHIP_RS880:
1221	serge	1366	tmp += 32;
		1367	break;
		1368	case CHIP_RV670:
		1369	tmp += 128;
		1370	break;
		1371	default:
		1372	break;
		1373	}
		1374	if (tmp > 256) {
		1375	tmp = 256;
		1376	}
		1377	WREG32(VGT_ES_PER_GS, 128);
		1378	WREG32(VGT_GS_PER_ES, tmp);
		1379	WREG32(VGT_GS_PER_VS, 2);
		1380	WREG32(VGT_GS_VERTEX_REUSE, 16);
		1381
		1382	/* more default values. 2D/3D driver should adjust as needed */
		1383	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		1384	WREG32(VGT_STRMOUT_EN, 0);
		1385	WREG32(SX_MISC, 0);
		1386	WREG32(PA_SC_MODE_CNTL, 0);
		1387	WREG32(PA_SC_AA_CONFIG, 0);
		1388	WREG32(PA_SC_LINE_STIPPLE, 0);
		1389	WREG32(SPI_INPUT_Z, 0);
		1390	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
		1391	WREG32(CB_COLOR7_FRAG, 0);
		1392
		1393	/* Clear render buffer base addresses */
		1394	WREG32(CB_COLOR0_BASE, 0);
		1395	WREG32(CB_COLOR1_BASE, 0);
		1396	WREG32(CB_COLOR2_BASE, 0);
		1397	WREG32(CB_COLOR3_BASE, 0);
		1398	WREG32(CB_COLOR4_BASE, 0);
		1399	WREG32(CB_COLOR5_BASE, 0);
		1400	WREG32(CB_COLOR6_BASE, 0);
		1401	WREG32(CB_COLOR7_BASE, 0);
		1402	WREG32(CB_COLOR7_FRAG, 0);
		1403
		1404	switch (rdev->family) {
		1405	case CHIP_RV610:
1268	serge	1406	case CHIP_RV620:
1221	serge	1407	case CHIP_RS780:
1268	serge	1408	case CHIP_RS880:
1221	serge	1409	tmp = TC_L2_SIZE(8);
		1410	break;
		1411	case CHIP_RV630:
		1412	case CHIP_RV635:
		1413	tmp = TC_L2_SIZE(4);
		1414	break;
		1415	case CHIP_R600:
		1416	tmp = TC_L2_SIZE(0) \| L2_DISABLE_LATE_HIT;
		1417	break;
		1418	default:
		1419	tmp = TC_L2_SIZE(0);
		1420	break;
		1421	}
		1422	WREG32(TC_CNTL, tmp);
		1423
		1424	tmp = RREG32(HDP_HOST_PATH_CNTL);
		1425	WREG32(HDP_HOST_PATH_CNTL, tmp);
		1426
		1427	tmp = RREG32(ARB_POP);
		1428	tmp \|= ENABLE_TC128;
		1429	WREG32(ARB_POP, tmp);
		1430
		1431	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1432	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA \|
		1433	NUM_CLIP_SEQ(3)));
		1434	WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
		1435	}
		1436
		1437
1128	serge	1438	/*
		1439	* Indirect registers accessor
		1440	*/
1221	serge	1441	u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1128	serge	1442	{
1221	serge	1443	u32 r;
1128	serge	1444
1221	serge	1445	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1446	(void)RREG32(PCIE_PORT_INDEX);
		1447	r = RREG32(PCIE_PORT_DATA);
1128	serge	1448	return r;
		1449	}
		1450
1221	serge	1451	void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1128	serge	1452	{
1221	serge	1453	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1454	(void)RREG32(PCIE_PORT_INDEX);
		1455	WREG32(PCIE_PORT_DATA, (v));
		1456	(void)RREG32(PCIE_PORT_DATA);
1128	serge	1457	}
1221	serge	1458
		1459	/*
		1460	* CP & Ring
		1461	*/
		1462	void r600_cp_stop(struct radeon_device *rdev)
		1463	{
1963	serge	1464	// radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1221	serge	1465	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1963	serge	1466	WREG32(SCRATCH_UMSK, 0);
1221	serge	1467	}
1413	serge	1468
		1469	int r600_init_microcode(struct radeon_device *rdev)
		1470	{
		1471	struct platform_device *pdev;
		1472	const char *chip_name;
		1473	const char *rlc_chip_name;
		1474	size_t pfp_req_size, me_req_size, rlc_req_size;
		1475	char fw_name[30];
		1476	int err;
		1477
		1478	DRM_DEBUG("\n");
		1479
		1480	pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
		1481	err = IS_ERR(pdev);
		1482	if (err) {
		1483	printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
		1484	return -EINVAL;
		1485	}
		1486
		1487	switch (rdev->family) {
		1488	case CHIP_R600:
		1489	chip_name = "R600";
		1490	rlc_chip_name = "R600";
		1491	break;
		1492	case CHIP_RV610:
		1493	chip_name = "RV610";
		1494	rlc_chip_name = "R600";
		1495	break;
		1496	case CHIP_RV630:
		1497	chip_name = "RV630";
		1498	rlc_chip_name = "R600";
		1499	break;
		1500	case CHIP_RV620:
		1501	chip_name = "RV620";
		1502	rlc_chip_name = "R600";
		1503	break;
		1504	case CHIP_RV635:
		1505	chip_name = "RV635";
		1506	rlc_chip_name = "R600";
		1507	break;
		1508	case CHIP_RV670:
		1509	chip_name = "RV670";
		1510	rlc_chip_name = "R600";
		1511	break;
		1512	case CHIP_RS780:
		1513	case CHIP_RS880:
		1514	chip_name = "RS780";
		1515	rlc_chip_name = "R600";
		1516	break;
		1517	case CHIP_RV770:
		1518	chip_name = "RV770";
		1519	rlc_chip_name = "R700";
		1520	break;
		1521	case CHIP_RV730:
		1522	case CHIP_RV740:
		1523	chip_name = "RV730";
		1524	rlc_chip_name = "R700";
		1525	break;
		1526	case CHIP_RV710:
		1527	chip_name = "RV710";
		1528	rlc_chip_name = "R700";
		1529	break;
1963	serge	1530	case CHIP_CEDAR:
		1531	chip_name = "CEDAR";
		1532	rlc_chip_name = "CEDAR";
		1533	break;
		1534	case CHIP_REDWOOD:
		1535	chip_name = "REDWOOD";
		1536	rlc_chip_name = "REDWOOD";
		1537	break;
		1538	case CHIP_JUNIPER:
		1539	chip_name = "JUNIPER";
		1540	rlc_chip_name = "JUNIPER";
		1541	break;
		1542	case CHIP_CYPRESS:
		1543	case CHIP_HEMLOCK:
		1544	chip_name = "CYPRESS";
		1545	rlc_chip_name = "CYPRESS";
		1546	break;
		1547	case CHIP_PALM:
		1548	chip_name = "PALM";
		1549	rlc_chip_name = "SUMO";
		1550	break;
1986	serge	1551	case CHIP_SUMO:
		1552	chip_name = "SUMO";
		1553	rlc_chip_name = "SUMO";
		1554	break;
		1555	case CHIP_SUMO2:
		1556	chip_name = "SUMO2";
		1557	rlc_chip_name = "SUMO";
		1558	break;
1413	serge	1559	default: BUG();
		1560	}
		1561
1963	serge	1562	if (rdev->family >= CHIP_CEDAR) {
		1563	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		1564	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		1565	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		1566	} else if (rdev->family >= CHIP_RV770) {
1413	serge	1567	pfp_req_size = R700_PFP_UCODE_SIZE * 4;
		1568	me_req_size = R700_PM4_UCODE_SIZE * 4;
		1569	rlc_req_size = R700_RLC_UCODE_SIZE * 4;
		1570	} else {
		1571	pfp_req_size = PFP_UCODE_SIZE * 4;
		1572	me_req_size = PM4_UCODE_SIZE * 12;
		1573	rlc_req_size = RLC_UCODE_SIZE * 4;
		1574	}
		1575
		1576	DRM_INFO("Loading %s Microcode\n", chip_name);
		1577
		1578	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
		1579	err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
		1580	if (err)
		1581	goto out;
		1582	if (rdev->pfp_fw->size != pfp_req_size) {
		1583	printk(KERN_ERR
		1584	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1585	rdev->pfp_fw->size, fw_name);
		1586	err = -EINVAL;
		1587	goto out;
		1588	}
		1589
		1590	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
		1591	err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
		1592	if (err)
		1593	goto out;
		1594	if (rdev->me_fw->size != me_req_size) {
		1595	printk(KERN_ERR
		1596	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1597	rdev->me_fw->size, fw_name);
		1598	err = -EINVAL;
		1599	}
		1600
		1601	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
		1602	err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
		1603	if (err)
		1604	goto out;
		1605	if (rdev->rlc_fw->size != rlc_req_size) {
		1606	printk(KERN_ERR
		1607	"r600_rlc: Bogus length %zu in firmware \"%s\"\n",
		1608	rdev->rlc_fw->size, fw_name);
		1609	err = -EINVAL;
		1610	}
		1611
		1612	out:
		1613	platform_device_unregister(pdev);
		1614
		1615	if (err) {
		1616	if (err != -EINVAL)
		1617	printk(KERN_ERR
		1618	"r600_cp: Failed to load firmware \"%s\"\n",
		1619	fw_name);
		1620	release_firmware(rdev->pfp_fw);
		1621	rdev->pfp_fw = NULL;
		1622	release_firmware(rdev->me_fw);
		1623	rdev->me_fw = NULL;
		1624	release_firmware(rdev->rlc_fw);
		1625	rdev->rlc_fw = NULL;
		1626	}
		1627	return err;
		1628	}
		1629
		1630	static int r600_cp_load_microcode(struct radeon_device *rdev)
		1631	{
		1632	const __be32 *fw_data;
		1633	int i;
		1634
		1635	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1636	return -EINVAL;
		1637
		1638	r600_cp_stop(rdev);
		1639
1963	serge	1640	WREG32(CP_RB_CNTL,
		1641	#ifdef __BIG_ENDIAN
		1642	BUF_SWAP_32BIT \|
		1643	#endif
		1644	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
1413	serge	1645
		1646	/* Reset cp */
		1647	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1648	RREG32(GRBM_SOFT_RESET);
		1649	mdelay(15);
		1650	WREG32(GRBM_SOFT_RESET, 0);
		1651
		1652	WREG32(CP_ME_RAM_WADDR, 0);
		1653
		1654	fw_data = (const __be32 *)rdev->me_fw->data;
		1655	WREG32(CP_ME_RAM_WADDR, 0);
		1656	for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
		1657	WREG32(CP_ME_RAM_DATA,
		1658	be32_to_cpup(fw_data++));
		1659
		1660	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1661	WREG32(CP_PFP_UCODE_ADDR, 0);
		1662	for (i = 0; i < PFP_UCODE_SIZE; i++)
		1663	WREG32(CP_PFP_UCODE_DATA,
		1664	be32_to_cpup(fw_data++));
		1665
		1666	WREG32(CP_PFP_UCODE_ADDR, 0);
		1667	WREG32(CP_ME_RAM_WADDR, 0);
		1668	WREG32(CP_ME_RAM_RADDR, 0);
		1669	return 0;
		1670	}
		1671
1221	serge	1672	int r600_cp_start(struct radeon_device *rdev)
		1673	{
		1674	int r;
		1675	uint32_t cp_me;
		1676
		1677	r = radeon_ring_lock(rdev, 7);
		1678	if (r) {
		1679	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1680	return r;
		1681	}
		1682	radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1683	radeon_ring_write(rdev, 0x1);
1963	serge	1684	if (rdev->family >= CHIP_RV770) {
		1685	radeon_ring_write(rdev, 0x0);
		1686	radeon_ring_write(rdev, rdev->config.rv770.max_hw_contexts - 1);
		1687	} else {
1221	serge	1688	radeon_ring_write(rdev, 0x3);
		1689	radeon_ring_write(rdev, rdev->config.r600.max_hw_contexts - 1);
		1690	}
		1691	radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1692	radeon_ring_write(rdev, 0);
		1693	radeon_ring_write(rdev, 0);
		1694	radeon_ring_unlock_commit(rdev);
		1695
		1696	cp_me = 0xff;
		1697	WREG32(R_0086D8_CP_ME_CNTL, cp_me);
		1698	return 0;
		1699	}
1413	serge	1700
		1701	int r600_cp_resume(struct radeon_device *rdev)
		1702	{
		1703	u32 tmp;
		1704	u32 rb_bufsz;
		1705	int r;
		1706
		1707	/* Reset cp */
		1708	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1709	RREG32(GRBM_SOFT_RESET);
		1710	mdelay(15);
		1711	WREG32(GRBM_SOFT_RESET, 0);
		1712
		1713	/* Set ring buffer size */
		1714	rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1963	serge	1715	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
1413	serge	1716	#ifdef __BIG_ENDIAN
		1717	tmp \|= BUF_SWAP_32BIT;
		1718	#endif
		1719	WREG32(CP_RB_CNTL, tmp);
		1720	WREG32(CP_SEM_WAIT_TIMER, 0x4);
		1721
		1722	/* Set the write pointer delay */
		1723	WREG32(CP_RB_WPTR_DELAY, 0);
		1724
		1725	/* Initialize the ring buffer's read and write pointers */
		1726	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1727	WREG32(CP_RB_RPTR_WR, 0);
		1728	WREG32(CP_RB_WPTR, 0);
1963	serge	1729
		1730	/* set the wb address whether it's enabled or not */
		1731	WREG32(CP_RB_RPTR_ADDR,
		1732	#ifdef __BIG_ENDIAN
		1733	RB_RPTR_SWAP(2) \|
		1734	#endif
		1735	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1736	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1737	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1738
		1739	if (rdev->wb.enabled)
		1740	WREG32(SCRATCH_UMSK, 0xff);
		1741	else {
		1742	tmp \|= RB_NO_UPDATE;
		1743	WREG32(SCRATCH_UMSK, 0);
		1744	}
		1745
1413	serge	1746	mdelay(1);
		1747	WREG32(CP_RB_CNTL, tmp);
		1748
		1749	WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
		1750	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1751
		1752	rdev->cp.rptr = RREG32(CP_RB_RPTR);
		1753	rdev->cp.wptr = RREG32(CP_RB_WPTR);
		1754
		1755	r600_cp_start(rdev);
		1756	rdev->cp.ready = true;
		1757	r = radeon_ring_test(rdev);
		1758	if (r) {
		1759	rdev->cp.ready = false;
		1760	return r;
		1761	}
		1762	return 0;
		1763	}
		1764
1221	serge	1765	void r600_cp_commit(struct radeon_device *rdev)
		1766	{
		1767	WREG32(CP_RB_WPTR, rdev->cp.wptr);
		1768	(void)RREG32(CP_RB_WPTR);
		1769	}
		1770
1233	serge	1771	void r600_ring_init(struct radeon_device *rdev, unsigned ring_size)
		1772	{
		1773	u32 rb_bufsz;
1221	serge	1774
1233	serge	1775	/* Align ring size */
		1776	rb_bufsz = drm_order(ring_size / 8);
		1777	ring_size = (1 << (rb_bufsz + 1)) * 4;
		1778	rdev->cp.ring_size = ring_size;
		1779	rdev->cp.align_mask = 16 - 1;
		1780	}
		1781
1963	serge	1782	void r600_cp_fini(struct radeon_device *rdev)
		1783	{
		1784	r600_cp_stop(rdev);
		1785	radeon_ring_fini(rdev);
		1786	}
1233	serge	1787
1963	serge	1788
1233	serge	1789	/*
		1790	* GPU scratch registers helpers function.
		1791	*/
		1792	void r600_scratch_init(struct radeon_device *rdev)
		1793	{
		1794	int i;
		1795
		1796	rdev->scratch.num_reg = 7;
1963	serge	1797	rdev->scratch.reg_base = SCRATCH_REG0;
1233	serge	1798	for (i = 0; i < rdev->scratch.num_reg; i++) {
		1799	rdev->scratch.free[i] = true;
1963	serge	1800	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
1233	serge	1801	}
		1802	}
1413	serge	1803
		1804	int r600_ring_test(struct radeon_device *rdev)
		1805	{
		1806	uint32_t scratch;
		1807	uint32_t tmp = 0;
		1808	unsigned i;
		1809	int r;
		1810
		1811	r = radeon_scratch_get(rdev, &scratch);
		1812	if (r) {
		1813	DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
		1814	return r;
		1815	}
		1816	WREG32(scratch, 0xCAFEDEAD);
		1817	r = radeon_ring_lock(rdev, 3);
		1818	if (r) {
		1819	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1820	radeon_scratch_free(rdev, scratch);
		1821	return r;
		1822	}
		1823	radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1824	radeon_ring_write(rdev, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		1825	radeon_ring_write(rdev, 0xDEADBEEF);
		1826	radeon_ring_unlock_commit(rdev);
		1827	for (i = 0; i < rdev->usec_timeout; i++) {
		1828	tmp = RREG32(scratch);
		1829	if (tmp == 0xDEADBEEF)
		1830	break;
		1831	DRM_UDELAY(1);
		1832	}
		1833	if (i < rdev->usec_timeout) {
		1834	DRM_INFO("ring test succeeded in %d usecs\n", i);
		1835	} else {
		1836	DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n",
		1837	scratch, tmp);
		1838	r = -EINVAL;
		1839	}
		1840	radeon_scratch_free(rdev, scratch);
		1841	return r;
		1842	}
1963	serge	1843
1413	serge	1844	void r600_fence_ring_emit(struct radeon_device *rdev,
		1845	struct radeon_fence *fence)
		1846	{
1963	serge	1847	if (rdev->wb.use_event) {
		1848	u64 addr = rdev->wb.gpu_addr + R600_WB_EVENT_OFFSET +
		1849	(u64)(rdev->fence_drv.scratch_reg - rdev->scratch.reg_base);
		1850	/* EVENT_WRITE_EOP - flush caches, send int */
		1851	radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		1852	radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) \| EVENT_INDEX(5));
		1853	radeon_ring_write(rdev, addr & 0xffffffff);
		1854	radeon_ring_write(rdev, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		1855	radeon_ring_write(rdev, fence->seq);
		1856	radeon_ring_write(rdev, 0);
		1857	} else {
1430	serge	1858	radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
1963	serge	1859	radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) \| EVENT_INDEX(0));
1430	serge	1860	/* wait for 3D idle clean */
		1861	radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1862	radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		1863	radeon_ring_write(rdev, WAIT_3D_IDLE_bit \| WAIT_3D_IDLECLEAN_bit);
1413	serge	1864	/* Emit fence sequence & fire IRQ */
		1865	radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1866	radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		1867	radeon_ring_write(rdev, fence->seq);
		1868	/* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
		1869	radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
		1870	radeon_ring_write(rdev, RB_INT_STAT);
1963	serge	1871	}
1413	serge	1872	}
1963	serge	1873
2005	serge	1874	int r600_copy_blit(struct radeon_device *rdev,
		1875	uint64_t src_offset, uint64_t dst_offset,
		1876	unsigned num_pages, struct radeon_fence *fence)
		1877	{
		1878	int r;
1963	serge	1879
2005	serge	1880	mutex_lock(&rdev->r600_blit.mutex);
		1881	rdev->r600_blit.vb_ib = NULL;
		1882	r = r600_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
		1883	if (r) {
		1884	// if (rdev->r600_blit.vb_ib)
		1885	// radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
		1886	mutex_unlock(&rdev->r600_blit.mutex);
		1887	return r;
		1888	}
		1889	r600_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
		1890	r600_blit_done_copy(rdev, fence);
		1891	mutex_unlock(&rdev->r600_blit.mutex);
		1892	return 0;
		1893	}
		1894
1221	serge	1895	int r600_set_surface_reg(struct radeon_device *rdev, int reg,
		1896	uint32_t tiling_flags, uint32_t pitch,
		1897	uint32_t offset, uint32_t obj_size)
		1898	{
		1899	/* FIXME: implement */
		1900	return 0;
		1901	}
		1902
		1903	void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
		1904	{
		1905	/* FIXME: implement */
		1906	}
		1907
		1908	int r600_startup(struct radeon_device *rdev)
		1909	{
		1910	int r;
		1911
1963	serge	1912	/* enable pcie gen2 link */
		1913	r600_pcie_gen2_enable(rdev);
		1914
1413	serge	1915	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		1916	r = r600_init_microcode(rdev);
		1917	if (r) {
		1918	DRM_ERROR("Failed to load firmware!\n");
		1919	return r;
		1920	}
		1921	}
		1922
1221	serge	1923	r600_mc_program(rdev);
		1924	if (rdev->flags & RADEON_IS_AGP) {
		1925	r600_agp_enable(rdev);
		1926	} else {
		1927	r = r600_pcie_gart_enable(rdev);
		1928	if (r)
		1929	return r;
		1930	}
		1931	r600_gpu_init(rdev);
2005	serge	1932	r = r600_blit_init(rdev);
		1933	if (r) {
		1934	// r600_blit_fini(rdev);
		1935	rdev->asic->copy = NULL;
		1936	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		1937	}
1221	serge	1938
2005	serge	1939	/* allocate wb buffer */
		1940	r = radeon_wb_init(rdev);
		1941	if (r)
		1942	return r;
		1943
		1944	/* Enable IRQ */
		1945	r = r600_irq_init(rdev);
		1946	if (r) {
		1947	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		1948	// radeon_irq_kms_fini(rdev);
		1949	return r;
		1950	}
		1951	r600_irq_set(rdev);
		1952
1413	serge	1953	r = radeon_ring_init(rdev, rdev->cp.ring_size);
		1954	if (r)
		1955	return r;
		1956	r = r600_cp_load_microcode(rdev);
		1957	if (r)
		1958	return r;
		1959	r = r600_cp_resume(rdev);
		1960	if (r)
		1961	return r;
1963	serge	1962
1221	serge	1963	return 0;
		1964	}
		1965
		1966	void r600_vga_set_state(struct radeon_device *rdev, bool state)
		1967	{
		1968	uint32_t temp;
		1969
		1970	temp = RREG32(CONFIG_CNTL);
		1971	if (state == false) {
		1972	temp &= ~(1<<0);
		1973	temp \|= (1<<1);
		1974	} else {
		1975	temp &= ~(1<<1);
		1976	}
		1977	WREG32(CONFIG_CNTL, temp);
		1978	}
		1979
		1980
		1981
		1982
		1983
		1984	/* Plan is to move initialization in that function and use
		1985	* helper function so that radeon_device_init pretty much
		1986	* do nothing more than calling asic specific function. This
		1987	* should also allow to remove a bunch of callback function
		1988	* like vram_info.
		1989	*/
		1990	int r600_init(struct radeon_device *rdev)
		1991	{
		1992	int r;
		1993
		1994	if (r600_debugfs_mc_info_init(rdev)) {
		1995	DRM_ERROR("Failed to register debugfs file for mc !\n");
		1996	}
		1997	/* This don't do much */
2004	serge	1998	r = radeon_gem_init(rdev);
		1999	if (r)
		2000	return r;
1221	serge	2001	/* Read BIOS */
		2002	if (!radeon_get_bios(rdev)) {
		2003	if (ASIC_IS_AVIVO(rdev))
		2004	return -EINVAL;
		2005	}
		2006	/* Must be an ATOMBIOS */
		2007	if (!rdev->is_atom_bios) {
		2008	dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
		2009	return -EINVAL;
		2010	}
		2011	r = radeon_atombios_init(rdev);
		2012	if (r)
		2013	return r;
		2014	/* Post card if necessary */
1963	serge	2015	if (!radeon_card_posted(rdev)) {
1321	serge	2016	if (!rdev->bios) {
		2017	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		2018	return -EINVAL;
		2019	}
1221	serge	2020	DRM_INFO("GPU not posted. posting now...\n");
		2021	atom_asic_init(rdev->mode_info.atom_context);
		2022	}
		2023	/* Initialize scratch registers */
		2024	r600_scratch_init(rdev);
		2025	/* Initialize surface registers */
		2026	radeon_surface_init(rdev);
1268	serge	2027	/* Initialize clocks */
1221	serge	2028	radeon_get_clock_info(rdev->ddev);
		2029	/* Fence driver */
2004	serge	2030	r = radeon_fence_driver_init(rdev);
		2031	if (r)
		2032	return r;
1403	serge	2033	if (rdev->flags & RADEON_IS_AGP) {
		2034	r = radeon_agp_init(rdev);
		2035	if (r)
		2036	radeon_agp_disable(rdev);
		2037	}
1221	serge	2038	r = r600_mc_init(rdev);
		2039	if (r)
		2040	return r;
		2041	/* Memory manager */
1321	serge	2042	r = radeon_bo_init(rdev);
1221	serge	2043	if (r)
		2044	return r;
1321	serge	2045
2004	serge	2046	r = radeon_irq_kms_init(rdev);
		2047	if (r)
		2048	return r;
1321	serge	2049
1413	serge	2050	rdev->cp.ring_obj = NULL;
		2051	r600_ring_init(rdev, 1024 * 1024);
1221	serge	2052
2004	serge	2053	rdev->ih.ring_obj = NULL;
		2054	r600_ih_ring_init(rdev, 64 * 1024);
1221	serge	2055
		2056	r = r600_pcie_gart_init(rdev);
		2057	if (r)
		2058	return r;
		2059
1321	serge	2060	rdev->accel_working = true;
1221	serge	2061	r = r600_startup(rdev);
		2062	if (r) {
1428	serge	2063	dev_err(rdev->dev, "disabling GPU acceleration\n");
1221	serge	2064	// r600_suspend(rdev);
		2065	// r600_wb_fini(rdev);
		2066	// radeon_ring_fini(rdev);
		2067	r600_pcie_gart_fini(rdev);
		2068	rdev->accel_working = false;
		2069	}
		2070	if (rdev->accel_working) {
2005	serge	2071	r = radeon_ib_pool_init(rdev);
		2072	if (r) {
		2073	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		2074	rdev->accel_working = false;
		2075	} else {
		2076	r = r600_ib_test(rdev);
		2077	if (r) {
		2078	dev_err(rdev->dev, "IB test failed (%d).\n", r);
		2079	rdev->accel_working = false;
		2080	}
1221	serge	2081	}
2005	serge	2082	}
		2083
1221	serge	2084	return 0;
		2085	}
		2086
2004	serge	2087	/*
		2088	* CS stuff
		2089	*/
		2090	void r600_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		2091	{
		2092	/* FIXME: implement */
		2093	radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		2094	radeon_ring_write(rdev,
		2095	#ifdef __BIG_ENDIAN
		2096	(2 << 0) \|
		2097	#endif
		2098	(ib->gpu_addr & 0xFFFFFFFC));
		2099	radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
		2100	radeon_ring_write(rdev, ib->length_dw);
		2101	}
		2102
		2103	int r600_ib_test(struct radeon_device *rdev)
		2104	{
		2105	struct radeon_ib *ib;
		2106	uint32_t scratch;
		2107	uint32_t tmp = 0;
		2108	unsigned i;
		2109	int r;
		2110
		2111	r = radeon_scratch_get(rdev, &scratch);
		2112	if (r) {
		2113	DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
		2114	return r;
		2115	}
		2116	WREG32(scratch, 0xCAFEDEAD);
		2117	r = radeon_ib_get(rdev, &ib);
		2118	if (r) {
		2119	DRM_ERROR("radeon: failed to get ib (%d).\n", r);
		2120	return r;
		2121	}
		2122	ib->ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
		2123	ib->ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		2124	ib->ptr[2] = 0xDEADBEEF;
		2125	ib->ptr[3] = PACKET2(0);
		2126	ib->ptr[4] = PACKET2(0);
		2127	ib->ptr[5] = PACKET2(0);
		2128	ib->ptr[6] = PACKET2(0);
		2129	ib->ptr[7] = PACKET2(0);
		2130	ib->ptr[8] = PACKET2(0);
		2131	ib->ptr[9] = PACKET2(0);
		2132	ib->ptr[10] = PACKET2(0);
		2133	ib->ptr[11] = PACKET2(0);
		2134	ib->ptr[12] = PACKET2(0);
		2135	ib->ptr[13] = PACKET2(0);
		2136	ib->ptr[14] = PACKET2(0);
		2137	ib->ptr[15] = PACKET2(0);
		2138	ib->length_dw = 16;
		2139	r = radeon_ib_schedule(rdev, ib);
		2140	if (r) {
		2141	radeon_scratch_free(rdev, scratch);
		2142	radeon_ib_free(rdev, &ib);
		2143	DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
		2144	return r;
		2145	}
		2146	r = radeon_fence_wait(ib->fence, false);
		2147	if (r) {
		2148	DRM_ERROR("radeon: fence wait failed (%d).\n", r);
		2149	return r;
		2150	}
		2151	for (i = 0; i < rdev->usec_timeout; i++) {
		2152	tmp = RREG32(scratch);
		2153	if (tmp == 0xDEADBEEF)
		2154	break;
		2155	DRM_UDELAY(1);
		2156	}
		2157	if (i < rdev->usec_timeout) {
		2158	DRM_INFO("ib test succeeded in %u usecs\n", i);
		2159	} else {
		2160	DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
		2161	scratch, tmp);
		2162	r = -EINVAL;
		2163	}
		2164	radeon_scratch_free(rdev, scratch);
		2165	radeon_ib_free(rdev, &ib);
		2166	return r;
		2167	}
		2168
		2169	/*
		2170	* Interrupts
		2171	*
		2172	* Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
		2173	* the same as the CP ring buffer, but in reverse. Rather than the CPU
		2174	* writing to the ring and the GPU consuming, the GPU writes to the ring
		2175	* and host consumes. As the host irq handler processes interrupts, it
		2176	* increments the rptr. When the rptr catches up with the wptr, all the
		2177	* current interrupts have been processed.
		2178	*/
		2179
		2180	void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
		2181	{
		2182	u32 rb_bufsz;
		2183
		2184	/* Align ring size */
		2185	rb_bufsz = drm_order(ring_size / 4);
		2186	ring_size = (1 << rb_bufsz) * 4;
		2187	rdev->ih.ring_size = ring_size;
		2188	rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
		2189	rdev->ih.rptr = 0;
		2190	}
		2191
		2192	static int r600_ih_ring_alloc(struct radeon_device *rdev)
		2193	{
		2194	int r;
		2195
		2196	/* Allocate ring buffer */
		2197	if (rdev->ih.ring_obj == NULL) {
		2198	r = radeon_bo_create(rdev, rdev->ih.ring_size,
		2199	PAGE_SIZE, true,
		2200	RADEON_GEM_DOMAIN_GTT,
		2201	&rdev->ih.ring_obj);
		2202	if (r) {
		2203	DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
		2204	return r;
		2205	}
		2206	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		2207	if (unlikely(r != 0))
		2208	return r;
		2209	r = radeon_bo_pin(rdev->ih.ring_obj,
		2210	RADEON_GEM_DOMAIN_GTT,
		2211	&rdev->ih.gpu_addr);
		2212	if (r) {
		2213	radeon_bo_unreserve(rdev->ih.ring_obj);
		2214	DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
		2215	return r;
		2216	}
		2217	r = radeon_bo_kmap(rdev->ih.ring_obj,
		2218	(void **)&rdev->ih.ring);
		2219	radeon_bo_unreserve(rdev->ih.ring_obj);
		2220	if (r) {
		2221	DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
		2222	return r;
		2223	}
		2224	}
		2225	return 0;
		2226	}
		2227
		2228	static void r600_ih_ring_fini(struct radeon_device *rdev)
		2229	{
		2230	int r;
		2231	if (rdev->ih.ring_obj) {
		2232	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		2233	if (likely(r == 0)) {
		2234	radeon_bo_kunmap(rdev->ih.ring_obj);
		2235	radeon_bo_unpin(rdev->ih.ring_obj);
		2236	radeon_bo_unreserve(rdev->ih.ring_obj);
		2237	}
		2238	radeon_bo_unref(&rdev->ih.ring_obj);
		2239	rdev->ih.ring = NULL;
		2240	rdev->ih.ring_obj = NULL;
		2241	}
		2242	}
		2243
		2244	void r600_rlc_stop(struct radeon_device *rdev)
		2245	{
		2246
		2247	if ((rdev->family >= CHIP_RV770) &&
		2248	(rdev->family <= CHIP_RV740)) {
		2249	/* r7xx asics need to soft reset RLC before halting */
		2250	WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
		2251	RREG32(SRBM_SOFT_RESET);
		2252	udelay(15000);
		2253	WREG32(SRBM_SOFT_RESET, 0);
		2254	RREG32(SRBM_SOFT_RESET);
		2255	}
		2256
		2257	WREG32(RLC_CNTL, 0);
		2258	}
		2259
		2260	static void r600_rlc_start(struct radeon_device *rdev)
		2261	{
		2262	WREG32(RLC_CNTL, RLC_ENABLE);
		2263	}
		2264
		2265	static int r600_rlc_init(struct radeon_device *rdev)
		2266	{
		2267	u32 i;
		2268	const __be32 *fw_data;
		2269
		2270	if (!rdev->rlc_fw)
		2271	return -EINVAL;
		2272
		2273	r600_rlc_stop(rdev);
		2274
		2275	WREG32(RLC_HB_BASE, 0);
		2276	WREG32(RLC_HB_CNTL, 0);
		2277	WREG32(RLC_HB_RPTR, 0);
		2278	WREG32(RLC_HB_WPTR, 0);
		2279	if (rdev->family <= CHIP_CAICOS) {
		2280	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
		2281	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
		2282	}
		2283	WREG32(RLC_MC_CNTL, 0);
		2284	WREG32(RLC_UCODE_CNTL, 0);
		2285
		2286	fw_data = (const __be32 *)rdev->rlc_fw->data;
		2287	if (rdev->family >= CHIP_CAYMAN) {
		2288	for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
		2289	WREG32(RLC_UCODE_ADDR, i);
		2290	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2291	}
		2292	} else if (rdev->family >= CHIP_CEDAR) {
		2293	for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
		2294	WREG32(RLC_UCODE_ADDR, i);
		2295	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2296	}
		2297	} else if (rdev->family >= CHIP_RV770) {
		2298	for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
		2299	WREG32(RLC_UCODE_ADDR, i);
		2300	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2301	}
		2302	} else {
		2303	for (i = 0; i < RLC_UCODE_SIZE; i++) {
		2304	WREG32(RLC_UCODE_ADDR, i);
		2305	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2306	}
		2307	}
		2308	WREG32(RLC_UCODE_ADDR, 0);
		2309
		2310	r600_rlc_start(rdev);
		2311
		2312	return 0;
		2313	}
		2314
		2315	static void r600_enable_interrupts(struct radeon_device *rdev)
		2316	{
		2317	u32 ih_cntl = RREG32(IH_CNTL);
		2318	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		2319
		2320	ih_cntl \|= ENABLE_INTR;
		2321	ih_rb_cntl \|= IH_RB_ENABLE;
		2322	WREG32(IH_CNTL, ih_cntl);
		2323	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2324	rdev->ih.enabled = true;
		2325	}
		2326
		2327	void r600_disable_interrupts(struct radeon_device *rdev)
		2328	{
		2329	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		2330	u32 ih_cntl = RREG32(IH_CNTL);
		2331
		2332	ih_rb_cntl &= ~IH_RB_ENABLE;
		2333	ih_cntl &= ~ENABLE_INTR;
		2334	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2335	WREG32(IH_CNTL, ih_cntl);
		2336	/* set rptr, wptr to 0 */
		2337	WREG32(IH_RB_RPTR, 0);
		2338	WREG32(IH_RB_WPTR, 0);
		2339	rdev->ih.enabled = false;
		2340	rdev->ih.wptr = 0;
		2341	rdev->ih.rptr = 0;
		2342	}
		2343
1963	serge	2344	static void r600_disable_interrupt_state(struct radeon_device *rdev)
		2345	{
		2346	u32 tmp;
1221	serge	2347
1963	serge	2348	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		2349	WREG32(GRBM_INT_CNTL, 0);
		2350	WREG32(DxMODE_INT_MASK, 0);
		2351	WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
		2352	WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
		2353	if (ASIC_IS_DCE3(rdev)) {
		2354	WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
		2355	WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
		2356	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2357	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2358	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2359	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2360	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2361	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2362	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2363	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2364	if (ASIC_IS_DCE32(rdev)) {
		2365	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2366	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2367	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2368	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2369	}
		2370	} else {
		2371	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		2372	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2373	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2374	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		2375	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2376	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		2377	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2378	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		2379	}
		2380	}
1221	serge	2381
2004	serge	2382	int r600_irq_init(struct radeon_device *rdev)
		2383	{
		2384	int ret = 0;
		2385	int rb_bufsz;
		2386	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
1221	serge	2387
2004	serge	2388	/* allocate ring */
		2389	ret = r600_ih_ring_alloc(rdev);
		2390	if (ret)
		2391	return ret;
1221	serge	2392
2004	serge	2393	/* disable irqs */
		2394	r600_disable_interrupts(rdev);
1221	serge	2395
2004	serge	2396	/* init rlc */
		2397	ret = r600_rlc_init(rdev);
		2398	if (ret) {
		2399	r600_ih_ring_fini(rdev);
		2400	return ret;
		2401	}
1221	serge	2402
2004	serge	2403	/* setup interrupt control */
		2404	/* set dummy read address to ring address */
		2405	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
		2406	interrupt_cntl = RREG32(INTERRUPT_CNTL);
		2407	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
		2408	* IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
		2409	*/
		2410	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
		2411	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
		2412	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
		2413	WREG32(INTERRUPT_CNTL, interrupt_cntl);
1221	serge	2414
2004	serge	2415	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
		2416	rb_bufsz = drm_order(rdev->ih.ring_size / 4);
1221	serge	2417
2004	serge	2418	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE \|
		2419	IH_WPTR_OVERFLOW_CLEAR \|
		2420	(rb_bufsz << 1));
1963	serge	2421
2004	serge	2422	if (rdev->wb.enabled)
		2423	ih_rb_cntl \|= IH_WPTR_WRITEBACK_ENABLE;
		2424
		2425	/* set the writeback address whether it's enabled or not */
		2426	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
		2427	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
		2428
		2429	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2430
		2431	/* set rptr, wptr to 0 */
		2432	WREG32(IH_RB_RPTR, 0);
		2433	WREG32(IH_RB_WPTR, 0);
		2434
		2435	/* Default settings for IH_CNTL (disabled at first) */
		2436	ih_cntl = MC_WRREQ_CREDIT(0x10) \| MC_WR_CLEAN_CNT(0x10);
		2437	/* RPTR_REARM only works if msi's are enabled */
		2438	if (rdev->msi_enabled)
		2439	ih_cntl \|= RPTR_REARM;
		2440
		2441	#ifdef __BIG_ENDIAN
		2442	ih_cntl \|= IH_MC_SWAP(IH_MC_SWAP_32BIT);
		2443	#endif
		2444	WREG32(IH_CNTL, ih_cntl);
		2445
		2446	/* force the active interrupt state to all disabled */
		2447	if (rdev->family >= CHIP_CEDAR)
		2448	evergreen_disable_interrupt_state(rdev);
		2449	else
		2450	r600_disable_interrupt_state(rdev);
		2451
		2452	/* enable irqs */
		2453	r600_enable_interrupts(rdev);
		2454
		2455	return ret;
		2456	}
		2457	int r600_irq_set(struct radeon_device *rdev)
		2458	{
		2459	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
		2460	u32 mode_int = 0;
		2461	u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
		2462	u32 grbm_int_cntl = 0;
		2463	u32 hdmi1, hdmi2;
		2464	u32 d1grph = 0, d2grph = 0;
		2465
		2466	if (!rdev->irq.installed) {
		2467	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		2468	return -EINVAL;
		2469	}
		2470	/* don't enable anything if the ih is disabled */
		2471	if (!rdev->ih.enabled) {
		2472	r600_disable_interrupts(rdev);
		2473	/* force the active interrupt state to all disabled */
		2474	r600_disable_interrupt_state(rdev);
		2475	return 0;
		2476	}
		2477
		2478	hdmi1 = RREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
		2479	if (ASIC_IS_DCE3(rdev)) {
		2480	hdmi2 = RREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
		2481	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2482	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2483	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2484	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2485	if (ASIC_IS_DCE32(rdev)) {
		2486	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2487	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2488	}
		2489	} else {
		2490	hdmi2 = RREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
		2491	hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2492	hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2493	hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2494	}
		2495
		2496	if (rdev->irq.sw_int) {
		2497	DRM_DEBUG("r600_irq_set: sw int\n");
		2498	cp_int_cntl \|= RB_INT_ENABLE;
		2499	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		2500	}
		2501	if (rdev->irq.crtc_vblank_int[0] \|\|
		2502	rdev->irq.pflip[0]) {
		2503	DRM_DEBUG("r600_irq_set: vblank 0\n");
		2504	mode_int \|= D1MODE_VBLANK_INT_MASK;
		2505	}
		2506	if (rdev->irq.crtc_vblank_int[1] \|\|
		2507	rdev->irq.pflip[1]) {
		2508	DRM_DEBUG("r600_irq_set: vblank 1\n");
		2509	mode_int \|= D2MODE_VBLANK_INT_MASK;
		2510	}
		2511	if (rdev->irq.hpd[0]) {
		2512	DRM_DEBUG("r600_irq_set: hpd 1\n");
		2513	hpd1 \|= DC_HPDx_INT_EN;
		2514	}
		2515	if (rdev->irq.hpd[1]) {
		2516	DRM_DEBUG("r600_irq_set: hpd 2\n");
		2517	hpd2 \|= DC_HPDx_INT_EN;
		2518	}
		2519	if (rdev->irq.hpd[2]) {
		2520	DRM_DEBUG("r600_irq_set: hpd 3\n");
		2521	hpd3 \|= DC_HPDx_INT_EN;
		2522	}
		2523	if (rdev->irq.hpd[3]) {
		2524	DRM_DEBUG("r600_irq_set: hpd 4\n");
		2525	hpd4 \|= DC_HPDx_INT_EN;
		2526	}
		2527	if (rdev->irq.hpd[4]) {
		2528	DRM_DEBUG("r600_irq_set: hpd 5\n");
		2529	hpd5 \|= DC_HPDx_INT_EN;
		2530	}
		2531	if (rdev->irq.hpd[5]) {
		2532	DRM_DEBUG("r600_irq_set: hpd 6\n");
		2533	hpd6 \|= DC_HPDx_INT_EN;
		2534	}
		2535	if (rdev->irq.hdmi[0]) {
		2536	DRM_DEBUG("r600_irq_set: hdmi 1\n");
		2537	hdmi1 \|= R600_HDMI_INT_EN;
		2538	}
		2539	if (rdev->irq.hdmi[1]) {
		2540	DRM_DEBUG("r600_irq_set: hdmi 2\n");
		2541	hdmi2 \|= R600_HDMI_INT_EN;
		2542	}
		2543	if (rdev->irq.gui_idle) {
		2544	DRM_DEBUG("gui idle\n");
		2545	grbm_int_cntl \|= GUI_IDLE_INT_ENABLE;
		2546	}
		2547
		2548	WREG32(CP_INT_CNTL, cp_int_cntl);
		2549	WREG32(DxMODE_INT_MASK, mode_int);
		2550	WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
		2551	WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
		2552	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		2553	WREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, hdmi1);
		2554	if (ASIC_IS_DCE3(rdev)) {
		2555	WREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, hdmi2);
		2556	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		2557	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		2558	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		2559	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		2560	if (ASIC_IS_DCE32(rdev)) {
		2561	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		2562	WREG32(DC_HPD6_INT_CONTROL, hpd6);
		2563	}
		2564	} else {
		2565	WREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, hdmi2);
		2566	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
		2567	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
		2568	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
		2569	}
		2570
		2571	return 0;
		2572	}
		2573
		2574	static inline void r600_irq_ack(struct radeon_device *rdev)
		2575	{
		2576	u32 tmp;
		2577
		2578	if (ASIC_IS_DCE3(rdev)) {
		2579	rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
		2580	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
		2581	rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
		2582	} else {
		2583	rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		2584	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		2585	rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
		2586	}
		2587	rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
		2588	rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
		2589
		2590	if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		2591	WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		2592	if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		2593	WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		2594	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
		2595	WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		2596	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
		2597	WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		2598	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
		2599	WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		2600	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
		2601	WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		2602	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		2603	if (ASIC_IS_DCE3(rdev)) {
		2604	tmp = RREG32(DC_HPD1_INT_CONTROL);
		2605	tmp \|= DC_HPDx_INT_ACK;
		2606	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2607	} else {
		2608	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		2609	tmp \|= DC_HPDx_INT_ACK;
		2610	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		2611	}
		2612	}
		2613	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		2614	if (ASIC_IS_DCE3(rdev)) {
		2615	tmp = RREG32(DC_HPD2_INT_CONTROL);
		2616	tmp \|= DC_HPDx_INT_ACK;
		2617	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2618	} else {
		2619	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		2620	tmp \|= DC_HPDx_INT_ACK;
		2621	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		2622	}
		2623	}
		2624	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		2625	if (ASIC_IS_DCE3(rdev)) {
		2626	tmp = RREG32(DC_HPD3_INT_CONTROL);
		2627	tmp \|= DC_HPDx_INT_ACK;
		2628	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2629	} else {
		2630	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		2631	tmp \|= DC_HPDx_INT_ACK;
		2632	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		2633	}
		2634	}
		2635	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		2636	tmp = RREG32(DC_HPD4_INT_CONTROL);
		2637	tmp \|= DC_HPDx_INT_ACK;
		2638	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2639	}
		2640	if (ASIC_IS_DCE32(rdev)) {
		2641	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		2642	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2643	tmp \|= DC_HPDx_INT_ACK;
		2644	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2645	}
		2646	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		2647	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2648	tmp \|= DC_HPDx_INT_ACK;
		2649	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2650	}
		2651	}
		2652	if (RREG32(R600_HDMI_BLOCK1 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
		2653	WREG32_P(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
		2654	}
		2655	if (ASIC_IS_DCE3(rdev)) {
		2656	if (RREG32(R600_HDMI_BLOCK3 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
		2657	WREG32_P(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
		2658	}
		2659	} else {
		2660	if (RREG32(R600_HDMI_BLOCK2 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
		2661	WREG32_P(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
		2662	}
		2663	}
		2664	}
		2665
		2666	static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
		2667	{
		2668	u32 wptr, tmp;
		2669
		2670	if (rdev->wb.enabled)
		2671	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		2672	else
		2673	wptr = RREG32(IH_RB_WPTR);
		2674
		2675	if (wptr & RB_OVERFLOW) {
		2676	/* When a ring buffer overflow happen start parsing interrupt
		2677	* from the last not overwritten vector (wptr + 16). Hopefully
		2678	* this should allow us to catchup.
		2679	*/
		2680	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
		2681	wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
		2682	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		2683	tmp = RREG32(IH_RB_CNTL);
		2684	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		2685	WREG32(IH_RB_CNTL, tmp);
		2686	}
		2687	return (wptr & rdev->ih.ptr_mask);
		2688	}
		2689
		2690	/* r600 IV Ring
		2691	* Each IV ring entry is 128 bits:
		2692	* [7:0] - interrupt source id
		2693	* [31:8] - reserved
		2694	* [59:32] - interrupt source data
		2695	* [127:60] - reserved
		2696	*
		2697	* The basic interrupt vector entries
		2698	* are decoded as follows:
		2699	* src_id src_data description
		2700	* 1 0 D1 Vblank
		2701	* 1 1 D1 Vline
		2702	* 5 0 D2 Vblank
		2703	* 5 1 D2 Vline
		2704	* 19 0 FP Hot plug detection A
		2705	* 19 1 FP Hot plug detection B
		2706	* 19 2 DAC A auto-detection
		2707	* 19 3 DAC B auto-detection
		2708	* 21 4 HDMI block A
		2709	* 21 5 HDMI block B
		2710	* 176 - CP_INT RB
		2711	* 177 - CP_INT IB1
		2712	* 178 - CP_INT IB2
		2713	* 181 - EOP Interrupt
		2714	* 233 - GUI Idle
		2715	*
		2716	* Note, these are based on r600 and may need to be
		2717	* adjusted or added to on newer asics
		2718	*/
		2719
		2720	int r600_irq_process(struct radeon_device *rdev)
		2721	{
		2722	u32 wptr;
		2723	u32 rptr;
		2724	u32 src_id, src_data;
		2725	u32 ring_index;
		2726	unsigned long flags;
		2727	bool queue_hotplug = false;
		2728
		2729	if (!rdev->ih.enabled \|\| rdev->shutdown)
		2730	return IRQ_NONE;
		2731
		2732	wptr = r600_get_ih_wptr(rdev);
		2733	rptr = rdev->ih.rptr;
		2734	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		2735
		2736	spin_lock_irqsave(&rdev->ih.lock, flags);
		2737
		2738	if (rptr == wptr) {
		2739	spin_unlock_irqrestore(&rdev->ih.lock, flags);
		2740	return IRQ_NONE;
		2741	}
		2742
		2743	restart_ih:
		2744	/* display interrupts */
		2745	r600_irq_ack(rdev);
		2746
		2747	rdev->ih.wptr = wptr;
		2748	while (rptr != wptr) {
		2749	/* wptr/rptr are in bytes! */
		2750	ring_index = rptr / 4;
		2751	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		2752	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		2753
		2754	switch (src_id) {
		2755	case 1: /* D1 vblank/vline */
		2756	switch (src_data) {
		2757	case 0: /* D1 vblank */
		2758	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
		2759	if (rdev->irq.crtc_vblank_int[0]) {
		2760	// drm_handle_vblank(rdev->ddev, 0);
		2761	rdev->pm.vblank_sync = true;
		2762	// wake_up(&rdev->irq.vblank_queue);
		2763	}
		2764	// if (rdev->irq.pflip[0])
		2765	// radeon_crtc_handle_flip(rdev, 0);
		2766	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		2767	DRM_DEBUG("IH: D1 vblank\n");
		2768	}
		2769	break;
		2770	case 1: /* D1 vline */
		2771	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
		2772	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		2773	DRM_DEBUG("IH: D1 vline\n");
		2774	}
		2775	break;
		2776	default:
		2777	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2778	break;
		2779	}
		2780	break;
		2781	case 5: /* D2 vblank/vline */
		2782	switch (src_data) {
		2783	case 0: /* D2 vblank */
		2784	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
		2785	if (rdev->irq.crtc_vblank_int[1]) {
		2786	// drm_handle_vblank(rdev->ddev, 1);
		2787	rdev->pm.vblank_sync = true;
		2788	// wake_up(&rdev->irq.vblank_queue);
		2789	}
		2790	// if (rdev->irq.pflip[1])
		2791	// radeon_crtc_handle_flip(rdev, 1);
		2792	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
		2793	DRM_DEBUG("IH: D2 vblank\n");
		2794	}
		2795	break;
		2796	case 1: /* D1 vline */
		2797	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
		2798	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
		2799	DRM_DEBUG("IH: D2 vline\n");
		2800	}
		2801	break;
		2802	default:
		2803	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2804	break;
		2805	}
		2806	break;
		2807	case 19: /* HPD/DAC hotplug */
		2808	switch (src_data) {
		2809	case 0:
		2810	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		2811	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
		2812	queue_hotplug = true;
		2813	DRM_DEBUG("IH: HPD1\n");
		2814	}
		2815	break;
		2816	case 1:
		2817	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		2818	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
		2819	queue_hotplug = true;
		2820	DRM_DEBUG("IH: HPD2\n");
		2821	}
		2822	break;
		2823	case 4:
		2824	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		2825	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
		2826	queue_hotplug = true;
		2827	DRM_DEBUG("IH: HPD3\n");
		2828	}
		2829	break;
		2830	case 5:
		2831	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		2832	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
		2833	queue_hotplug = true;
		2834	DRM_DEBUG("IH: HPD4\n");
		2835	}
		2836	break;
		2837	case 10:
		2838	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		2839	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
		2840	queue_hotplug = true;
		2841	DRM_DEBUG("IH: HPD5\n");
		2842	}
		2843	break;
		2844	case 12:
		2845	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		2846	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
		2847	queue_hotplug = true;
		2848	DRM_DEBUG("IH: HPD6\n");
		2849	}
		2850	break;
		2851	default:
		2852	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2853	break;
		2854	}
		2855	break;
		2856	case 21: /* HDMI */
		2857	DRM_DEBUG("IH: HDMI: 0x%x\n", src_data);
		2858	// r600_audio_schedule_polling(rdev);
		2859	break;
		2860	case 176: /* CP_INT in ring buffer */
		2861	case 177: /* CP_INT in IB1 */
		2862	case 178: /* CP_INT in IB2 */
		2863	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2005	serge	2864	radeon_fence_process(rdev);
2004	serge	2865	break;
		2866	case 181: /* CP EOP event */
		2867	DRM_DEBUG("IH: CP EOP\n");
2005	serge	2868	radeon_fence_process(rdev);
2004	serge	2869	break;
		2870	case 233: /* GUI IDLE */
		2871	DRM_DEBUG("IH: GUI idle\n");
		2872	rdev->pm.gui_idle = true;
		2873	// wake_up(&rdev->irq.idle_queue);
		2874	break;
		2875	default:
		2876	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2877	break;
		2878	}
		2879
		2880	/* wptr/rptr are in bytes! */
		2881	rptr += 16;
		2882	rptr &= rdev->ih.ptr_mask;
		2883	}
		2884	/* make sure wptr hasn't changed while processing */
		2885	wptr = r600_get_ih_wptr(rdev);
		2886	if (wptr != rdev->ih.wptr)
		2887	goto restart_ih;
		2888	// if (queue_hotplug)
		2889	// schedule_work(&rdev->hotplug_work);
		2890	rdev->ih.rptr = rptr;
		2891	WREG32(IH_RB_RPTR, rdev->ih.rptr);
		2892	spin_unlock_irqrestore(&rdev->ih.lock, flags);
		2893	return IRQ_HANDLED;
		2894	}
		2895
1221	serge	2896	/*
		2897	* Debugfs info
		2898	*/
		2899	#if defined(CONFIG_DEBUG_FS)
		2900
		2901	static int r600_debugfs_cp_ring_info(struct seq_file m, void data)
		2902	{
		2903	struct drm_info_node node = (struct drm_info_node ) m->private;
		2904	struct drm_device *dev = node->minor->dev;
		2905	struct radeon_device *rdev = dev->dev_private;
		2906	unsigned count, i, j;
		2907
		2908	radeon_ring_free_size(rdev);
1321	serge	2909	count = (rdev->cp.ring_size / 4) - rdev->cp.ring_free_dw;
1221	serge	2910	seq_printf(m, "CP_STAT 0x%08x\n", RREG32(CP_STAT));
1321	serge	2911	seq_printf(m, "CP_RB_WPTR 0x%08x\n", RREG32(CP_RB_WPTR));
		2912	seq_printf(m, "CP_RB_RPTR 0x%08x\n", RREG32(CP_RB_RPTR));
		2913	seq_printf(m, "driver's copy of the CP_RB_WPTR 0x%08x\n", rdev->cp.wptr);
		2914	seq_printf(m, "driver's copy of the CP_RB_RPTR 0x%08x\n", rdev->cp.rptr);
1221	serge	2915	seq_printf(m, "%u free dwords in ring\n", rdev->cp.ring_free_dw);
		2916	seq_printf(m, "%u dwords in ring\n", count);
1321	serge	2917	i = rdev->cp.rptr;
1221	serge	2918	for (j = 0; j <= count; j++) {
		2919	seq_printf(m, "r[%04d]=0x%08x\n", i, rdev->cp.ring[i]);
1321	serge	2920	i = (i + 1) & rdev->cp.ptr_mask;
1221	serge	2921	}
		2922	return 0;
		2923	}
		2924
		2925	static int r600_debugfs_mc_info(struct seq_file m, void data)
		2926	{
		2927	struct drm_info_node node = (struct drm_info_node ) m->private;
		2928	struct drm_device *dev = node->minor->dev;
		2929	struct radeon_device *rdev = dev->dev_private;
		2930
		2931	DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
		2932	DREG32_SYS(m, rdev, VM_L2_STATUS);
		2933	return 0;
		2934	}
		2935
		2936	static struct drm_info_list r600_mc_info_list[] = {
		2937	{"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
		2938	{"r600_ring_info", r600_debugfs_cp_ring_info, 0, NULL},
		2939	};
		2940	#endif
		2941
		2942	int r600_debugfs_mc_info_init(struct radeon_device *rdev)
		2943	{
		2944	#if defined(CONFIG_DEBUG_FS)
		2945	return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
		2946	#else
		2947	return 0;
		2948	#endif
		2949	}
1404	serge	2950
		2951	/**
		2952	* r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
		2953	* rdev: radeon device structure
		2954	* bo: buffer object struct which userspace is waiting for idle
		2955	*
		2956	* Some R6XX/R7XX doesn't seems to take into account HDP flush performed
		2957	* through ring buffer, this leads to corruption in rendering, see
		2958	* http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
		2959	* directly perform HDP flush by writing register through MMIO.
		2960	*/
		2961	void r600_ioctl_wait_idle(struct radeon_device rdev, struct radeon_bo bo)
		2962	{
1963	serge	2963	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		2964	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
		2965	* This seems to cause problems on some AGP cards. Just use the old
		2966	* method for them.
		2967	*/
		2968	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		2969	rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
		2970	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		2971	u32 tmp;
		2972
		2973	WREG32(HDP_DEBUG1, 0);
		2974	tmp = readl((void __iomem *)ptr);
		2975	} else
1404	serge	2976	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		2977	}
1963	serge	2978
		2979	void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
		2980	{
		2981	u32 link_width_cntl, mask, target_reg;
		2982
		2983	if (rdev->flags & RADEON_IS_IGP)
		2984	return;
		2985
		2986	if (!(rdev->flags & RADEON_IS_PCIE))
		2987	return;
		2988
		2989	/* x2 cards have a special sequence */
		2990	if (ASIC_IS_X2(rdev))
		2991	return;
		2992
		2993	/* FIXME wait for idle */
		2994
		2995	switch (lanes) {
		2996	case 0:
		2997	mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
		2998	break;
		2999	case 1:
		3000	mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
		3001	break;
		3002	case 2:
		3003	mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
		3004	break;
		3005	case 4:
		3006	mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
		3007	break;
		3008	case 8:
		3009	mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
		3010	break;
		3011	case 12:
		3012	mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
		3013	break;
		3014	case 16:
		3015	default:
		3016	mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
		3017	break;
		3018	}
		3019
		3020	link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		3021
		3022	if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
		3023	(mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
		3024	return;
		3025
		3026	if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
		3027	return;
		3028
		3029	link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK \|
		3030	RADEON_PCIE_LC_RECONFIG_NOW \|
		3031	R600_PCIE_LC_RENEGOTIATE_EN \|
		3032	R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
		3033	link_width_cntl \|= mask;
		3034
		3035	WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3036
2005	serge	3037	/* some northbridges can renegotiate the link rather than requiring
		3038	* a complete re-config.
		3039	* e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)
1963	serge	3040	*/
		3041	if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
		3042	link_width_cntl \|= R600_PCIE_LC_RENEGOTIATE_EN \| R600_PCIE_LC_UPCONFIGURE_SUPPORT;
		3043	else
		3044	link_width_cntl \|= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
		3045
		3046	WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl \|
		3047	RADEON_PCIE_LC_RECONFIG_NOW));
		3048
		3049	if (rdev->family >= CHIP_RV770)
		3050	target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
		3051	else
		3052	target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
		3053
		3054	/* wait for lane set to complete */
		3055	link_width_cntl = RREG32(target_reg);
		3056	while (link_width_cntl == 0xffffffff)
		3057	link_width_cntl = RREG32(target_reg);
		3058
		3059	}
		3060
		3061	int r600_get_pcie_lanes(struct radeon_device *rdev)
		3062	{
		3063	u32 link_width_cntl;
		3064
		3065	if (rdev->flags & RADEON_IS_IGP)
		3066	return 0;
		3067
		3068	if (!(rdev->flags & RADEON_IS_PCIE))
		3069	return 0;
		3070
		3071	/* x2 cards have a special sequence */
		3072	if (ASIC_IS_X2(rdev))
		3073	return 0;
		3074
		3075	/* FIXME wait for idle */
		3076
		3077	link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		3078
		3079	switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
		3080	case RADEON_PCIE_LC_LINK_WIDTH_X0:
		3081	return 0;
		3082	case RADEON_PCIE_LC_LINK_WIDTH_X1:
		3083	return 1;
		3084	case RADEON_PCIE_LC_LINK_WIDTH_X2:
		3085	return 2;
		3086	case RADEON_PCIE_LC_LINK_WIDTH_X4:
		3087	return 4;
		3088	case RADEON_PCIE_LC_LINK_WIDTH_X8:
		3089	return 8;
		3090	case RADEON_PCIE_LC_LINK_WIDTH_X16:
		3091	default:
		3092	return 16;
		3093	}
		3094	}
		3095
		3096	static void r600_pcie_gen2_enable(struct radeon_device *rdev)
		3097	{
		3098	u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
		3099	u16 link_cntl2;
		3100
		3101	if (radeon_pcie_gen2 == 0)
		3102	return;
		3103
		3104	if (rdev->flags & RADEON_IS_IGP)
		3105	return;
		3106
		3107	if (!(rdev->flags & RADEON_IS_PCIE))
		3108	return;
		3109
		3110	/* x2 cards have a special sequence */
		3111	if (ASIC_IS_X2(rdev))
		3112	return;
		3113
		3114	/* only RV6xx+ chips are supported */
		3115	if (rdev->family <= CHIP_R600)
		3116	return;
		3117
		3118	/* 55 nm r6xx asics */
		3119	if ((rdev->family == CHIP_RV670) \|\|
		3120	(rdev->family == CHIP_RV620) \|\|
		3121	(rdev->family == CHIP_RV635)) {
		3122	/* advertise upconfig capability */
		3123	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3124	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3125	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3126	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3127	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
		3128	lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
		3129	link_width_cntl &= ~(LC_LINK_WIDTH_MASK \|
		3130	LC_RECONFIG_ARC_MISSING_ESCAPE);
		3131	link_width_cntl \|= lanes \| LC_RECONFIG_NOW \| LC_RENEGOTIATE_EN;
		3132	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3133	} else {
		3134	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3135	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3136	}
		3137	}
		3138
		3139	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3140	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
		3141	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		3142
		3143	/* 55 nm r6xx asics */
		3144	if ((rdev->family == CHIP_RV670) \|\|
		3145	(rdev->family == CHIP_RV620) \|\|
		3146	(rdev->family == CHIP_RV635)) {
		3147	WREG32(MM_CFGREGS_CNTL, 0x8);
		3148	link_cntl2 = RREG32(0x4088);
		3149	WREG32(MM_CFGREGS_CNTL, 0);
		3150	/* not supported yet */
		3151	if (link_cntl2 & SELECTABLE_DEEMPHASIS)
		3152	return;
		3153	}
		3154
		3155	speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
		3156	speed_cntl \|= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
		3157	speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
		3158	speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
		3159	speed_cntl \|= LC_FORCE_EN_HW_SPEED_CHANGE;
		3160	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3161
		3162	tmp = RREG32(0x541c);
		3163	WREG32(0x541c, tmp \| 0x8);
		3164	WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
		3165	link_cntl2 = RREG16(0x4088);
		3166	link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
		3167	link_cntl2 \|= 0x2;
		3168	WREG16(0x4088, link_cntl2);
		3169	WREG32(MM_CFGREGS_CNTL, 0);
		3170
		3171	if ((rdev->family == CHIP_RV670) \|\|
		3172	(rdev->family == CHIP_RV620) \|\|
		3173	(rdev->family == CHIP_RV635)) {
		3174	training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
		3175	training_cntl &= ~LC_POINT_7_PLUS_EN;
		3176	WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
		3177	} else {
		3178	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3179	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		3180	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3181	}
		3182
		3183	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3184	speed_cntl \|= LC_GEN2_EN_STRAP;
		3185	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3186
		3187	} else {
		3188	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3189	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		3190	if (1)
		3191	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3192	else
		3193	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3194	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3195	}
		3196	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/r600.c – Rev 2005