WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/r600.c

Rev	Author	Line No.	Line
1128	serge	1	/*
		2	* Copyright 2008 Advanced Micro Devices, Inc.
		3	* Copyright 2008 Red Hat Inc.
		4	* Copyright 2009 Jerome Glisse.
		5	*
		6	* Permission is hereby granted, free of charge, to any person obtaining a
		7	* copy of this software and associated documentation files (the "Software"),
		8	* to deal in the Software without restriction, including without limitation
		9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		10	* and/or sell copies of the Software, and to permit persons to whom the
		11	* Software is furnished to do so, subject to the following conditions:
		12	*
		13	* The above copyright notice and this permission notice shall be included in
		14	* all copies or substantial portions of the Software.
		15	*
		16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		22	* OTHER DEALINGS IN THE SOFTWARE.
		23	*
		24	* Authors: Dave Airlie
		25	* Alex Deucher
		26	* Jerome Glisse
		27	*/
1963	serge	28	#include
1233	serge	29	#include
1221	serge	30	#include
1128	serge	31	#include "drmP.h"
1221	serge	32	#include "radeon_drm.h"
1128	serge	33	#include "radeon.h"
1963	serge	34	#include "radeon_asic.h"
1221	serge	35	#include "radeon_mode.h"
		36	#include "r600d.h"
		37	#include "atom.h"
		38	#include "avivod.h"
1128	serge	39
1221	serge	40	#define PFP_UCODE_SIZE 576
		41	#define PM4_UCODE_SIZE 1792
1321	serge	42	#define RLC_UCODE_SIZE 768
1221	serge	43	#define R700_PFP_UCODE_SIZE 848
		44	#define R700_PM4_UCODE_SIZE 1360
1321	serge	45	#define R700_RLC_UCODE_SIZE 1024
1963	serge	46	#define EVERGREEN_PFP_UCODE_SIZE 1120
		47	#define EVERGREEN_PM4_UCODE_SIZE 1376
		48	#define EVERGREEN_RLC_UCODE_SIZE 768
		49	#define CAYMAN_RLC_UCODE_SIZE 1024
1128	serge	50
1221	serge	51	/* Firmware Names */
		52	MODULE_FIRMWARE("radeon/R600_pfp.bin");
		53	MODULE_FIRMWARE("radeon/R600_me.bin");
		54	MODULE_FIRMWARE("radeon/RV610_pfp.bin");
		55	MODULE_FIRMWARE("radeon/RV610_me.bin");
		56	MODULE_FIRMWARE("radeon/RV630_pfp.bin");
		57	MODULE_FIRMWARE("radeon/RV630_me.bin");
		58	MODULE_FIRMWARE("radeon/RV620_pfp.bin");
		59	MODULE_FIRMWARE("radeon/RV620_me.bin");
		60	MODULE_FIRMWARE("radeon/RV635_pfp.bin");
		61	MODULE_FIRMWARE("radeon/RV635_me.bin");
		62	MODULE_FIRMWARE("radeon/RV670_pfp.bin");
		63	MODULE_FIRMWARE("radeon/RV670_me.bin");
		64	MODULE_FIRMWARE("radeon/RS780_pfp.bin");
		65	MODULE_FIRMWARE("radeon/RS780_me.bin");
		66	MODULE_FIRMWARE("radeon/RV770_pfp.bin");
		67	MODULE_FIRMWARE("radeon/RV770_me.bin");
		68	MODULE_FIRMWARE("radeon/RV730_pfp.bin");
		69	MODULE_FIRMWARE("radeon/RV730_me.bin");
		70	MODULE_FIRMWARE("radeon/RV710_pfp.bin");
		71	MODULE_FIRMWARE("radeon/RV710_me.bin");
1321	serge	72	MODULE_FIRMWARE("radeon/R600_rlc.bin");
		73	MODULE_FIRMWARE("radeon/R700_rlc.bin");
1963	serge	74	MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
		75	MODULE_FIRMWARE("radeon/CEDAR_me.bin");
		76	MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
		77	MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
		78	MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
		79	MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
		80	MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
		81	MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
		82	MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
		83	MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
		84	MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
		85	MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
		86	MODULE_FIRMWARE("radeon/PALM_pfp.bin");
		87	MODULE_FIRMWARE("radeon/PALM_me.bin");
		88	MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
		89	MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
		90	MODULE_FIRMWARE("radeon/SUMO_me.bin");
		91	MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
		92	MODULE_FIRMWARE("radeon/SUMO2_me.bin");
1221	serge	93
		94	int r600_debugfs_mc_info_init(struct radeon_device *rdev);
		95
		96	/* r600,rv610,rv630,rv620,rv635,rv670 */
1128	serge	97	int r600_mc_wait_for_idle(struct radeon_device *rdev);
		98	void r600_gpu_init(struct radeon_device *rdev);
1221	serge	99	void r600_fini(struct radeon_device *rdev);
1963	serge	100	void r600_irq_disable(struct radeon_device *rdev);
		101	static void r600_pcie_gen2_enable(struct radeon_device *rdev);
1128	serge	102
1963	serge	103	/* get temperature in millidegrees */
		104	int rv6xx_get_temp(struct radeon_device *rdev)
		105	{
		106	u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
		107	ASIC_T_SHIFT;
		108	int actual_temp = temp & 0xff;
		109
		110	if (temp & 0x100)
		111	actual_temp -= 256;
		112
		113	return actual_temp * 1000;
		114	}
		115
		116
		117
		118
		119
		120
		121	bool r600_gui_idle(struct radeon_device *rdev)
		122	{
		123	if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
		124	return false;
		125	else
		126	return true;
		127	}
		128
1321	serge	129	/* hpd for digital panel detect/disconnect */
		130	bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		131	{
		132	bool connected = false;
		133
		134	if (ASIC_IS_DCE3(rdev)) {
		135	switch (hpd) {
		136	case RADEON_HPD_1:
		137	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		138	connected = true;
		139	break;
		140	case RADEON_HPD_2:
		141	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		142	connected = true;
		143	break;
		144	case RADEON_HPD_3:
		145	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		146	connected = true;
		147	break;
		148	case RADEON_HPD_4:
		149	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		150	connected = true;
		151	break;
		152	/* DCE 3.2 */
		153	case RADEON_HPD_5:
		154	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		155	connected = true;
		156	break;
		157	case RADEON_HPD_6:
		158	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		159	connected = true;
		160	break;
		161	default:
		162	break;
		163	}
		164	} else {
		165	switch (hpd) {
		166	case RADEON_HPD_1:
		167	if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		168	connected = true;
		169	break;
		170	case RADEON_HPD_2:
		171	if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		172	connected = true;
		173	break;
		174	case RADEON_HPD_3:
		175	if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		176	connected = true;
		177	break;
		178	default:
		179	break;
		180	}
		181	}
		182	return connected;
		183	}
		184
		185	void r600_hpd_set_polarity(struct radeon_device *rdev,
		186	enum radeon_hpd_id hpd)
		187	{
		188	u32 tmp;
		189	bool connected = r600_hpd_sense(rdev, hpd);
		190
		191	if (ASIC_IS_DCE3(rdev)) {
		192	switch (hpd) {
		193	case RADEON_HPD_1:
		194	tmp = RREG32(DC_HPD1_INT_CONTROL);
		195	if (connected)
		196	tmp &= ~DC_HPDx_INT_POLARITY;
		197	else
		198	tmp \|= DC_HPDx_INT_POLARITY;
		199	WREG32(DC_HPD1_INT_CONTROL, tmp);
		200	break;
		201	case RADEON_HPD_2:
		202	tmp = RREG32(DC_HPD2_INT_CONTROL);
		203	if (connected)
		204	tmp &= ~DC_HPDx_INT_POLARITY;
		205	else
		206	tmp \|= DC_HPDx_INT_POLARITY;
		207	WREG32(DC_HPD2_INT_CONTROL, tmp);
		208	break;
		209	case RADEON_HPD_3:
		210	tmp = RREG32(DC_HPD3_INT_CONTROL);
		211	if (connected)
		212	tmp &= ~DC_HPDx_INT_POLARITY;
		213	else
		214	tmp \|= DC_HPDx_INT_POLARITY;
		215	WREG32(DC_HPD3_INT_CONTROL, tmp);
		216	break;
		217	case RADEON_HPD_4:
		218	tmp = RREG32(DC_HPD4_INT_CONTROL);
		219	if (connected)
		220	tmp &= ~DC_HPDx_INT_POLARITY;
		221	else
		222	tmp \|= DC_HPDx_INT_POLARITY;
		223	WREG32(DC_HPD4_INT_CONTROL, tmp);
		224	break;
		225	case RADEON_HPD_5:
		226	tmp = RREG32(DC_HPD5_INT_CONTROL);
		227	if (connected)
		228	tmp &= ~DC_HPDx_INT_POLARITY;
		229	else
		230	tmp \|= DC_HPDx_INT_POLARITY;
		231	WREG32(DC_HPD5_INT_CONTROL, tmp);
		232	break;
		233	/* DCE 3.2 */
		234	case RADEON_HPD_6:
		235	tmp = RREG32(DC_HPD6_INT_CONTROL);
		236	if (connected)
		237	tmp &= ~DC_HPDx_INT_POLARITY;
		238	else
		239	tmp \|= DC_HPDx_INT_POLARITY;
		240	WREG32(DC_HPD6_INT_CONTROL, tmp);
		241	break;
		242	default:
		243	break;
		244	}
		245	} else {
		246	switch (hpd) {
		247	case RADEON_HPD_1:
		248	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		249	if (connected)
		250	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		251	else
		252	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		253	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		254	break;
		255	case RADEON_HPD_2:
		256	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		257	if (connected)
		258	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		259	else
		260	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		261	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		262	break;
		263	case RADEON_HPD_3:
		264	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		265	if (connected)
		266	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		267	else
		268	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		269	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		270	break;
		271	default:
		272	break;
		273	}
		274	}
		275	}
		276
		277	void r600_hpd_init(struct radeon_device *rdev)
		278	{
		279	struct drm_device *dev = rdev->ddev;
		280	struct drm_connector *connector;
		281
		282	if (ASIC_IS_DCE3(rdev)) {
		283	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \| DC_HPDx_RX_INT_TIMER(0xfa);
		284	if (ASIC_IS_DCE32(rdev))
		285	tmp \|= DC_HPDx_EN;
		286
		287	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		288	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		289	switch (radeon_connector->hpd.hpd) {
		290	case RADEON_HPD_1:
		291	WREG32(DC_HPD1_CONTROL, tmp);
2004	serge	292	rdev->irq.hpd[0] = true;
1321	serge	293	break;
		294	case RADEON_HPD_2:
		295	WREG32(DC_HPD2_CONTROL, tmp);
2004	serge	296	rdev->irq.hpd[1] = true;
1321	serge	297	break;
		298	case RADEON_HPD_3:
		299	WREG32(DC_HPD3_CONTROL, tmp);
2004	serge	300	rdev->irq.hpd[2] = true;
1321	serge	301	break;
		302	case RADEON_HPD_4:
		303	WREG32(DC_HPD4_CONTROL, tmp);
2004	serge	304	rdev->irq.hpd[3] = true;
1321	serge	305	break;
		306	/* DCE 3.2 */
		307	case RADEON_HPD_5:
		308	WREG32(DC_HPD5_CONTROL, tmp);
2004	serge	309	rdev->irq.hpd[4] = true;
1321	serge	310	break;
		311	case RADEON_HPD_6:
		312	WREG32(DC_HPD6_CONTROL, tmp);
2004	serge	313	rdev->irq.hpd[5] = true;
1321	serge	314	break;
		315	default:
		316	break;
		317	}
		318	}
		319	} else {
		320	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		321	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		322	switch (radeon_connector->hpd.hpd) {
		323	case RADEON_HPD_1:
		324	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
2004	serge	325	rdev->irq.hpd[0] = true;
1321	serge	326	break;
		327	case RADEON_HPD_2:
		328	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
2004	serge	329	rdev->irq.hpd[1] = true;
1321	serge	330	break;
		331	case RADEON_HPD_3:
		332	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
2004	serge	333	rdev->irq.hpd[2] = true;
1321	serge	334	break;
		335	default:
		336	break;
		337	}
		338	}
		339	}
2004	serge	340	if (rdev->irq.installed)
		341	r600_irq_set(rdev);
1321	serge	342	}
		343
		344	void r600_hpd_fini(struct radeon_device *rdev)
		345	{
		346	struct drm_device *dev = rdev->ddev;
		347	struct drm_connector *connector;
		348
		349	if (ASIC_IS_DCE3(rdev)) {
		350	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		351	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		352	switch (radeon_connector->hpd.hpd) {
		353	case RADEON_HPD_1:
		354	WREG32(DC_HPD1_CONTROL, 0);
2004	serge	355	rdev->irq.hpd[0] = false;
1321	serge	356	break;
		357	case RADEON_HPD_2:
		358	WREG32(DC_HPD2_CONTROL, 0);
2004	serge	359	rdev->irq.hpd[1] = false;
1321	serge	360	break;
		361	case RADEON_HPD_3:
		362	WREG32(DC_HPD3_CONTROL, 0);
2004	serge	363	rdev->irq.hpd[2] = false;
1321	serge	364	break;
		365	case RADEON_HPD_4:
		366	WREG32(DC_HPD4_CONTROL, 0);
2004	serge	367	rdev->irq.hpd[3] = false;
1321	serge	368	break;
		369	/* DCE 3.2 */
		370	case RADEON_HPD_5:
		371	WREG32(DC_HPD5_CONTROL, 0);
2004	serge	372	rdev->irq.hpd[4] = false;
1321	serge	373	break;
		374	case RADEON_HPD_6:
		375	WREG32(DC_HPD6_CONTROL, 0);
2004	serge	376	rdev->irq.hpd[5] = false;
1321	serge	377	break;
		378	default:
		379	break;
		380	}
		381	}
		382	} else {
		383	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		384	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		385	switch (radeon_connector->hpd.hpd) {
		386	case RADEON_HPD_1:
		387	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
2004	serge	388	rdev->irq.hpd[0] = false;
1321	serge	389	break;
		390	case RADEON_HPD_2:
		391	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
2004	serge	392	rdev->irq.hpd[1] = false;
1321	serge	393	break;
		394	case RADEON_HPD_3:
		395	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
2004	serge	396	rdev->irq.hpd[2] = false;
1321	serge	397	break;
		398	default:
		399	break;
		400	}
		401	}
		402	}
		403	}
		404
1128	serge	405	/*
1221	serge	406	* R600 PCIE GART
1128	serge	407	*/
1221	serge	408	void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
		409	{
		410	unsigned i;
		411	u32 tmp;
1128	serge	412
1430	serge	413	/* flush hdp cache so updates hit vram */
1963	serge	414	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		415	!(rdev->flags & RADEON_IS_AGP)) {
		416	void __iomem ptr = (void )rdev->gart.table.vram.ptr;
		417	u32 tmp;
		418
		419	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		420	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
		421	* This seems to cause problems on some AGP cards. Just use the old
		422	* method for them.
		423	*/
		424	WREG32(HDP_DEBUG1, 0);
		425	tmp = readl((void __iomem *)ptr);
		426	} else
1430	serge	427	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		428
1221	serge	429	WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
		430	WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
		431	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		432	for (i = 0; i < rdev->usec_timeout; i++) {
		433	/* read MC_STATUS */
		434	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		435	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		436	if (tmp == 2) {
		437	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		438	return;
		439	}
		440	if (tmp) {
		441	return;
		442	}
		443	udelay(1);
1128	serge	444	}
1221	serge	445	}
1128	serge	446
1221	serge	447	int r600_pcie_gart_init(struct radeon_device *rdev)
		448	{
		449	int r;
		450
		451	if (rdev->gart.table.vram.robj) {
1963	serge	452	WARN(1, "R600 PCIE GART already initialized\n");
1221	serge	453	return 0;
		454	}
		455	/* Initialize common gart structure */
		456	r = radeon_gart_init(rdev);
		457	if (r)
		458	return r;
		459	rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
		460	return radeon_gart_table_vram_alloc(rdev);
		461	}
		462
		463	int r600_pcie_gart_enable(struct radeon_device *rdev)
		464	{
		465	u32 tmp;
		466	int r, i;
		467
		468	if (rdev->gart.table.vram.robj == NULL) {
		469	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		470	return -EINVAL;
		471	}
		472	r = radeon_gart_table_vram_pin(rdev);
		473	if (r)
		474	return r;
1430	serge	475	radeon_gart_restore(rdev);
1221	serge	476
		477	/* Setup L2 cache */
		478	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		479	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		480	EFFECTIVE_L2_QUEUE_SIZE(7));
		481	WREG32(VM_L2_CNTL2, 0);
		482	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		483	/* Setup TLB control */
		484	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		485	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		486	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		487	ENABLE_WAIT_L2_QUERY;
		488	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		489	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		490	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		491	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		492	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		493	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		494	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		495	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		496	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		497	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		498	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		499	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		500	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		501	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		502	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		503	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		504	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		505	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		506	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		507	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		508	(u32)(rdev->dummy_page.addr >> 12));
		509	for (i = 1; i < 7; i++)
		510	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		511
		512	r600_pcie_gart_tlb_flush(rdev);
		513	rdev->gart.ready = true;
1128	serge	514	return 0;
		515	}
		516
1221	serge	517	void r600_pcie_gart_disable(struct radeon_device *rdev)
1128	serge	518	{
1221	serge	519	u32 tmp;
1321	serge	520	int i, r;
1221	serge	521
		522	/* Disable all tables */
		523	for (i = 0; i < 7; i++)
		524	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		525
		526	/* Disable L2 cache */
		527	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		528	EFFECTIVE_L2_QUEUE_SIZE(7));
		529	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		530	/* Setup L1 TLB control */
		531	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		532	ENABLE_WAIT_L2_QUERY;
		533	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		534	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		535	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		536	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		537	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		538	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		539	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		540	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		541	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
		542	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
		543	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		544	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		545	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
		546	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		547	if (rdev->gart.table.vram.robj) {
1403	serge	548	r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
		549	if (likely(r == 0)) {
		550	radeon_bo_kunmap(rdev->gart.table.vram.robj);
		551	radeon_bo_unpin(rdev->gart.table.vram.robj);
		552	radeon_bo_unreserve(rdev->gart.table.vram.robj);
		553	}
1221	serge	554	}
1128	serge	555	}
		556
1221	serge	557	void r600_pcie_gart_fini(struct radeon_device *rdev)
		558	{
1963	serge	559	radeon_gart_fini(rdev);
1221	serge	560	r600_pcie_gart_disable(rdev);
		561	radeon_gart_table_vram_free(rdev);
		562	}
1128	serge	563
1221	serge	564	void r600_agp_enable(struct radeon_device *rdev)
1128	serge	565	{
1221	serge	566	u32 tmp;
		567	int i;
		568
		569	/* Setup L2 cache */
		570	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		571	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		572	EFFECTIVE_L2_QUEUE_SIZE(7));
		573	WREG32(VM_L2_CNTL2, 0);
		574	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		575	/* Setup TLB control */
		576	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		577	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		578	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		579	ENABLE_WAIT_L2_QUERY;
		580	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		581	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		582	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		583	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		584	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		585	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		586	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		587	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		588	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		589	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		590	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		591	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		592	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		593	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		594	for (i = 0; i < 7; i++)
		595	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1128	serge	596	}
		597
		598	int r600_mc_wait_for_idle(struct radeon_device *rdev)
		599	{
1221	serge	600	unsigned i;
		601	u32 tmp;
		602
		603	for (i = 0; i < rdev->usec_timeout; i++) {
		604	/* read MC_STATUS */
		605	tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
		606	if (!tmp)
1128	serge	607	return 0;
1221	serge	608	udelay(1);
		609	}
		610	return -1;
1128	serge	611	}
		612
1221	serge	613	static void r600_mc_program(struct radeon_device *rdev)
1128	serge	614	{
1221	serge	615	struct rv515_mc_save save;
		616	u32 tmp;
		617	int i, j;
		618
		619	/* Initialize HDP */
		620	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		621	WREG32((0x2c14 + j), 0x00000000);
		622	WREG32((0x2c18 + j), 0x00000000);
		623	WREG32((0x2c1c + j), 0x00000000);
		624	WREG32((0x2c20 + j), 0x00000000);
		625	WREG32((0x2c24 + j), 0x00000000);
		626	}
		627	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		628
		629	rv515_mc_stop(rdev, &save);
		630	if (r600_mc_wait_for_idle(rdev)) {
		631	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		632	}
		633	/* Lockout access through VGA aperture (doesn't exist before R600) */
		634	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		635	/* Update configuration */
		636	if (rdev->flags & RADEON_IS_AGP) {
		637	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		638	/* VRAM before AGP */
		639	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		640	rdev->mc.vram_start >> 12);
		641	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		642	rdev->mc.gtt_end >> 12);
		643	} else {
		644	/* VRAM after AGP */
		645	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		646	rdev->mc.gtt_start >> 12);
		647	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		648	rdev->mc.vram_end >> 12);
		649	}
		650	} else {
		651	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
		652	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
		653	}
		654	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
		655	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		656	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		657	WREG32(MC_VM_FB_LOCATION, tmp);
		658	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
		659	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1963	serge	660	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1221	serge	661	if (rdev->flags & RADEON_IS_AGP) {
		662	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
		663	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
		664	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		665	} else {
		666	WREG32(MC_VM_AGP_BASE, 0);
		667	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		668	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		669	}
		670	if (r600_mc_wait_for_idle(rdev)) {
		671	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		672	}
		673	rv515_mc_resume(rdev, &save);
		674	/* we need to own VRAM, so turn off the VGA renderer here
		675	* to stop it overwriting our objects */
		676	rv515_vga_render_disable(rdev);
1128	serge	677	}
		678
1430	serge	679	/**
		680	* r600_vram_gtt_location - try to find VRAM & GTT location
		681	* @rdev: radeon device structure holding all necessary informations
		682	* @mc: memory controller structure holding memory informations
		683	*
		684	* Function will place try to place VRAM at same place as in CPU (PCI)
		685	* address space as some GPU seems to have issue when we reprogram at
		686	* different address space.
		687	*
		688	* If there is not enough space to fit the unvisible VRAM after the
		689	* aperture then we limit the VRAM size to the aperture.
		690	*
		691	* If we are using AGP then place VRAM adjacent to AGP aperture are we need
		692	* them to be in one from GPU point of view so that we can program GPU to
		693	* catch access outside them (weird GPU policy see ??).
		694	*
		695	* This function will never fails, worst case are limiting VRAM or GTT.
		696	*
		697	* Note: GTT start, end, size should be initialized before calling this
		698	* function on AGP platform.
		699	*/
1963	serge	700	static void r600_vram_gtt_location(struct radeon_device rdev, struct radeon_mc mc)
1430	serge	701	{
		702	u64 size_bf, size_af;
		703
		704	if (mc->mc_vram_size > 0xE0000000) {
		705	/* leave room for at least 512M GTT */
		706	dev_warn(rdev->dev, "limiting VRAM\n");
		707	mc->real_vram_size = 0xE0000000;
		708	mc->mc_vram_size = 0xE0000000;
		709	}
		710	if (rdev->flags & RADEON_IS_AGP) {
		711	size_bf = mc->gtt_start;
		712	size_af = 0xFFFFFFFF - mc->gtt_end + 1;
		713	if (size_bf > size_af) {
		714	if (mc->mc_vram_size > size_bf) {
		715	dev_warn(rdev->dev, "limiting VRAM\n");
		716	mc->real_vram_size = size_bf;
		717	mc->mc_vram_size = size_bf;
		718	}
		719	mc->vram_start = mc->gtt_start - mc->mc_vram_size;
		720	} else {
		721	if (mc->mc_vram_size > size_af) {
		722	dev_warn(rdev->dev, "limiting VRAM\n");
		723	mc->real_vram_size = size_af;
		724	mc->mc_vram_size = size_af;
		725	}
		726	mc->vram_start = mc->gtt_end;
		727	}
		728	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
		729	dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
		730	mc->mc_vram_size >> 20, mc->vram_start,
		731	mc->vram_end, mc->real_vram_size >> 20);
		732	} else {
		733	u64 base = 0;
1963	serge	734	if (rdev->flags & RADEON_IS_IGP) {
		735	base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
		736	base <<= 24;
		737	}
1430	serge	738	radeon_vram_location(rdev, &rdev->mc, base);
1963	serge	739	rdev->mc.gtt_base_align = 0;
1430	serge	740	radeon_gtt_location(rdev, mc);
		741	}
		742	}
		743
1221	serge	744	int r600_mc_init(struct radeon_device *rdev)
1128	serge	745	{
1221	serge	746	u32 tmp;
1268	serge	747	int chansize, numchan;
1128	serge	748
1221	serge	749	/* Get VRAM informations */
1128	serge	750	rdev->mc.vram_is_ddr = true;
1221	serge	751	tmp = RREG32(RAMCFG);
		752	if (tmp & CHANSIZE_OVERRIDE) {
1128	serge	753	chansize = 16;
1221	serge	754	} else if (tmp & CHANSIZE_MASK) {
1128	serge	755	chansize = 64;
		756	} else {
		757	chansize = 32;
		758	}
1268	serge	759	tmp = RREG32(CHMAP);
		760	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		761	case 0:
		762	default:
		763	numchan = 1;
		764	break;
		765	case 1:
		766	numchan = 2;
		767	break;
		768	case 2:
		769	numchan = 4;
		770	break;
		771	case 3:
		772	numchan = 8;
		773	break;
1128	serge	774	}
1268	serge	775	rdev->mc.vram_width = numchan * chansize;
1221	serge	776	/* Could aper size report 0 ? */
1963	serge	777	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		778	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1221	serge	779	/* Setup GPU memory space */
		780	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		781	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1430	serge	782	rdev->mc.visible_vram_size = rdev->mc.aper_size;
		783	r600_vram_gtt_location(rdev, &rdev->mc);
1963	serge	784
		785	if (rdev->flags & RADEON_IS_IGP) {
		786	rs690_pm_info(rdev);
1403	serge	787	rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1963	serge	788	}
		789	radeon_update_bandwidth_info(rdev);
1221	serge	790	return 0;
1128	serge	791	}
		792
1221	serge	793	/* We doesn't check that the GPU really needs a reset we simply do the
		794	* reset, it's up to the caller to determine if the GPU needs one. We
		795	* might add an helper function to check that.
		796	*/
		797	int r600_gpu_soft_reset(struct radeon_device *rdev)
1128	serge	798	{
1221	serge	799	struct rv515_mc_save save;
		800	u32 grbm_busy_mask = S_008010_VC_BUSY(1) \| S_008010_VGT_BUSY_NO_DMA(1) \|
		801	S_008010_VGT_BUSY(1) \| S_008010_TA03_BUSY(1) \|
		802	S_008010_TC_BUSY(1) \| S_008010_SX_BUSY(1) \|
		803	S_008010_SH_BUSY(1) \| S_008010_SPI03_BUSY(1) \|
		804	S_008010_SMX_BUSY(1) \| S_008010_SC_BUSY(1) \|
		805	S_008010_PA_BUSY(1) \| S_008010_DB03_BUSY(1) \|
		806	S_008010_CR_BUSY(1) \| S_008010_CB03_BUSY(1) \|
		807	S_008010_GUI_ACTIVE(1);
		808	u32 grbm2_busy_mask = S_008014_SPI0_BUSY(1) \| S_008014_SPI1_BUSY(1) \|
		809	S_008014_SPI2_BUSY(1) \| S_008014_SPI3_BUSY(1) \|
		810	S_008014_TA0_BUSY(1) \| S_008014_TA1_BUSY(1) \|
		811	S_008014_TA2_BUSY(1) \| S_008014_TA3_BUSY(1) \|
		812	S_008014_DB0_BUSY(1) \| S_008014_DB1_BUSY(1) \|
		813	S_008014_DB2_BUSY(1) \| S_008014_DB3_BUSY(1) \|
		814	S_008014_CB0_BUSY(1) \| S_008014_CB1_BUSY(1) \|
		815	S_008014_CB2_BUSY(1) \| S_008014_CB3_BUSY(1);
		816	u32 tmp;
1128	serge	817
1963	serge	818	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
		819	return 0;
		820
1221	serge	821	dev_info(rdev->dev, "GPU softreset \n");
		822	dev_info(rdev->dev, " R_008010_GRBM_STATUS=0x%08X\n",
		823	RREG32(R_008010_GRBM_STATUS));
		824	dev_info(rdev->dev, " R_008014_GRBM_STATUS2=0x%08X\n",
		825	RREG32(R_008014_GRBM_STATUS2));
		826	dev_info(rdev->dev, " R_000E50_SRBM_STATUS=0x%08X\n",
		827	RREG32(R_000E50_SRBM_STATUS));
		828	rv515_mc_stop(rdev, &save);
		829	if (r600_mc_wait_for_idle(rdev)) {
		830	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		831	}
		832	/* Disable CP parsing/prefetching */
1963	serge	833	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1221	serge	834	/* Check if any of the rendering block is busy and reset it */
		835	if ((RREG32(R_008010_GRBM_STATUS) & grbm_busy_mask) \|\|
		836	(RREG32(R_008014_GRBM_STATUS2) & grbm2_busy_mask)) {
		837	tmp = S_008020_SOFT_RESET_CR(1) \|
		838	S_008020_SOFT_RESET_DB(1) \|
		839	S_008020_SOFT_RESET_CB(1) \|
		840	S_008020_SOFT_RESET_PA(1) \|
		841	S_008020_SOFT_RESET_SC(1) \|
		842	S_008020_SOFT_RESET_SMX(1) \|
		843	S_008020_SOFT_RESET_SPI(1) \|
		844	S_008020_SOFT_RESET_SX(1) \|
		845	S_008020_SOFT_RESET_SH(1) \|
		846	S_008020_SOFT_RESET_TC(1) \|
		847	S_008020_SOFT_RESET_TA(1) \|
		848	S_008020_SOFT_RESET_VC(1) \|
		849	S_008020_SOFT_RESET_VGT(1);
		850	dev_info(rdev->dev, " R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		851	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1963	serge	852	RREG32(R_008020_GRBM_SOFT_RESET);
		853	mdelay(15);
1221	serge	854	WREG32(R_008020_GRBM_SOFT_RESET, 0);
		855	}
		856	/* Reset CP (we always reset CP) */
		857	tmp = S_008020_SOFT_RESET_CP(1);
		858	dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		859	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1963	serge	860	RREG32(R_008020_GRBM_SOFT_RESET);
		861	mdelay(15);
1221	serge	862	WREG32(R_008020_GRBM_SOFT_RESET, 0);
		863	/* Wait a little for things to settle down */
1963	serge	864	mdelay(1);
1221	serge	865	dev_info(rdev->dev, " R_008010_GRBM_STATUS=0x%08X\n",
		866	RREG32(R_008010_GRBM_STATUS));
		867	dev_info(rdev->dev, " R_008014_GRBM_STATUS2=0x%08X\n",
		868	RREG32(R_008014_GRBM_STATUS2));
		869	dev_info(rdev->dev, " R_000E50_SRBM_STATUS=0x%08X\n",
		870	RREG32(R_000E50_SRBM_STATUS));
		871	rv515_mc_resume(rdev, &save);
		872	return 0;
1128	serge	873	}
		874
1963	serge	875	bool r600_gpu_is_lockup(struct radeon_device *rdev)
1221	serge	876	{
1963	serge	877	u32 srbm_status;
		878	u32 grbm_status;
		879	u32 grbm_status2;
		880	struct r100_gpu_lockup *lockup;
		881	int r;
		882
		883	if (rdev->family >= CHIP_RV770)
		884	lockup = &rdev->config.rv770.lockup;
		885	else
		886	lockup = &rdev->config.r600.lockup;
		887
		888	srbm_status = RREG32(R_000E50_SRBM_STATUS);
		889	grbm_status = RREG32(R_008010_GRBM_STATUS);
		890	grbm_status2 = RREG32(R_008014_GRBM_STATUS2);
		891	if (!G_008010_GUI_ACTIVE(grbm_status)) {
		892	r100_gpu_lockup_update(lockup, &rdev->cp);
		893	return false;
		894	}
		895	/* force CP activities */
		896	r = radeon_ring_lock(rdev, 2);
		897	if (!r) {
		898	/* PACKET2 NOP */
		899	radeon_ring_write(rdev, 0x80000000);
		900	radeon_ring_write(rdev, 0x80000000);
		901	radeon_ring_unlock_commit(rdev);
		902	}
		903	rdev->cp.rptr = RREG32(R600_CP_RB_RPTR);
		904	return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
		905	}
		906
		907	int r600_asic_reset(struct radeon_device *rdev)
		908	{
1221	serge	909	return r600_gpu_soft_reset(rdev);
		910	}
		911
		912	static u32 r600_get_tile_pipe_to_backend_map(u32 num_tile_pipes,
		913	u32 num_backends,
		914	u32 backend_disable_mask)
		915	{
		916	u32 backend_map = 0;
		917	u32 enabled_backends_mask;
		918	u32 enabled_backends_count;
		919	u32 cur_pipe;
		920	u32 swizzle_pipe[R6XX_MAX_PIPES];
		921	u32 cur_backend;
		922	u32 i;
		923
		924	if (num_tile_pipes > R6XX_MAX_PIPES)
		925	num_tile_pipes = R6XX_MAX_PIPES;
		926	if (num_tile_pipes < 1)
		927	num_tile_pipes = 1;
		928	if (num_backends > R6XX_MAX_BACKENDS)
		929	num_backends = R6XX_MAX_BACKENDS;
		930	if (num_backends < 1)
		931	num_backends = 1;
		932
		933	enabled_backends_mask = 0;
		934	enabled_backends_count = 0;
		935	for (i = 0; i < R6XX_MAX_BACKENDS; ++i) {
		936	if (((backend_disable_mask >> i) & 1) == 0) {
		937	enabled_backends_mask \|= (1 << i);
		938	++enabled_backends_count;
		939	}
		940	if (enabled_backends_count == num_backends)
		941	break;
		942	}
		943
		944	if (enabled_backends_count == 0) {
		945	enabled_backends_mask = 1;
		946	enabled_backends_count = 1;
		947	}
		948
		949	if (enabled_backends_count != num_backends)
		950	num_backends = enabled_backends_count;
		951
		952	memset((uint8_t )&swizzle_pipe[0], 0, sizeof(u32) R6XX_MAX_PIPES);
		953	switch (num_tile_pipes) {
		954	case 1:
		955	swizzle_pipe[0] = 0;
		956	break;
		957	case 2:
		958	swizzle_pipe[0] = 0;
		959	swizzle_pipe[1] = 1;
		960	break;
		961	case 3:
		962	swizzle_pipe[0] = 0;
		963	swizzle_pipe[1] = 1;
		964	swizzle_pipe[2] = 2;
		965	break;
		966	case 4:
		967	swizzle_pipe[0] = 0;
		968	swizzle_pipe[1] = 1;
		969	swizzle_pipe[2] = 2;
		970	swizzle_pipe[3] = 3;
		971	break;
		972	case 5:
		973	swizzle_pipe[0] = 0;
		974	swizzle_pipe[1] = 1;
		975	swizzle_pipe[2] = 2;
		976	swizzle_pipe[3] = 3;
		977	swizzle_pipe[4] = 4;
		978	break;
		979	case 6:
		980	swizzle_pipe[0] = 0;
		981	swizzle_pipe[1] = 2;
		982	swizzle_pipe[2] = 4;
		983	swizzle_pipe[3] = 5;
		984	swizzle_pipe[4] = 1;
		985	swizzle_pipe[5] = 3;
		986	break;
		987	case 7:
		988	swizzle_pipe[0] = 0;
		989	swizzle_pipe[1] = 2;
		990	swizzle_pipe[2] = 4;
		991	swizzle_pipe[3] = 6;
		992	swizzle_pipe[4] = 1;
		993	swizzle_pipe[5] = 3;
		994	swizzle_pipe[6] = 5;
		995	break;
		996	case 8:
		997	swizzle_pipe[0] = 0;
		998	swizzle_pipe[1] = 2;
		999	swizzle_pipe[2] = 4;
		1000	swizzle_pipe[3] = 6;
		1001	swizzle_pipe[4] = 1;
		1002	swizzle_pipe[5] = 3;
		1003	swizzle_pipe[6] = 5;
		1004	swizzle_pipe[7] = 7;
		1005	break;
		1006	}
		1007
		1008	cur_backend = 0;
		1009	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
		1010	while (((1 << cur_backend) & enabled_backends_mask) == 0)
		1011	cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
		1012
		1013	backend_map \|= (u32)(((cur_backend & 3) << (swizzle_pipe[cur_pipe] * 2)));
		1014
		1015	cur_backend = (cur_backend + 1) % R6XX_MAX_BACKENDS;
		1016	}
		1017
		1018	return backend_map;
		1019	}
		1020
		1021	int r600_count_pipe_bits(uint32_t val)
		1022	{
		1023	int i, ret = 0;
		1024
		1025	for (i = 0; i < 32; i++) {
		1026	ret += val & 1;
		1027	val >>= 1;
		1028	}
		1029	return ret;
		1030	}
		1031
		1032	void r600_gpu_init(struct radeon_device *rdev)
		1033	{
		1034	u32 tiling_config;
		1035	u32 ramcfg;
1430	serge	1036	u32 backend_map;
		1037	u32 cc_rb_backend_disable;
		1038	u32 cc_gc_shader_pipe_config;
1221	serge	1039	u32 tmp;
		1040	int i, j;
		1041	u32 sq_config;
		1042	u32 sq_gpr_resource_mgmt_1 = 0;
		1043	u32 sq_gpr_resource_mgmt_2 = 0;
		1044	u32 sq_thread_resource_mgmt = 0;
		1045	u32 sq_stack_resource_mgmt_1 = 0;
		1046	u32 sq_stack_resource_mgmt_2 = 0;
		1047
		1048	/* FIXME: implement */
		1049	switch (rdev->family) {
		1050	case CHIP_R600:
		1051	rdev->config.r600.max_pipes = 4;
		1052	rdev->config.r600.max_tile_pipes = 8;
		1053	rdev->config.r600.max_simds = 4;
		1054	rdev->config.r600.max_backends = 4;
		1055	rdev->config.r600.max_gprs = 256;
		1056	rdev->config.r600.max_threads = 192;
		1057	rdev->config.r600.max_stack_entries = 256;
		1058	rdev->config.r600.max_hw_contexts = 8;
		1059	rdev->config.r600.max_gs_threads = 16;
		1060	rdev->config.r600.sx_max_export_size = 128;
		1061	rdev->config.r600.sx_max_export_pos_size = 16;
		1062	rdev->config.r600.sx_max_export_smx_size = 128;
		1063	rdev->config.r600.sq_num_cf_insts = 2;
		1064	break;
		1065	case CHIP_RV630:
		1066	case CHIP_RV635:
		1067	rdev->config.r600.max_pipes = 2;
		1068	rdev->config.r600.max_tile_pipes = 2;
		1069	rdev->config.r600.max_simds = 3;
		1070	rdev->config.r600.max_backends = 1;
		1071	rdev->config.r600.max_gprs = 128;
		1072	rdev->config.r600.max_threads = 192;
		1073	rdev->config.r600.max_stack_entries = 128;
		1074	rdev->config.r600.max_hw_contexts = 8;
		1075	rdev->config.r600.max_gs_threads = 4;
		1076	rdev->config.r600.sx_max_export_size = 128;
		1077	rdev->config.r600.sx_max_export_pos_size = 16;
		1078	rdev->config.r600.sx_max_export_smx_size = 128;
		1079	rdev->config.r600.sq_num_cf_insts = 2;
		1080	break;
		1081	case CHIP_RV610:
		1082	case CHIP_RV620:
		1083	case CHIP_RS780:
		1084	case CHIP_RS880:
		1085	rdev->config.r600.max_pipes = 1;
		1086	rdev->config.r600.max_tile_pipes = 1;
		1087	rdev->config.r600.max_simds = 2;
		1088	rdev->config.r600.max_backends = 1;
		1089	rdev->config.r600.max_gprs = 128;
		1090	rdev->config.r600.max_threads = 192;
		1091	rdev->config.r600.max_stack_entries = 128;
		1092	rdev->config.r600.max_hw_contexts = 4;
		1093	rdev->config.r600.max_gs_threads = 4;
		1094	rdev->config.r600.sx_max_export_size = 128;
		1095	rdev->config.r600.sx_max_export_pos_size = 16;
		1096	rdev->config.r600.sx_max_export_smx_size = 128;
		1097	rdev->config.r600.sq_num_cf_insts = 1;
		1098	break;
		1099	case CHIP_RV670:
		1100	rdev->config.r600.max_pipes = 4;
		1101	rdev->config.r600.max_tile_pipes = 4;
		1102	rdev->config.r600.max_simds = 4;
		1103	rdev->config.r600.max_backends = 4;
		1104	rdev->config.r600.max_gprs = 192;
		1105	rdev->config.r600.max_threads = 192;
		1106	rdev->config.r600.max_stack_entries = 256;
		1107	rdev->config.r600.max_hw_contexts = 8;
		1108	rdev->config.r600.max_gs_threads = 16;
		1109	rdev->config.r600.sx_max_export_size = 128;
		1110	rdev->config.r600.sx_max_export_pos_size = 16;
		1111	rdev->config.r600.sx_max_export_smx_size = 128;
		1112	rdev->config.r600.sq_num_cf_insts = 2;
		1113	break;
		1114	default:
		1115	break;
		1116	}
		1117
		1118	/* Initialize HDP */
		1119	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1120	WREG32((0x2c14 + j), 0x00000000);
		1121	WREG32((0x2c18 + j), 0x00000000);
		1122	WREG32((0x2c1c + j), 0x00000000);
		1123	WREG32((0x2c20 + j), 0x00000000);
		1124	WREG32((0x2c24 + j), 0x00000000);
		1125	}
		1126
		1127	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1128
		1129	/* Setup tiling */
		1130	tiling_config = 0;
		1131	ramcfg = RREG32(RAMCFG);
		1132	switch (rdev->config.r600.max_tile_pipes) {
		1133	case 1:
		1134	tiling_config \|= PIPE_TILING(0);
		1135	break;
		1136	case 2:
		1137	tiling_config \|= PIPE_TILING(1);
		1138	break;
		1139	case 4:
		1140	tiling_config \|= PIPE_TILING(2);
		1141	break;
		1142	case 8:
		1143	tiling_config \|= PIPE_TILING(3);
		1144	break;
		1145	default:
		1146	break;
		1147	}
1430	serge	1148	rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
		1149	rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1221	serge	1150	tiling_config \|= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1963	serge	1151	tiling_config \|= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
		1152	if ((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT)
		1153	rdev->config.r600.tiling_group_size = 512;
		1154	else
1430	serge	1155	rdev->config.r600.tiling_group_size = 256;
1221	serge	1156	tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
		1157	if (tmp > 3) {
		1158	tiling_config \|= ROW_TILING(3);
		1159	tiling_config \|= SAMPLE_SPLIT(3);
		1160	} else {
		1161	tiling_config \|= ROW_TILING(tmp);
		1162	tiling_config \|= SAMPLE_SPLIT(tmp);
		1163	}
		1164	tiling_config \|= BANK_SWAPS(1);
1430	serge	1165
		1166	cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
		1167	cc_rb_backend_disable \|=
		1168	BACKEND_DISABLE((R6XX_MAX_BACKENDS_MASK << rdev->config.r600.max_backends) & R6XX_MAX_BACKENDS_MASK);
		1169
		1170	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffffff00;
		1171	cc_gc_shader_pipe_config \|=
		1172	INACTIVE_QD_PIPES((R6XX_MAX_PIPES_MASK << rdev->config.r600.max_pipes) & R6XX_MAX_PIPES_MASK);
		1173	cc_gc_shader_pipe_config \|=
		1174	INACTIVE_SIMDS((R6XX_MAX_SIMDS_MASK << rdev->config.r600.max_simds) & R6XX_MAX_SIMDS_MASK);
		1175
		1176	backend_map = r600_get_tile_pipe_to_backend_map(rdev->config.r600.max_tile_pipes,
		1177	(R6XX_MAX_BACKENDS -
		1178	r600_count_pipe_bits((cc_rb_backend_disable &
		1179	R6XX_MAX_BACKENDS_MASK) >> 16)),
		1180	(cc_rb_backend_disable >> 16));
1963	serge	1181	rdev->config.r600.tile_config = tiling_config;
2160	serge	1182	rdev->config.r600.backend_map = backend_map;
1430	serge	1183	tiling_config \|= BACKEND_MAP(backend_map);
1221	serge	1184	WREG32(GB_TILING_CONFIG, tiling_config);
		1185	WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
		1186	WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
		1187
		1188	/* Setup pipes */
1430	serge	1189	WREG32(CC_RB_BACKEND_DISABLE, cc_rb_backend_disable);
		1190	WREG32(CC_GC_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1963	serge	1191	WREG32(GC_USER_SHADER_PIPE_CONFIG, cc_gc_shader_pipe_config);
1221	serge	1192
1430	serge	1193	tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1221	serge	1194	WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
		1195	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
		1196
		1197	/* Setup some CP states */
		1198	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \| ROQ_IB2_START(0x2b)));
		1199	WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) \| ROQ_END(0x40)));
		1200
		1201	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \| SYNC_GRADIENT \|
		1202	SYNC_WALKER \| SYNC_ALIGNER));
		1203	/* Setup various GPU states */
		1204	if (rdev->family == CHIP_RV670)
		1205	WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
		1206
		1207	tmp = RREG32(SX_DEBUG_1);
		1208	tmp \|= SMX_EVENT_RELEASE;
		1209	if ((rdev->family > CHIP_R600))
		1210	tmp \|= ENABLE_NEW_SMX_ADDRESS;
		1211	WREG32(SX_DEBUG_1, tmp);
		1212
		1213	if (((rdev->family) == CHIP_R600) \|\|
		1214	((rdev->family) == CHIP_RV630) \|\|
		1215	((rdev->family) == CHIP_RV610) \|\|
		1216	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1217	((rdev->family) == CHIP_RS780) \|\|
		1218	((rdev->family) == CHIP_RS880)) {
1221	serge	1219	WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
		1220	} else {
		1221	WREG32(DB_DEBUG, 0);
		1222	}
		1223	WREG32(DB_WATERMARKS, (DEPTH_FREE(4) \| DEPTH_CACHELINE_FREE(16) \|
		1224	DEPTH_FLUSH(16) \| DEPTH_PENDING_FREE(4)));
		1225
		1226	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1227	WREG32(VGT_NUM_INSTANCES, 0);
		1228
		1229	WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
		1230	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
		1231
		1232	tmp = RREG32(SQ_MS_FIFO_SIZES);
		1233	if (((rdev->family) == CHIP_RV610) \|\|
		1234	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1235	((rdev->family) == CHIP_RS780) \|\|
		1236	((rdev->family) == CHIP_RS880)) {
1221	serge	1237	tmp = (CACHE_FIFO_SIZE(0xa) \|
		1238	FETCH_FIFO_HIWATER(0xa) \|
		1239	DONE_FIFO_HIWATER(0xe0) \|
		1240	ALU_UPDATE_FIFO_HIWATER(0x8));
		1241	} else if (((rdev->family) == CHIP_R600) \|\|
		1242	((rdev->family) == CHIP_RV630)) {
		1243	tmp &= ~DONE_FIFO_HIWATER(0xff);
		1244	tmp \|= DONE_FIFO_HIWATER(0x4);
		1245	}
		1246	WREG32(SQ_MS_FIFO_SIZES, tmp);
		1247
		1248	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
		1249	* should be adjusted as needed by the 2D/3D drivers. This just sets default values
		1250	*/
		1251	sq_config = RREG32(SQ_CONFIG);
		1252	sq_config &= ~(PS_PRIO(3) \|
		1253	VS_PRIO(3) \|
		1254	GS_PRIO(3) \|
		1255	ES_PRIO(3));
		1256	sq_config \|= (DX9_CONSTS \|
		1257	VC_ENABLE \|
		1258	PS_PRIO(0) \|
		1259	VS_PRIO(1) \|
		1260	GS_PRIO(2) \|
		1261	ES_PRIO(3));
		1262
		1263	if ((rdev->family) == CHIP_R600) {
		1264	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) \|
		1265	NUM_VS_GPRS(124) \|
		1266	NUM_CLAUSE_TEMP_GPRS(4));
		1267	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) \|
		1268	NUM_ES_GPRS(0));
		1269	sq_thread_resource_mgmt = (NUM_PS_THREADS(136) \|
		1270	NUM_VS_THREADS(48) \|
		1271	NUM_GS_THREADS(4) \|
		1272	NUM_ES_THREADS(4));
		1273	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) \|
		1274	NUM_VS_STACK_ENTRIES(128));
		1275	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) \|
		1276	NUM_ES_STACK_ENTRIES(0));
		1277	} else if (((rdev->family) == CHIP_RV610) \|\|
		1278	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1279	((rdev->family) == CHIP_RS780) \|\|
		1280	((rdev->family) == CHIP_RS880)) {
1221	serge	1281	/* no vertex cache */
		1282	sq_config &= ~VC_ENABLE;
		1283
		1284	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1285	NUM_VS_GPRS(44) \|
		1286	NUM_CLAUSE_TEMP_GPRS(2));
		1287	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1288	NUM_ES_GPRS(17));
		1289	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1290	NUM_VS_THREADS(78) \|
		1291	NUM_GS_THREADS(4) \|
		1292	NUM_ES_THREADS(31));
		1293	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1294	NUM_VS_STACK_ENTRIES(40));
		1295	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1296	NUM_ES_STACK_ENTRIES(16));
		1297	} else if (((rdev->family) == CHIP_RV630) \|\|
		1298	((rdev->family) == CHIP_RV635)) {
		1299	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1300	NUM_VS_GPRS(44) \|
		1301	NUM_CLAUSE_TEMP_GPRS(2));
		1302	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) \|
		1303	NUM_ES_GPRS(18));
		1304	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1305	NUM_VS_THREADS(78) \|
		1306	NUM_GS_THREADS(4) \|
		1307	NUM_ES_THREADS(31));
		1308	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1309	NUM_VS_STACK_ENTRIES(40));
		1310	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1311	NUM_ES_STACK_ENTRIES(16));
		1312	} else if ((rdev->family) == CHIP_RV670) {
		1313	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1314	NUM_VS_GPRS(44) \|
		1315	NUM_CLAUSE_TEMP_GPRS(2));
		1316	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1317	NUM_ES_GPRS(17));
		1318	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1319	NUM_VS_THREADS(78) \|
		1320	NUM_GS_THREADS(4) \|
		1321	NUM_ES_THREADS(31));
		1322	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) \|
		1323	NUM_VS_STACK_ENTRIES(64));
		1324	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) \|
		1325	NUM_ES_STACK_ENTRIES(64));
		1326	}
		1327
		1328	WREG32(SQ_CONFIG, sq_config);
		1329	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		1330	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		1331	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		1332	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		1333	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		1334
		1335	if (((rdev->family) == CHIP_RV610) \|\|
		1336	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1337	((rdev->family) == CHIP_RS780) \|\|
		1338	((rdev->family) == CHIP_RS880)) {
1221	serge	1339	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
		1340	} else {
		1341	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
		1342	}
		1343
		1344	/* More default values. 2D/3D driver should adjust as needed */
		1345	WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) \| S0_Y(0x4) \|
		1346	S1_X(0x4) \| S1_Y(0xc)));
		1347	WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) \| S0_Y(0xe) \|
		1348	S1_X(0x2) \| S1_Y(0x2) \|
		1349	S2_X(0xa) \| S2_Y(0x6) \|
		1350	S3_X(0x6) \| S3_Y(0xa)));
		1351	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) \| S0_Y(0xb) \|
		1352	S1_X(0x4) \| S1_Y(0xc) \|
		1353	S2_X(0x1) \| S2_Y(0x6) \|
		1354	S3_X(0xa) \| S3_Y(0xe)));
		1355	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) \| S4_Y(0x1) \|
		1356	S5_X(0x0) \| S5_Y(0x0) \|
		1357	S6_X(0xb) \| S6_Y(0x4) \|
		1358	S7_X(0x7) \| S7_Y(0x8)));
		1359
		1360	WREG32(VGT_STRMOUT_EN, 0);
		1361	tmp = rdev->config.r600.max_pipes * 16;
		1362	switch (rdev->family) {
		1363	case CHIP_RV610:
1268	serge	1364	case CHIP_RV620:
1221	serge	1365	case CHIP_RS780:
1268	serge	1366	case CHIP_RS880:
1221	serge	1367	tmp += 32;
		1368	break;
		1369	case CHIP_RV670:
		1370	tmp += 128;
		1371	break;
		1372	default:
		1373	break;
		1374	}
		1375	if (tmp > 256) {
		1376	tmp = 256;
		1377	}
		1378	WREG32(VGT_ES_PER_GS, 128);
		1379	WREG32(VGT_GS_PER_ES, tmp);
		1380	WREG32(VGT_GS_PER_VS, 2);
		1381	WREG32(VGT_GS_VERTEX_REUSE, 16);
		1382
		1383	/* more default values. 2D/3D driver should adjust as needed */
		1384	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		1385	WREG32(VGT_STRMOUT_EN, 0);
		1386	WREG32(SX_MISC, 0);
		1387	WREG32(PA_SC_MODE_CNTL, 0);
		1388	WREG32(PA_SC_AA_CONFIG, 0);
		1389	WREG32(PA_SC_LINE_STIPPLE, 0);
		1390	WREG32(SPI_INPUT_Z, 0);
		1391	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
		1392	WREG32(CB_COLOR7_FRAG, 0);
		1393
		1394	/* Clear render buffer base addresses */
		1395	WREG32(CB_COLOR0_BASE, 0);
		1396	WREG32(CB_COLOR1_BASE, 0);
		1397	WREG32(CB_COLOR2_BASE, 0);
		1398	WREG32(CB_COLOR3_BASE, 0);
		1399	WREG32(CB_COLOR4_BASE, 0);
		1400	WREG32(CB_COLOR5_BASE, 0);
		1401	WREG32(CB_COLOR6_BASE, 0);
		1402	WREG32(CB_COLOR7_BASE, 0);
		1403	WREG32(CB_COLOR7_FRAG, 0);
		1404
		1405	switch (rdev->family) {
		1406	case CHIP_RV610:
1268	serge	1407	case CHIP_RV620:
1221	serge	1408	case CHIP_RS780:
1268	serge	1409	case CHIP_RS880:
1221	serge	1410	tmp = TC_L2_SIZE(8);
		1411	break;
		1412	case CHIP_RV630:
		1413	case CHIP_RV635:
		1414	tmp = TC_L2_SIZE(4);
		1415	break;
		1416	case CHIP_R600:
		1417	tmp = TC_L2_SIZE(0) \| L2_DISABLE_LATE_HIT;
		1418	break;
		1419	default:
		1420	tmp = TC_L2_SIZE(0);
		1421	break;
		1422	}
		1423	WREG32(TC_CNTL, tmp);
		1424
		1425	tmp = RREG32(HDP_HOST_PATH_CNTL);
		1426	WREG32(HDP_HOST_PATH_CNTL, tmp);
		1427
		1428	tmp = RREG32(ARB_POP);
		1429	tmp \|= ENABLE_TC128;
		1430	WREG32(ARB_POP, tmp);
		1431
		1432	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1433	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA \|
		1434	NUM_CLIP_SEQ(3)));
		1435	WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
		1436	}
		1437
		1438
1128	serge	1439	/*
		1440	* Indirect registers accessor
		1441	*/
1221	serge	1442	u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1128	serge	1443	{
1221	serge	1444	u32 r;
1128	serge	1445
1221	serge	1446	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1447	(void)RREG32(PCIE_PORT_INDEX);
		1448	r = RREG32(PCIE_PORT_DATA);
1128	serge	1449	return r;
		1450	}
		1451
1221	serge	1452	void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1128	serge	1453	{
1221	serge	1454	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1455	(void)RREG32(PCIE_PORT_INDEX);
		1456	WREG32(PCIE_PORT_DATA, (v));
		1457	(void)RREG32(PCIE_PORT_DATA);
1128	serge	1458	}
1221	serge	1459
		1460	/*
		1461	* CP & Ring
		1462	*/
		1463	void r600_cp_stop(struct radeon_device *rdev)
		1464	{
1963	serge	1465	// radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1221	serge	1466	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1963	serge	1467	WREG32(SCRATCH_UMSK, 0);
1221	serge	1468	}
1413	serge	1469
		1470	int r600_init_microcode(struct radeon_device *rdev)
		1471	{
		1472	struct platform_device *pdev;
		1473	const char *chip_name;
		1474	const char *rlc_chip_name;
		1475	size_t pfp_req_size, me_req_size, rlc_req_size;
		1476	char fw_name[30];
		1477	int err;
		1478
		1479	DRM_DEBUG("\n");
		1480
		1481	pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
		1482	err = IS_ERR(pdev);
		1483	if (err) {
		1484	printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
		1485	return -EINVAL;
		1486	}
		1487
		1488	switch (rdev->family) {
		1489	case CHIP_R600:
		1490	chip_name = "R600";
		1491	rlc_chip_name = "R600";
		1492	break;
		1493	case CHIP_RV610:
		1494	chip_name = "RV610";
		1495	rlc_chip_name = "R600";
		1496	break;
		1497	case CHIP_RV630:
		1498	chip_name = "RV630";
		1499	rlc_chip_name = "R600";
		1500	break;
		1501	case CHIP_RV620:
		1502	chip_name = "RV620";
		1503	rlc_chip_name = "R600";
		1504	break;
		1505	case CHIP_RV635:
		1506	chip_name = "RV635";
		1507	rlc_chip_name = "R600";
		1508	break;
		1509	case CHIP_RV670:
		1510	chip_name = "RV670";
		1511	rlc_chip_name = "R600";
		1512	break;
		1513	case CHIP_RS780:
		1514	case CHIP_RS880:
		1515	chip_name = "RS780";
		1516	rlc_chip_name = "R600";
		1517	break;
		1518	case CHIP_RV770:
		1519	chip_name = "RV770";
		1520	rlc_chip_name = "R700";
		1521	break;
		1522	case CHIP_RV730:
		1523	case CHIP_RV740:
		1524	chip_name = "RV730";
		1525	rlc_chip_name = "R700";
		1526	break;
		1527	case CHIP_RV710:
		1528	chip_name = "RV710";
		1529	rlc_chip_name = "R700";
		1530	break;
1963	serge	1531	case CHIP_CEDAR:
		1532	chip_name = "CEDAR";
		1533	rlc_chip_name = "CEDAR";
		1534	break;
		1535	case CHIP_REDWOOD:
		1536	chip_name = "REDWOOD";
		1537	rlc_chip_name = "REDWOOD";
		1538	break;
		1539	case CHIP_JUNIPER:
		1540	chip_name = "JUNIPER";
		1541	rlc_chip_name = "JUNIPER";
		1542	break;
		1543	case CHIP_CYPRESS:
		1544	case CHIP_HEMLOCK:
		1545	chip_name = "CYPRESS";
		1546	rlc_chip_name = "CYPRESS";
		1547	break;
		1548	case CHIP_PALM:
		1549	chip_name = "PALM";
		1550	rlc_chip_name = "SUMO";
		1551	break;
1986	serge	1552	case CHIP_SUMO:
		1553	chip_name = "SUMO";
		1554	rlc_chip_name = "SUMO";
		1555	break;
		1556	case CHIP_SUMO2:
		1557	chip_name = "SUMO2";
		1558	rlc_chip_name = "SUMO";
		1559	break;
1413	serge	1560	default: BUG();
		1561	}
		1562
1963	serge	1563	if (rdev->family >= CHIP_CEDAR) {
		1564	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		1565	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		1566	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		1567	} else if (rdev->family >= CHIP_RV770) {
1413	serge	1568	pfp_req_size = R700_PFP_UCODE_SIZE * 4;
		1569	me_req_size = R700_PM4_UCODE_SIZE * 4;
		1570	rlc_req_size = R700_RLC_UCODE_SIZE * 4;
		1571	} else {
		1572	pfp_req_size = PFP_UCODE_SIZE * 4;
		1573	me_req_size = PM4_UCODE_SIZE * 12;
		1574	rlc_req_size = RLC_UCODE_SIZE * 4;
		1575	}
		1576
		1577	DRM_INFO("Loading %s Microcode\n", chip_name);
		1578
		1579	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
		1580	err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
		1581	if (err)
		1582	goto out;
		1583	if (rdev->pfp_fw->size != pfp_req_size) {
		1584	printk(KERN_ERR
		1585	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1586	rdev->pfp_fw->size, fw_name);
		1587	err = -EINVAL;
		1588	goto out;
		1589	}
		1590
		1591	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
		1592	err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
		1593	if (err)
		1594	goto out;
		1595	if (rdev->me_fw->size != me_req_size) {
		1596	printk(KERN_ERR
		1597	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1598	rdev->me_fw->size, fw_name);
		1599	err = -EINVAL;
		1600	}
		1601
		1602	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
		1603	err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
		1604	if (err)
		1605	goto out;
		1606	if (rdev->rlc_fw->size != rlc_req_size) {
		1607	printk(KERN_ERR
		1608	"r600_rlc: Bogus length %zu in firmware \"%s\"\n",
		1609	rdev->rlc_fw->size, fw_name);
		1610	err = -EINVAL;
		1611	}
		1612
		1613	out:
		1614	platform_device_unregister(pdev);
		1615
		1616	if (err) {
		1617	if (err != -EINVAL)
		1618	printk(KERN_ERR
		1619	"r600_cp: Failed to load firmware \"%s\"\n",
		1620	fw_name);
		1621	release_firmware(rdev->pfp_fw);
		1622	rdev->pfp_fw = NULL;
		1623	release_firmware(rdev->me_fw);
		1624	rdev->me_fw = NULL;
		1625	release_firmware(rdev->rlc_fw);
		1626	rdev->rlc_fw = NULL;
		1627	}
		1628	return err;
		1629	}
		1630
		1631	static int r600_cp_load_microcode(struct radeon_device *rdev)
		1632	{
		1633	const __be32 *fw_data;
		1634	int i;
		1635
		1636	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1637	return -EINVAL;
		1638
		1639	r600_cp_stop(rdev);
		1640
1963	serge	1641	WREG32(CP_RB_CNTL,
		1642	#ifdef __BIG_ENDIAN
		1643	BUF_SWAP_32BIT \|
		1644	#endif
		1645	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
1413	serge	1646
		1647	/* Reset cp */
		1648	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1649	RREG32(GRBM_SOFT_RESET);
		1650	mdelay(15);
		1651	WREG32(GRBM_SOFT_RESET, 0);
		1652
		1653	WREG32(CP_ME_RAM_WADDR, 0);
		1654
		1655	fw_data = (const __be32 *)rdev->me_fw->data;
		1656	WREG32(CP_ME_RAM_WADDR, 0);
		1657	for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
		1658	WREG32(CP_ME_RAM_DATA,
		1659	be32_to_cpup(fw_data++));
		1660
		1661	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1662	WREG32(CP_PFP_UCODE_ADDR, 0);
		1663	for (i = 0; i < PFP_UCODE_SIZE; i++)
		1664	WREG32(CP_PFP_UCODE_DATA,
		1665	be32_to_cpup(fw_data++));
		1666
		1667	WREG32(CP_PFP_UCODE_ADDR, 0);
		1668	WREG32(CP_ME_RAM_WADDR, 0);
		1669	WREG32(CP_ME_RAM_RADDR, 0);
		1670	return 0;
		1671	}
		1672
1221	serge	1673	int r600_cp_start(struct radeon_device *rdev)
		1674	{
		1675	int r;
		1676	uint32_t cp_me;
		1677
		1678	r = radeon_ring_lock(rdev, 7);
		1679	if (r) {
		1680	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1681	return r;
		1682	}
		1683	radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1684	radeon_ring_write(rdev, 0x1);
1963	serge	1685	if (rdev->family >= CHIP_RV770) {
		1686	radeon_ring_write(rdev, 0x0);
		1687	radeon_ring_write(rdev, rdev->config.rv770.max_hw_contexts - 1);
		1688	} else {
1221	serge	1689	radeon_ring_write(rdev, 0x3);
		1690	radeon_ring_write(rdev, rdev->config.r600.max_hw_contexts - 1);
		1691	}
		1692	radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1693	radeon_ring_write(rdev, 0);
		1694	radeon_ring_write(rdev, 0);
		1695	radeon_ring_unlock_commit(rdev);
		1696
		1697	cp_me = 0xff;
		1698	WREG32(R_0086D8_CP_ME_CNTL, cp_me);
		1699	return 0;
		1700	}
1413	serge	1701
		1702	int r600_cp_resume(struct radeon_device *rdev)
		1703	{
		1704	u32 tmp;
		1705	u32 rb_bufsz;
		1706	int r;
		1707
		1708	/* Reset cp */
		1709	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1710	RREG32(GRBM_SOFT_RESET);
		1711	mdelay(15);
		1712	WREG32(GRBM_SOFT_RESET, 0);
		1713
		1714	/* Set ring buffer size */
		1715	rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1963	serge	1716	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
1413	serge	1717	#ifdef __BIG_ENDIAN
		1718	tmp \|= BUF_SWAP_32BIT;
		1719	#endif
		1720	WREG32(CP_RB_CNTL, tmp);
		1721	WREG32(CP_SEM_WAIT_TIMER, 0x4);
		1722
		1723	/* Set the write pointer delay */
		1724	WREG32(CP_RB_WPTR_DELAY, 0);
		1725
		1726	/* Initialize the ring buffer's read and write pointers */
		1727	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1728	WREG32(CP_RB_RPTR_WR, 0);
		1729	WREG32(CP_RB_WPTR, 0);
1963	serge	1730
		1731	/* set the wb address whether it's enabled or not */
		1732	WREG32(CP_RB_RPTR_ADDR,
		1733	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1734	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1735	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1736
		1737	if (rdev->wb.enabled)
		1738	WREG32(SCRATCH_UMSK, 0xff);
		1739	else {
		1740	tmp \|= RB_NO_UPDATE;
		1741	WREG32(SCRATCH_UMSK, 0);
		1742	}
		1743
1413	serge	1744	mdelay(1);
		1745	WREG32(CP_RB_CNTL, tmp);
		1746
		1747	WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
		1748	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		1749
		1750	rdev->cp.rptr = RREG32(CP_RB_RPTR);
		1751	rdev->cp.wptr = RREG32(CP_RB_WPTR);
		1752
		1753	r600_cp_start(rdev);
		1754	rdev->cp.ready = true;
		1755	r = radeon_ring_test(rdev);
		1756	if (r) {
		1757	rdev->cp.ready = false;
		1758	return r;
		1759	}
		1760	return 0;
		1761	}
		1762
1221	serge	1763	void r600_cp_commit(struct radeon_device *rdev)
		1764	{
		1765	WREG32(CP_RB_WPTR, rdev->cp.wptr);
		1766	(void)RREG32(CP_RB_WPTR);
		1767	}
		1768
1233	serge	1769	void r600_ring_init(struct radeon_device *rdev, unsigned ring_size)
		1770	{
		1771	u32 rb_bufsz;
1221	serge	1772
1233	serge	1773	/* Align ring size */
		1774	rb_bufsz = drm_order(ring_size / 8);
		1775	ring_size = (1 << (rb_bufsz + 1)) * 4;
		1776	rdev->cp.ring_size = ring_size;
		1777	rdev->cp.align_mask = 16 - 1;
		1778	}
		1779
1963	serge	1780	void r600_cp_fini(struct radeon_device *rdev)
		1781	{
		1782	r600_cp_stop(rdev);
		1783	radeon_ring_fini(rdev);
		1784	}
1233	serge	1785
1963	serge	1786
1233	serge	1787	/*
		1788	* GPU scratch registers helpers function.
		1789	*/
		1790	void r600_scratch_init(struct radeon_device *rdev)
		1791	{
		1792	int i;
		1793
		1794	rdev->scratch.num_reg = 7;
1963	serge	1795	rdev->scratch.reg_base = SCRATCH_REG0;
1233	serge	1796	for (i = 0; i < rdev->scratch.num_reg; i++) {
		1797	rdev->scratch.free[i] = true;
1963	serge	1798	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
1233	serge	1799	}
		1800	}
1413	serge	1801
		1802	int r600_ring_test(struct radeon_device *rdev)
		1803	{
		1804	uint32_t scratch;
		1805	uint32_t tmp = 0;
		1806	unsigned i;
		1807	int r;
		1808
		1809	r = radeon_scratch_get(rdev, &scratch);
		1810	if (r) {
		1811	DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
		1812	return r;
		1813	}
		1814	WREG32(scratch, 0xCAFEDEAD);
		1815	r = radeon_ring_lock(rdev, 3);
		1816	if (r) {
		1817	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1818	radeon_scratch_free(rdev, scratch);
		1819	return r;
		1820	}
		1821	radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1822	radeon_ring_write(rdev, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		1823	radeon_ring_write(rdev, 0xDEADBEEF);
		1824	radeon_ring_unlock_commit(rdev);
		1825	for (i = 0; i < rdev->usec_timeout; i++) {
		1826	tmp = RREG32(scratch);
		1827	if (tmp == 0xDEADBEEF)
		1828	break;
		1829	DRM_UDELAY(1);
		1830	}
		1831	if (i < rdev->usec_timeout) {
		1832	DRM_INFO("ring test succeeded in %d usecs\n", i);
		1833	} else {
		1834	DRM_ERROR("radeon: ring test failed (scratch(0x%04X)=0x%08X)\n",
		1835	scratch, tmp);
		1836	r = -EINVAL;
		1837	}
		1838	radeon_scratch_free(rdev, scratch);
		1839	return r;
		1840	}
1963	serge	1841
1413	serge	1842	void r600_fence_ring_emit(struct radeon_device *rdev,
		1843	struct radeon_fence *fence)
		1844	{
1963	serge	1845	if (rdev->wb.use_event) {
		1846	u64 addr = rdev->wb.gpu_addr + R600_WB_EVENT_OFFSET +
		1847	(u64)(rdev->fence_drv.scratch_reg - rdev->scratch.reg_base);
		1848	/* EVENT_WRITE_EOP - flush caches, send int */
		1849	radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		1850	radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) \| EVENT_INDEX(5));
		1851	radeon_ring_write(rdev, addr & 0xffffffff);
		1852	radeon_ring_write(rdev, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		1853	radeon_ring_write(rdev, fence->seq);
		1854	radeon_ring_write(rdev, 0);
		1855	} else {
1430	serge	1856	radeon_ring_write(rdev, PACKET3(PACKET3_EVENT_WRITE, 0));
1963	serge	1857	radeon_ring_write(rdev, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) \| EVENT_INDEX(0));
1430	serge	1858	/* wait for 3D idle clean */
		1859	radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1860	radeon_ring_write(rdev, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		1861	radeon_ring_write(rdev, WAIT_3D_IDLE_bit \| WAIT_3D_IDLECLEAN_bit);
1413	serge	1862	/* Emit fence sequence & fire IRQ */
		1863	radeon_ring_write(rdev, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		1864	radeon_ring_write(rdev, ((rdev->fence_drv.scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		1865	radeon_ring_write(rdev, fence->seq);
		1866	/* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
		1867	radeon_ring_write(rdev, PACKET0(CP_INT_STATUS, 0));
		1868	radeon_ring_write(rdev, RB_INT_STAT);
1963	serge	1869	}
1413	serge	1870	}
1963	serge	1871
2005	serge	1872	int r600_copy_blit(struct radeon_device *rdev,
		1873	uint64_t src_offset, uint64_t dst_offset,
		1874	unsigned num_pages, struct radeon_fence *fence)
		1875	{
		1876	int r;
1963	serge	1877
2005	serge	1878	mutex_lock(&rdev->r600_blit.mutex);
		1879	rdev->r600_blit.vb_ib = NULL;
		1880	r = r600_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
		1881	if (r) {
		1882	// if (rdev->r600_blit.vb_ib)
		1883	// radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
		1884	mutex_unlock(&rdev->r600_blit.mutex);
		1885	return r;
		1886	}
		1887	r600_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
		1888	r600_blit_done_copy(rdev, fence);
		1889	mutex_unlock(&rdev->r600_blit.mutex);
		1890	return 0;
		1891	}
		1892
1221	serge	1893	int r600_set_surface_reg(struct radeon_device *rdev, int reg,
		1894	uint32_t tiling_flags, uint32_t pitch,
		1895	uint32_t offset, uint32_t obj_size)
		1896	{
		1897	/* FIXME: implement */
		1898	return 0;
		1899	}
		1900
		1901	void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
		1902	{
		1903	/* FIXME: implement */
		1904	}
		1905
		1906	int r600_startup(struct radeon_device *rdev)
		1907	{
		1908	int r;
		1909
1963	serge	1910	/* enable pcie gen2 link */
		1911	r600_pcie_gen2_enable(rdev);
		1912
1413	serge	1913	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		1914	r = r600_init_microcode(rdev);
		1915	if (r) {
		1916	DRM_ERROR("Failed to load firmware!\n");
		1917	return r;
		1918	}
		1919	}
		1920
1221	serge	1921	r600_mc_program(rdev);
		1922	if (rdev->flags & RADEON_IS_AGP) {
		1923	r600_agp_enable(rdev);
		1924	} else {
		1925	r = r600_pcie_gart_enable(rdev);
		1926	if (r)
		1927	return r;
		1928	}
		1929	r600_gpu_init(rdev);
2005	serge	1930	r = r600_blit_init(rdev);
		1931	if (r) {
		1932	// r600_blit_fini(rdev);
		1933	rdev->asic->copy = NULL;
		1934	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		1935	}
1221	serge	1936
2005	serge	1937	/* allocate wb buffer */
		1938	r = radeon_wb_init(rdev);
		1939	if (r)
		1940	return r;
		1941
		1942	/* Enable IRQ */
		1943	r = r600_irq_init(rdev);
		1944	if (r) {
		1945	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		1946	// radeon_irq_kms_fini(rdev);
		1947	return r;
		1948	}
		1949	r600_irq_set(rdev);
		1950
1413	serge	1951	r = radeon_ring_init(rdev, rdev->cp.ring_size);
		1952	if (r)
		1953	return r;
		1954	r = r600_cp_load_microcode(rdev);
		1955	if (r)
		1956	return r;
		1957	r = r600_cp_resume(rdev);
		1958	if (r)
		1959	return r;
1963	serge	1960
1221	serge	1961	return 0;
		1962	}
		1963
		1964	void r600_vga_set_state(struct radeon_device *rdev, bool state)
		1965	{
		1966	uint32_t temp;
		1967
		1968	temp = RREG32(CONFIG_CNTL);
		1969	if (state == false) {
		1970	temp &= ~(1<<0);
		1971	temp \|= (1<<1);
		1972	} else {
		1973	temp &= ~(1<<1);
		1974	}
		1975	WREG32(CONFIG_CNTL, temp);
		1976	}
		1977
		1978
		1979
		1980
		1981
		1982	/* Plan is to move initialization in that function and use
		1983	* helper function so that radeon_device_init pretty much
		1984	* do nothing more than calling asic specific function. This
		1985	* should also allow to remove a bunch of callback function
		1986	* like vram_info.
		1987	*/
		1988	int r600_init(struct radeon_device *rdev)
		1989	{
		1990	int r;
		1991
		1992	if (r600_debugfs_mc_info_init(rdev)) {
		1993	DRM_ERROR("Failed to register debugfs file for mc !\n");
		1994	}
		1995	/* This don't do much */
2004	serge	1996	r = radeon_gem_init(rdev);
		1997	if (r)
		1998	return r;
1221	serge	1999	/* Read BIOS */
		2000	if (!radeon_get_bios(rdev)) {
		2001	if (ASIC_IS_AVIVO(rdev))
		2002	return -EINVAL;
		2003	}
		2004	/* Must be an ATOMBIOS */
		2005	if (!rdev->is_atom_bios) {
		2006	dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
		2007	return -EINVAL;
		2008	}
		2009	r = radeon_atombios_init(rdev);
		2010	if (r)
		2011	return r;
		2012	/* Post card if necessary */
1963	serge	2013	if (!radeon_card_posted(rdev)) {
1321	serge	2014	if (!rdev->bios) {
		2015	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		2016	return -EINVAL;
		2017	}
1221	serge	2018	DRM_INFO("GPU not posted. posting now...\n");
		2019	atom_asic_init(rdev->mode_info.atom_context);
		2020	}
		2021	/* Initialize scratch registers */
		2022	r600_scratch_init(rdev);
		2023	/* Initialize surface registers */
		2024	radeon_surface_init(rdev);
1268	serge	2025	/* Initialize clocks */
1221	serge	2026	radeon_get_clock_info(rdev->ddev);
		2027	/* Fence driver */
2004	serge	2028	r = radeon_fence_driver_init(rdev);
		2029	if (r)
		2030	return r;
1403	serge	2031	if (rdev->flags & RADEON_IS_AGP) {
		2032	r = radeon_agp_init(rdev);
		2033	if (r)
		2034	radeon_agp_disable(rdev);
		2035	}
1221	serge	2036	r = r600_mc_init(rdev);
		2037	if (r)
		2038	return r;
		2039	/* Memory manager */
1321	serge	2040	r = radeon_bo_init(rdev);
1221	serge	2041	if (r)
		2042	return r;
1321	serge	2043
2004	serge	2044	r = radeon_irq_kms_init(rdev);
		2045	if (r)
		2046	return r;
1321	serge	2047
1413	serge	2048	rdev->cp.ring_obj = NULL;
		2049	r600_ring_init(rdev, 1024 * 1024);
1221	serge	2050
2004	serge	2051	rdev->ih.ring_obj = NULL;
		2052	r600_ih_ring_init(rdev, 64 * 1024);
1221	serge	2053
		2054	r = r600_pcie_gart_init(rdev);
		2055	if (r)
		2056	return r;
		2057
1321	serge	2058	rdev->accel_working = true;
1221	serge	2059	r = r600_startup(rdev);
		2060	if (r) {
1428	serge	2061	dev_err(rdev->dev, "disabling GPU acceleration\n");
1221	serge	2062	// r600_suspend(rdev);
		2063	// r600_wb_fini(rdev);
		2064	// radeon_ring_fini(rdev);
		2065	r600_pcie_gart_fini(rdev);
		2066	rdev->accel_working = false;
		2067	}
		2068	if (rdev->accel_working) {
2005	serge	2069	r = radeon_ib_pool_init(rdev);
		2070	if (r) {
		2071	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		2072	rdev->accel_working = false;
		2073	} else {
		2074	r = r600_ib_test(rdev);
		2075	if (r) {
		2076	dev_err(rdev->dev, "IB test failed (%d).\n", r);
		2077	rdev->accel_working = false;
		2078	}
1221	serge	2079	}
2005	serge	2080	}
		2081
1221	serge	2082	return 0;
		2083	}
		2084
2004	serge	2085	/*
		2086	* CS stuff
		2087	*/
		2088	void r600_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		2089	{
		2090	/* FIXME: implement */
		2091	radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		2092	radeon_ring_write(rdev,
		2093	#ifdef __BIG_ENDIAN
		2094	(2 << 0) \|
		2095	#endif
		2096	(ib->gpu_addr & 0xFFFFFFFC));
		2097	radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
		2098	radeon_ring_write(rdev, ib->length_dw);
		2099	}
		2100
		2101	int r600_ib_test(struct radeon_device *rdev)
		2102	{
		2103	struct radeon_ib *ib;
		2104	uint32_t scratch;
		2105	uint32_t tmp = 0;
		2106	unsigned i;
		2107	int r;
		2108
		2109	r = radeon_scratch_get(rdev, &scratch);
		2110	if (r) {
		2111	DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
		2112	return r;
		2113	}
		2114	WREG32(scratch, 0xCAFEDEAD);
		2115	r = radeon_ib_get(rdev, &ib);
		2116	if (r) {
		2117	DRM_ERROR("radeon: failed to get ib (%d).\n", r);
		2118	return r;
		2119	}
		2120	ib->ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
		2121	ib->ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		2122	ib->ptr[2] = 0xDEADBEEF;
		2123	ib->ptr[3] = PACKET2(0);
		2124	ib->ptr[4] = PACKET2(0);
		2125	ib->ptr[5] = PACKET2(0);
		2126	ib->ptr[6] = PACKET2(0);
		2127	ib->ptr[7] = PACKET2(0);
		2128	ib->ptr[8] = PACKET2(0);
		2129	ib->ptr[9] = PACKET2(0);
		2130	ib->ptr[10] = PACKET2(0);
		2131	ib->ptr[11] = PACKET2(0);
		2132	ib->ptr[12] = PACKET2(0);
		2133	ib->ptr[13] = PACKET2(0);
		2134	ib->ptr[14] = PACKET2(0);
		2135	ib->ptr[15] = PACKET2(0);
		2136	ib->length_dw = 16;
		2137	r = radeon_ib_schedule(rdev, ib);
		2138	if (r) {
		2139	radeon_scratch_free(rdev, scratch);
		2140	radeon_ib_free(rdev, &ib);
		2141	DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
		2142	return r;
		2143	}
		2144	r = radeon_fence_wait(ib->fence, false);
		2145	if (r) {
		2146	DRM_ERROR("radeon: fence wait failed (%d).\n", r);
		2147	return r;
		2148	}
		2149	for (i = 0; i < rdev->usec_timeout; i++) {
		2150	tmp = RREG32(scratch);
		2151	if (tmp == 0xDEADBEEF)
		2152	break;
		2153	DRM_UDELAY(1);
		2154	}
		2155	if (i < rdev->usec_timeout) {
		2156	DRM_INFO("ib test succeeded in %u usecs\n", i);
		2157	} else {
		2158	DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
		2159	scratch, tmp);
		2160	r = -EINVAL;
		2161	}
		2162	radeon_scratch_free(rdev, scratch);
		2163	radeon_ib_free(rdev, &ib);
		2164	return r;
		2165	}
		2166
		2167	/*
		2168	* Interrupts
		2169	*
		2170	* Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
		2171	* the same as the CP ring buffer, but in reverse. Rather than the CPU
		2172	* writing to the ring and the GPU consuming, the GPU writes to the ring
		2173	* and host consumes. As the host irq handler processes interrupts, it
		2174	* increments the rptr. When the rptr catches up with the wptr, all the
		2175	* current interrupts have been processed.
		2176	*/
		2177
		2178	void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
		2179	{
		2180	u32 rb_bufsz;
		2181
		2182	/* Align ring size */
		2183	rb_bufsz = drm_order(ring_size / 4);
		2184	ring_size = (1 << rb_bufsz) * 4;
		2185	rdev->ih.ring_size = ring_size;
		2186	rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
		2187	rdev->ih.rptr = 0;
		2188	}
		2189
		2190	static int r600_ih_ring_alloc(struct radeon_device *rdev)
		2191	{
		2192	int r;
		2193
		2194	/* Allocate ring buffer */
		2195	if (rdev->ih.ring_obj == NULL) {
		2196	r = radeon_bo_create(rdev, rdev->ih.ring_size,
		2197	PAGE_SIZE, true,
		2198	RADEON_GEM_DOMAIN_GTT,
		2199	&rdev->ih.ring_obj);
		2200	if (r) {
		2201	DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
		2202	return r;
		2203	}
		2204	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		2205	if (unlikely(r != 0))
		2206	return r;
		2207	r = radeon_bo_pin(rdev->ih.ring_obj,
		2208	RADEON_GEM_DOMAIN_GTT,
		2209	&rdev->ih.gpu_addr);
		2210	if (r) {
		2211	radeon_bo_unreserve(rdev->ih.ring_obj);
		2212	DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
		2213	return r;
		2214	}
		2215	r = radeon_bo_kmap(rdev->ih.ring_obj,
		2216	(void **)&rdev->ih.ring);
		2217	radeon_bo_unreserve(rdev->ih.ring_obj);
		2218	if (r) {
		2219	DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
		2220	return r;
		2221	}
		2222	}
		2223	return 0;
		2224	}
		2225
		2226	static void r600_ih_ring_fini(struct radeon_device *rdev)
		2227	{
		2228	int r;
		2229	if (rdev->ih.ring_obj) {
		2230	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		2231	if (likely(r == 0)) {
		2232	radeon_bo_kunmap(rdev->ih.ring_obj);
		2233	radeon_bo_unpin(rdev->ih.ring_obj);
		2234	radeon_bo_unreserve(rdev->ih.ring_obj);
		2235	}
		2236	radeon_bo_unref(&rdev->ih.ring_obj);
		2237	rdev->ih.ring = NULL;
		2238	rdev->ih.ring_obj = NULL;
		2239	}
		2240	}
		2241
		2242	void r600_rlc_stop(struct radeon_device *rdev)
		2243	{
		2244
		2245	if ((rdev->family >= CHIP_RV770) &&
		2246	(rdev->family <= CHIP_RV740)) {
		2247	/* r7xx asics need to soft reset RLC before halting */
		2248	WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
		2249	RREG32(SRBM_SOFT_RESET);
		2250	udelay(15000);
		2251	WREG32(SRBM_SOFT_RESET, 0);
		2252	RREG32(SRBM_SOFT_RESET);
		2253	}
		2254
		2255	WREG32(RLC_CNTL, 0);
		2256	}
		2257
		2258	static void r600_rlc_start(struct radeon_device *rdev)
		2259	{
		2260	WREG32(RLC_CNTL, RLC_ENABLE);
		2261	}
		2262
		2263	static int r600_rlc_init(struct radeon_device *rdev)
		2264	{
		2265	u32 i;
		2266	const __be32 *fw_data;
		2267
		2268	if (!rdev->rlc_fw)
		2269	return -EINVAL;
		2270
		2271	r600_rlc_stop(rdev);
		2272
		2273	WREG32(RLC_HB_BASE, 0);
		2274	WREG32(RLC_HB_CNTL, 0);
		2275	WREG32(RLC_HB_RPTR, 0);
		2276	WREG32(RLC_HB_WPTR, 0);
		2277	if (rdev->family <= CHIP_CAICOS) {
		2278	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
		2279	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
		2280	}
		2281	WREG32(RLC_MC_CNTL, 0);
		2282	WREG32(RLC_UCODE_CNTL, 0);
		2283
		2284	fw_data = (const __be32 *)rdev->rlc_fw->data;
		2285	if (rdev->family >= CHIP_CAYMAN) {
		2286	for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
		2287	WREG32(RLC_UCODE_ADDR, i);
		2288	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2289	}
		2290	} else if (rdev->family >= CHIP_CEDAR) {
		2291	for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
		2292	WREG32(RLC_UCODE_ADDR, i);
		2293	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2294	}
		2295	} else if (rdev->family >= CHIP_RV770) {
		2296	for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
		2297	WREG32(RLC_UCODE_ADDR, i);
		2298	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2299	}
		2300	} else {
		2301	for (i = 0; i < RLC_UCODE_SIZE; i++) {
		2302	WREG32(RLC_UCODE_ADDR, i);
		2303	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		2304	}
		2305	}
		2306	WREG32(RLC_UCODE_ADDR, 0);
		2307
		2308	r600_rlc_start(rdev);
		2309
		2310	return 0;
		2311	}
		2312
		2313	static void r600_enable_interrupts(struct radeon_device *rdev)
		2314	{
		2315	u32 ih_cntl = RREG32(IH_CNTL);
		2316	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		2317
		2318	ih_cntl \|= ENABLE_INTR;
		2319	ih_rb_cntl \|= IH_RB_ENABLE;
		2320	WREG32(IH_CNTL, ih_cntl);
		2321	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2322	rdev->ih.enabled = true;
		2323	}
		2324
		2325	void r600_disable_interrupts(struct radeon_device *rdev)
		2326	{
		2327	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		2328	u32 ih_cntl = RREG32(IH_CNTL);
		2329
		2330	ih_rb_cntl &= ~IH_RB_ENABLE;
		2331	ih_cntl &= ~ENABLE_INTR;
		2332	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2333	WREG32(IH_CNTL, ih_cntl);
		2334	/* set rptr, wptr to 0 */
		2335	WREG32(IH_RB_RPTR, 0);
		2336	WREG32(IH_RB_WPTR, 0);
		2337	rdev->ih.enabled = false;
		2338	rdev->ih.wptr = 0;
		2339	rdev->ih.rptr = 0;
		2340	}
		2341
1963	serge	2342	static void r600_disable_interrupt_state(struct radeon_device *rdev)
		2343	{
		2344	u32 tmp;
1221	serge	2345
1963	serge	2346	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
		2347	WREG32(GRBM_INT_CNTL, 0);
		2348	WREG32(DxMODE_INT_MASK, 0);
		2349	WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
		2350	WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
		2351	if (ASIC_IS_DCE3(rdev)) {
		2352	WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
		2353	WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
		2354	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2355	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2356	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2357	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2358	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2359	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2360	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2361	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2362	if (ASIC_IS_DCE32(rdev)) {
		2363	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2364	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2365	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		2366	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2367	}
		2368	} else {
		2369	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		2370	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		2371	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2372	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		2373	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2374	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		2375	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		2376	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		2377	}
		2378	}
1221	serge	2379
2004	serge	2380	int r600_irq_init(struct radeon_device *rdev)
		2381	{
		2382	int ret = 0;
		2383	int rb_bufsz;
		2384	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
1221	serge	2385
2004	serge	2386	/* allocate ring */
		2387	ret = r600_ih_ring_alloc(rdev);
		2388	if (ret)
		2389	return ret;
1221	serge	2390
2004	serge	2391	/* disable irqs */
		2392	r600_disable_interrupts(rdev);
1221	serge	2393
2004	serge	2394	/* init rlc */
		2395	ret = r600_rlc_init(rdev);
		2396	if (ret) {
		2397	r600_ih_ring_fini(rdev);
		2398	return ret;
		2399	}
1221	serge	2400
2004	serge	2401	/* setup interrupt control */
		2402	/* set dummy read address to ring address */
		2403	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
		2404	interrupt_cntl = RREG32(INTERRUPT_CNTL);
		2405	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
		2406	* IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
		2407	*/
		2408	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
		2409	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
		2410	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
		2411	WREG32(INTERRUPT_CNTL, interrupt_cntl);
1221	serge	2412
2004	serge	2413	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
		2414	rb_bufsz = drm_order(rdev->ih.ring_size / 4);
1221	serge	2415
2004	serge	2416	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE \|
		2417	IH_WPTR_OVERFLOW_CLEAR \|
		2418	(rb_bufsz << 1));
1963	serge	2419
2004	serge	2420	if (rdev->wb.enabled)
		2421	ih_rb_cntl \|= IH_WPTR_WRITEBACK_ENABLE;
		2422
		2423	/* set the writeback address whether it's enabled or not */
		2424	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
		2425	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
		2426
		2427	WREG32(IH_RB_CNTL, ih_rb_cntl);
		2428
		2429	/* set rptr, wptr to 0 */
		2430	WREG32(IH_RB_RPTR, 0);
		2431	WREG32(IH_RB_WPTR, 0);
		2432
		2433	/* Default settings for IH_CNTL (disabled at first) */
		2434	ih_cntl = MC_WRREQ_CREDIT(0x10) \| MC_WR_CLEAN_CNT(0x10);
		2435	/* RPTR_REARM only works if msi's are enabled */
		2436	if (rdev->msi_enabled)
		2437	ih_cntl \|= RPTR_REARM;
		2438	WREG32(IH_CNTL, ih_cntl);
		2439
		2440	/* force the active interrupt state to all disabled */
		2441	if (rdev->family >= CHIP_CEDAR)
		2442	evergreen_disable_interrupt_state(rdev);
		2443	else
		2444	r600_disable_interrupt_state(rdev);
		2445
		2446	/* enable irqs */
		2447	r600_enable_interrupts(rdev);
		2448
		2449	return ret;
		2450	}
		2451	int r600_irq_set(struct radeon_device *rdev)
		2452	{
		2453	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
		2454	u32 mode_int = 0;
		2455	u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
		2456	u32 grbm_int_cntl = 0;
		2457	u32 hdmi1, hdmi2;
		2458	u32 d1grph = 0, d2grph = 0;
		2459
		2460	if (!rdev->irq.installed) {
		2461	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		2462	return -EINVAL;
		2463	}
		2464	/* don't enable anything if the ih is disabled */
		2465	if (!rdev->ih.enabled) {
		2466	r600_disable_interrupts(rdev);
		2467	/* force the active interrupt state to all disabled */
		2468	r600_disable_interrupt_state(rdev);
		2469	return 0;
		2470	}
		2471
		2472	hdmi1 = RREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
		2473	if (ASIC_IS_DCE3(rdev)) {
		2474	hdmi2 = RREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
		2475	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2476	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2477	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2478	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2479	if (ASIC_IS_DCE32(rdev)) {
		2480	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2481	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2482	}
		2483	} else {
		2484	hdmi2 = RREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL) & ~R600_HDMI_INT_EN;
		2485	hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2486	hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2487	hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		2488	}
		2489
		2490	if (rdev->irq.sw_int) {
		2491	DRM_DEBUG("r600_irq_set: sw int\n");
		2492	cp_int_cntl \|= RB_INT_ENABLE;
		2493	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		2494	}
		2495	if (rdev->irq.crtc_vblank_int[0] \|\|
		2496	rdev->irq.pflip[0]) {
		2497	DRM_DEBUG("r600_irq_set: vblank 0\n");
		2498	mode_int \|= D1MODE_VBLANK_INT_MASK;
		2499	}
		2500	if (rdev->irq.crtc_vblank_int[1] \|\|
		2501	rdev->irq.pflip[1]) {
		2502	DRM_DEBUG("r600_irq_set: vblank 1\n");
		2503	mode_int \|= D2MODE_VBLANK_INT_MASK;
		2504	}
		2505	if (rdev->irq.hpd[0]) {
		2506	DRM_DEBUG("r600_irq_set: hpd 1\n");
		2507	hpd1 \|= DC_HPDx_INT_EN;
		2508	}
		2509	if (rdev->irq.hpd[1]) {
		2510	DRM_DEBUG("r600_irq_set: hpd 2\n");
		2511	hpd2 \|= DC_HPDx_INT_EN;
		2512	}
		2513	if (rdev->irq.hpd[2]) {
		2514	DRM_DEBUG("r600_irq_set: hpd 3\n");
		2515	hpd3 \|= DC_HPDx_INT_EN;
		2516	}
		2517	if (rdev->irq.hpd[3]) {
		2518	DRM_DEBUG("r600_irq_set: hpd 4\n");
		2519	hpd4 \|= DC_HPDx_INT_EN;
		2520	}
		2521	if (rdev->irq.hpd[4]) {
		2522	DRM_DEBUG("r600_irq_set: hpd 5\n");
		2523	hpd5 \|= DC_HPDx_INT_EN;
		2524	}
		2525	if (rdev->irq.hpd[5]) {
		2526	DRM_DEBUG("r600_irq_set: hpd 6\n");
		2527	hpd6 \|= DC_HPDx_INT_EN;
		2528	}
		2529	if (rdev->irq.hdmi[0]) {
		2530	DRM_DEBUG("r600_irq_set: hdmi 1\n");
		2531	hdmi1 \|= R600_HDMI_INT_EN;
		2532	}
		2533	if (rdev->irq.hdmi[1]) {
		2534	DRM_DEBUG("r600_irq_set: hdmi 2\n");
		2535	hdmi2 \|= R600_HDMI_INT_EN;
		2536	}
		2537	if (rdev->irq.gui_idle) {
		2538	DRM_DEBUG("gui idle\n");
		2539	grbm_int_cntl \|= GUI_IDLE_INT_ENABLE;
		2540	}
		2541
		2542	WREG32(CP_INT_CNTL, cp_int_cntl);
		2543	WREG32(DxMODE_INT_MASK, mode_int);
		2544	WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
		2545	WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
		2546	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		2547	WREG32(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, hdmi1);
		2548	if (ASIC_IS_DCE3(rdev)) {
		2549	WREG32(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, hdmi2);
		2550	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		2551	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		2552	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		2553	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		2554	if (ASIC_IS_DCE32(rdev)) {
		2555	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		2556	WREG32(DC_HPD6_INT_CONTROL, hpd6);
		2557	}
		2558	} else {
		2559	WREG32(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, hdmi2);
		2560	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
		2561	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
		2562	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
		2563	}
		2564
		2565	return 0;
		2566	}
		2567
		2568	static inline void r600_irq_ack(struct radeon_device *rdev)
		2569	{
		2570	u32 tmp;
		2571
		2572	if (ASIC_IS_DCE3(rdev)) {
		2573	rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
		2574	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
		2575	rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
		2576	} else {
		2577	rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		2578	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		2579	rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
		2580	}
		2581	rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
		2582	rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
		2583
		2584	if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		2585	WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		2586	if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		2587	WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		2588	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
		2589	WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		2590	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
		2591	WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		2592	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
		2593	WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		2594	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
		2595	WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		2596	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		2597	if (ASIC_IS_DCE3(rdev)) {
		2598	tmp = RREG32(DC_HPD1_INT_CONTROL);
		2599	tmp \|= DC_HPDx_INT_ACK;
		2600	WREG32(DC_HPD1_INT_CONTROL, tmp);
		2601	} else {
		2602	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		2603	tmp \|= DC_HPDx_INT_ACK;
		2604	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		2605	}
		2606	}
		2607	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		2608	if (ASIC_IS_DCE3(rdev)) {
		2609	tmp = RREG32(DC_HPD2_INT_CONTROL);
		2610	tmp \|= DC_HPDx_INT_ACK;
		2611	WREG32(DC_HPD2_INT_CONTROL, tmp);
		2612	} else {
		2613	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		2614	tmp \|= DC_HPDx_INT_ACK;
		2615	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		2616	}
		2617	}
		2618	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		2619	if (ASIC_IS_DCE3(rdev)) {
		2620	tmp = RREG32(DC_HPD3_INT_CONTROL);
		2621	tmp \|= DC_HPDx_INT_ACK;
		2622	WREG32(DC_HPD3_INT_CONTROL, tmp);
		2623	} else {
		2624	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		2625	tmp \|= DC_HPDx_INT_ACK;
		2626	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		2627	}
		2628	}
		2629	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		2630	tmp = RREG32(DC_HPD4_INT_CONTROL);
		2631	tmp \|= DC_HPDx_INT_ACK;
		2632	WREG32(DC_HPD4_INT_CONTROL, tmp);
		2633	}
		2634	if (ASIC_IS_DCE32(rdev)) {
		2635	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		2636	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2637	tmp \|= DC_HPDx_INT_ACK;
		2638	WREG32(DC_HPD5_INT_CONTROL, tmp);
		2639	}
		2640	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		2641	tmp = RREG32(DC_HPD5_INT_CONTROL);
		2642	tmp \|= DC_HPDx_INT_ACK;
		2643	WREG32(DC_HPD6_INT_CONTROL, tmp);
		2644	}
		2645	}
		2646	if (RREG32(R600_HDMI_BLOCK1 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
		2647	WREG32_P(R600_HDMI_BLOCK1 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
		2648	}
		2649	if (ASIC_IS_DCE3(rdev)) {
		2650	if (RREG32(R600_HDMI_BLOCK3 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
		2651	WREG32_P(R600_HDMI_BLOCK3 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
		2652	}
		2653	} else {
		2654	if (RREG32(R600_HDMI_BLOCK2 + R600_HDMI_STATUS) & R600_HDMI_INT_PENDING) {
		2655	WREG32_P(R600_HDMI_BLOCK2 + R600_HDMI_CNTL, R600_HDMI_INT_ACK, ~R600_HDMI_INT_ACK);
		2656	}
		2657	}
		2658	}
		2659
		2660	static inline u32 r600_get_ih_wptr(struct radeon_device *rdev)
		2661	{
		2662	u32 wptr, tmp;
		2663
		2664	if (rdev->wb.enabled)
		2665	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		2666	else
		2667	wptr = RREG32(IH_RB_WPTR);
		2668
		2669	if (wptr & RB_OVERFLOW) {
		2670	/* When a ring buffer overflow happen start parsing interrupt
		2671	* from the last not overwritten vector (wptr + 16). Hopefully
		2672	* this should allow us to catchup.
		2673	*/
		2674	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
		2675	wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
		2676	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		2677	tmp = RREG32(IH_RB_CNTL);
		2678	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		2679	WREG32(IH_RB_CNTL, tmp);
		2680	}
		2681	return (wptr & rdev->ih.ptr_mask);
		2682	}
		2683
		2684	/* r600 IV Ring
		2685	* Each IV ring entry is 128 bits:
		2686	* [7:0] - interrupt source id
		2687	* [31:8] - reserved
		2688	* [59:32] - interrupt source data
		2689	* [127:60] - reserved
		2690	*
		2691	* The basic interrupt vector entries
		2692	* are decoded as follows:
		2693	* src_id src_data description
		2694	* 1 0 D1 Vblank
		2695	* 1 1 D1 Vline
		2696	* 5 0 D2 Vblank
		2697	* 5 1 D2 Vline
		2698	* 19 0 FP Hot plug detection A
		2699	* 19 1 FP Hot plug detection B
		2700	* 19 2 DAC A auto-detection
		2701	* 19 3 DAC B auto-detection
		2702	* 21 4 HDMI block A
		2703	* 21 5 HDMI block B
		2704	* 176 - CP_INT RB
		2705	* 177 - CP_INT IB1
		2706	* 178 - CP_INT IB2
		2707	* 181 - EOP Interrupt
		2708	* 233 - GUI Idle
		2709	*
		2710	* Note, these are based on r600 and may need to be
		2711	* adjusted or added to on newer asics
		2712	*/
		2713
2160	serge	2714	#define DRM_DEBUG(...)
		2715
2004	serge	2716	int r600_irq_process(struct radeon_device *rdev)
		2717	{
		2718	u32 wptr;
		2719	u32 rptr;
		2720	u32 src_id, src_data;
		2721	u32 ring_index;
		2722	unsigned long flags;
		2723	bool queue_hotplug = false;
		2724
		2725	if (!rdev->ih.enabled \|\| rdev->shutdown)
		2726	return IRQ_NONE;
		2727
2160	serge	2728	/* No MSIs, need a dummy read to flush PCI DMAs */
		2729	if (!rdev->msi_enabled)
		2730	RREG32(IH_RB_WPTR);
		2731
2004	serge	2732	wptr = r600_get_ih_wptr(rdev);
		2733	rptr = rdev->ih.rptr;
2160	serge	2734	// DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
2004	serge	2735
		2736	spin_lock_irqsave(&rdev->ih.lock, flags);
		2737
		2738	if (rptr == wptr) {
		2739	spin_unlock_irqrestore(&rdev->ih.lock, flags);
		2740	return IRQ_NONE;
		2741	}
		2742
		2743	restart_ih:
2160	serge	2744	/* Order reading of wptr vs. reading of IH ring data */
		2745	rmb();
		2746
2004	serge	2747	/* display interrupts */
		2748	r600_irq_ack(rdev);
		2749
		2750	rdev->ih.wptr = wptr;
		2751	while (rptr != wptr) {
		2752	/* wptr/rptr are in bytes! */
		2753	ring_index = rptr / 4;
		2754	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		2755	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		2756
		2757	switch (src_id) {
		2758	case 1: /* D1 vblank/vline */
		2759	switch (src_data) {
		2760	case 0: /* D1 vblank */
		2761	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
		2762	if (rdev->irq.crtc_vblank_int[0]) {
		2763	// drm_handle_vblank(rdev->ddev, 0);
		2764	rdev->pm.vblank_sync = true;
		2765	// wake_up(&rdev->irq.vblank_queue);
		2766	}
		2767	// if (rdev->irq.pflip[0])
		2768	// radeon_crtc_handle_flip(rdev, 0);
		2769	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		2770	DRM_DEBUG("IH: D1 vblank\n");
		2771	}
		2772	break;
		2773	case 1: /* D1 vline */
		2774	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
		2775	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		2776	DRM_DEBUG("IH: D1 vline\n");
		2777	}
		2778	break;
		2779	default:
		2780	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2781	break;
		2782	}
		2783	break;
		2784	case 5: /* D2 vblank/vline */
		2785	switch (src_data) {
		2786	case 0: /* D2 vblank */
		2787	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
		2788	if (rdev->irq.crtc_vblank_int[1]) {
		2789	// drm_handle_vblank(rdev->ddev, 1);
		2790	rdev->pm.vblank_sync = true;
		2791	// wake_up(&rdev->irq.vblank_queue);
		2792	}
		2793	// if (rdev->irq.pflip[1])
		2794	// radeon_crtc_handle_flip(rdev, 1);
		2795	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
		2796	DRM_DEBUG("IH: D2 vblank\n");
		2797	}
		2798	break;
		2799	case 1: /* D1 vline */
		2800	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
		2801	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
		2802	DRM_DEBUG("IH: D2 vline\n");
		2803	}
		2804	break;
		2805	default:
		2806	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2807	break;
		2808	}
		2809	break;
		2810	case 19: /* HPD/DAC hotplug */
		2811	switch (src_data) {
		2812	case 0:
		2813	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		2814	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
		2815	queue_hotplug = true;
		2816	DRM_DEBUG("IH: HPD1\n");
		2817	}
		2818	break;
		2819	case 1:
		2820	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		2821	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
		2822	queue_hotplug = true;
		2823	DRM_DEBUG("IH: HPD2\n");
		2824	}
		2825	break;
		2826	case 4:
		2827	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		2828	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
		2829	queue_hotplug = true;
		2830	DRM_DEBUG("IH: HPD3\n");
		2831	}
		2832	break;
		2833	case 5:
		2834	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		2835	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
		2836	queue_hotplug = true;
		2837	DRM_DEBUG("IH: HPD4\n");
		2838	}
		2839	break;
		2840	case 10:
		2841	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		2842	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
		2843	queue_hotplug = true;
		2844	DRM_DEBUG("IH: HPD5\n");
		2845	}
		2846	break;
		2847	case 12:
		2848	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		2849	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
		2850	queue_hotplug = true;
		2851	DRM_DEBUG("IH: HPD6\n");
		2852	}
		2853	break;
		2854	default:
		2855	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2856	break;
		2857	}
		2858	break;
		2859	case 21: /* HDMI */
		2860	DRM_DEBUG("IH: HDMI: 0x%x\n", src_data);
		2861	// r600_audio_schedule_polling(rdev);
		2862	break;
		2863	case 176: /* CP_INT in ring buffer */
		2864	case 177: /* CP_INT in IB1 */
		2865	case 178: /* CP_INT in IB2 */
		2866	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2005	serge	2867	radeon_fence_process(rdev);
2004	serge	2868	break;
		2869	case 181: /* CP EOP event */
		2870	DRM_DEBUG("IH: CP EOP\n");
2005	serge	2871	radeon_fence_process(rdev);
2004	serge	2872	break;
		2873	case 233: /* GUI IDLE */
		2874	DRM_DEBUG("IH: GUI idle\n");
		2875	rdev->pm.gui_idle = true;
		2876	// wake_up(&rdev->irq.idle_queue);
		2877	break;
		2878	default:
		2879	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		2880	break;
		2881	}
		2882
		2883	/* wptr/rptr are in bytes! */
		2884	rptr += 16;
		2885	rptr &= rdev->ih.ptr_mask;
		2886	}
		2887	/* make sure wptr hasn't changed while processing */
		2888	wptr = r600_get_ih_wptr(rdev);
		2889	if (wptr != rdev->ih.wptr)
		2890	goto restart_ih;
		2891	// if (queue_hotplug)
		2892	// schedule_work(&rdev->hotplug_work);
		2893	rdev->ih.rptr = rptr;
		2894	WREG32(IH_RB_RPTR, rdev->ih.rptr);
		2895	spin_unlock_irqrestore(&rdev->ih.lock, flags);
		2896	return IRQ_HANDLED;
		2897	}
		2898
1221	serge	2899	/*
		2900	* Debugfs info
		2901	*/
		2902	#if defined(CONFIG_DEBUG_FS)
		2903
		2904	static int r600_debugfs_cp_ring_info(struct seq_file m, void data)
		2905	{
		2906	struct drm_info_node node = (struct drm_info_node ) m->private;
		2907	struct drm_device *dev = node->minor->dev;
		2908	struct radeon_device *rdev = dev->dev_private;
		2909	unsigned count, i, j;
		2910
		2911	radeon_ring_free_size(rdev);
1321	serge	2912	count = (rdev->cp.ring_size / 4) - rdev->cp.ring_free_dw;
1221	serge	2913	seq_printf(m, "CP_STAT 0x%08x\n", RREG32(CP_STAT));
1321	serge	2914	seq_printf(m, "CP_RB_WPTR 0x%08x\n", RREG32(CP_RB_WPTR));
		2915	seq_printf(m, "CP_RB_RPTR 0x%08x\n", RREG32(CP_RB_RPTR));
		2916	seq_printf(m, "driver's copy of the CP_RB_WPTR 0x%08x\n", rdev->cp.wptr);
		2917	seq_printf(m, "driver's copy of the CP_RB_RPTR 0x%08x\n", rdev->cp.rptr);
1221	serge	2918	seq_printf(m, "%u free dwords in ring\n", rdev->cp.ring_free_dw);
		2919	seq_printf(m, "%u dwords in ring\n", count);
1321	serge	2920	i = rdev->cp.rptr;
1221	serge	2921	for (j = 0; j <= count; j++) {
		2922	seq_printf(m, "r[%04d]=0x%08x\n", i, rdev->cp.ring[i]);
1321	serge	2923	i = (i + 1) & rdev->cp.ptr_mask;
1221	serge	2924	}
		2925	return 0;
		2926	}
		2927
		2928	static int r600_debugfs_mc_info(struct seq_file m, void data)
		2929	{
		2930	struct drm_info_node node = (struct drm_info_node ) m->private;
		2931	struct drm_device *dev = node->minor->dev;
		2932	struct radeon_device *rdev = dev->dev_private;
		2933
		2934	DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
		2935	DREG32_SYS(m, rdev, VM_L2_STATUS);
		2936	return 0;
		2937	}
		2938
		2939	static struct drm_info_list r600_mc_info_list[] = {
		2940	{"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
		2941	{"r600_ring_info", r600_debugfs_cp_ring_info, 0, NULL},
		2942	};
		2943	#endif
		2944
		2945	int r600_debugfs_mc_info_init(struct radeon_device *rdev)
		2946	{
		2947	#if defined(CONFIG_DEBUG_FS)
		2948	return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
		2949	#else
		2950	return 0;
		2951	#endif
		2952	}
1404	serge	2953
		2954	/**
		2955	* r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
		2956	* rdev: radeon device structure
		2957	* bo: buffer object struct which userspace is waiting for idle
		2958	*
		2959	* Some R6XX/R7XX doesn't seems to take into account HDP flush performed
		2960	* through ring buffer, this leads to corruption in rendering, see
		2961	* http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
		2962	* directly perform HDP flush by writing register through MMIO.
		2963	*/
		2964	void r600_ioctl_wait_idle(struct radeon_device rdev, struct radeon_bo bo)
		2965	{
1963	serge	2966	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		2967	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
		2968	* This seems to cause problems on some AGP cards. Just use the old
		2969	* method for them.
		2970	*/
		2971	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		2972	rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
		2973	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		2974	u32 tmp;
		2975
		2976	WREG32(HDP_DEBUG1, 0);
		2977	tmp = readl((void __iomem *)ptr);
		2978	} else
1404	serge	2979	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		2980	}
1963	serge	2981
		2982	void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
		2983	{
		2984	u32 link_width_cntl, mask, target_reg;
		2985
		2986	if (rdev->flags & RADEON_IS_IGP)
		2987	return;
		2988
		2989	if (!(rdev->flags & RADEON_IS_PCIE))
		2990	return;
		2991
		2992	/* x2 cards have a special sequence */
		2993	if (ASIC_IS_X2(rdev))
		2994	return;
		2995
		2996	/* FIXME wait for idle */
		2997
		2998	switch (lanes) {
		2999	case 0:
		3000	mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
		3001	break;
		3002	case 1:
		3003	mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
		3004	break;
		3005	case 2:
		3006	mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
		3007	break;
		3008	case 4:
		3009	mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
		3010	break;
		3011	case 8:
		3012	mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
		3013	break;
		3014	case 12:
		3015	mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
		3016	break;
		3017	case 16:
		3018	default:
		3019	mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
		3020	break;
		3021	}
		3022
		3023	link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		3024
		3025	if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
		3026	(mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
		3027	return;
		3028
		3029	if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
		3030	return;
		3031
		3032	link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK \|
		3033	RADEON_PCIE_LC_RECONFIG_NOW \|
		3034	R600_PCIE_LC_RENEGOTIATE_EN \|
		3035	R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
		3036	link_width_cntl \|= mask;
		3037
		3038	WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3039
2005	serge	3040	/* some northbridges can renegotiate the link rather than requiring
		3041	* a complete re-config.
		3042	* e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)
1963	serge	3043	*/
		3044	if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
		3045	link_width_cntl \|= R600_PCIE_LC_RENEGOTIATE_EN \| R600_PCIE_LC_UPCONFIGURE_SUPPORT;
		3046	else
		3047	link_width_cntl \|= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
		3048
		3049	WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl \|
		3050	RADEON_PCIE_LC_RECONFIG_NOW));
		3051
		3052	if (rdev->family >= CHIP_RV770)
		3053	target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
		3054	else
		3055	target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
		3056
		3057	/* wait for lane set to complete */
		3058	link_width_cntl = RREG32(target_reg);
		3059	while (link_width_cntl == 0xffffffff)
		3060	link_width_cntl = RREG32(target_reg);
		3061
		3062	}
		3063
		3064	int r600_get_pcie_lanes(struct radeon_device *rdev)
		3065	{
		3066	u32 link_width_cntl;
		3067
		3068	if (rdev->flags & RADEON_IS_IGP)
		3069	return 0;
		3070
		3071	if (!(rdev->flags & RADEON_IS_PCIE))
		3072	return 0;
		3073
		3074	/* x2 cards have a special sequence */
		3075	if (ASIC_IS_X2(rdev))
		3076	return 0;
		3077
		3078	/* FIXME wait for idle */
		3079
		3080	link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		3081
		3082	switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
		3083	case RADEON_PCIE_LC_LINK_WIDTH_X0:
		3084	return 0;
		3085	case RADEON_PCIE_LC_LINK_WIDTH_X1:
		3086	return 1;
		3087	case RADEON_PCIE_LC_LINK_WIDTH_X2:
		3088	return 2;
		3089	case RADEON_PCIE_LC_LINK_WIDTH_X4:
		3090	return 4;
		3091	case RADEON_PCIE_LC_LINK_WIDTH_X8:
		3092	return 8;
		3093	case RADEON_PCIE_LC_LINK_WIDTH_X16:
		3094	default:
		3095	return 16;
		3096	}
		3097	}
		3098
		3099	static void r600_pcie_gen2_enable(struct radeon_device *rdev)
		3100	{
		3101	u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
		3102	u16 link_cntl2;
		3103
		3104	if (radeon_pcie_gen2 == 0)
		3105	return;
		3106
		3107	if (rdev->flags & RADEON_IS_IGP)
		3108	return;
		3109
		3110	if (!(rdev->flags & RADEON_IS_PCIE))
		3111	return;
		3112
		3113	/* x2 cards have a special sequence */
		3114	if (ASIC_IS_X2(rdev))
		3115	return;
		3116
		3117	/* only RV6xx+ chips are supported */
		3118	if (rdev->family <= CHIP_R600)
		3119	return;
		3120
		3121	/* 55 nm r6xx asics */
		3122	if ((rdev->family == CHIP_RV670) \|\|
		3123	(rdev->family == CHIP_RV620) \|\|
		3124	(rdev->family == CHIP_RV635)) {
		3125	/* advertise upconfig capability */
		3126	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3127	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3128	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3129	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3130	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
		3131	lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
		3132	link_width_cntl &= ~(LC_LINK_WIDTH_MASK \|
		3133	LC_RECONFIG_ARC_MISSING_ESCAPE);
		3134	link_width_cntl \|= lanes \| LC_RECONFIG_NOW \| LC_RENEGOTIATE_EN;
		3135	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3136	} else {
		3137	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3138	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3139	}
		3140	}
		3141
		3142	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3143	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
		3144	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		3145
		3146	/* 55 nm r6xx asics */
		3147	if ((rdev->family == CHIP_RV670) \|\|
		3148	(rdev->family == CHIP_RV620) \|\|
		3149	(rdev->family == CHIP_RV635)) {
		3150	WREG32(MM_CFGREGS_CNTL, 0x8);
		3151	link_cntl2 = RREG32(0x4088);
		3152	WREG32(MM_CFGREGS_CNTL, 0);
		3153	/* not supported yet */
		3154	if (link_cntl2 & SELECTABLE_DEEMPHASIS)
		3155	return;
		3156	}
		3157
		3158	speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
		3159	speed_cntl \|= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
		3160	speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
		3161	speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
		3162	speed_cntl \|= LC_FORCE_EN_HW_SPEED_CHANGE;
		3163	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3164
		3165	tmp = RREG32(0x541c);
		3166	WREG32(0x541c, tmp \| 0x8);
		3167	WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
		3168	link_cntl2 = RREG16(0x4088);
		3169	link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
		3170	link_cntl2 \|= 0x2;
		3171	WREG16(0x4088, link_cntl2);
		3172	WREG32(MM_CFGREGS_CNTL, 0);
		3173
		3174	if ((rdev->family == CHIP_RV670) \|\|
		3175	(rdev->family == CHIP_RV620) \|\|
		3176	(rdev->family == CHIP_RV635)) {
		3177	training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
		3178	training_cntl &= ~LC_POINT_7_PLUS_EN;
		3179	WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
		3180	} else {
		3181	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3182	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
		3183	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3184	}
		3185
		3186	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
		3187	speed_cntl \|= LC_GEN2_EN_STRAP;
		3188	WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
		3189
		3190	} else {
		3191	link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
		3192	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		3193	if (1)
		3194	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		3195	else
		3196	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
		3197	WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		3198	}
		3199	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/r600.c – Rev 2160