WebSVN – Kolibri OS – Blame – /drivers/video/drm/radeon/r600.c

Rev	Author	Line No.	Line
1128	serge	1	/*
		2	* Copyright 2008 Advanced Micro Devices, Inc.
		3	* Copyright 2008 Red Hat Inc.
		4	* Copyright 2009 Jerome Glisse.
		5	*
		6	* Permission is hereby granted, free of charge, to any person obtaining a
		7	* copy of this software and associated documentation files (the "Software"),
		8	* to deal in the Software without restriction, including without limitation
		9	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
		10	* and/or sell copies of the Software, and to permit persons to whom the
		11	* Software is furnished to do so, subject to the following conditions:
		12	*
		13	* The above copyright notice and this permission notice shall be included in
		14	* all copies or substantial portions of the Software.
		15	*
		16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
		19	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
		20	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
		21	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		22	* OTHER DEALINGS IN THE SOFTWARE.
		23	*
		24	* Authors: Dave Airlie
		25	* Alex Deucher
		26	* Jerome Glisse
		27	*/
1963	serge	28	#include
1233	serge	29	#include
1221	serge	30	#include
2997	Serge	31	#include
		32	#include
		33	#include
1128	serge	34	#include "radeon.h"
1963	serge	35	#include "radeon_asic.h"
1221	serge	36	#include "radeon_mode.h"
		37	#include "r600d.h"
		38	#include "atom.h"
		39	#include "avivod.h"
1128	serge	40
1221	serge	41	#define PFP_UCODE_SIZE 576
		42	#define PM4_UCODE_SIZE 1792
1321	serge	43	#define RLC_UCODE_SIZE 768
1221	serge	44	#define R700_PFP_UCODE_SIZE 848
		45	#define R700_PM4_UCODE_SIZE 1360
1321	serge	46	#define R700_RLC_UCODE_SIZE 1024
1963	serge	47	#define EVERGREEN_PFP_UCODE_SIZE 1120
		48	#define EVERGREEN_PM4_UCODE_SIZE 1376
		49	#define EVERGREEN_RLC_UCODE_SIZE 768
		50	#define CAYMAN_RLC_UCODE_SIZE 1024
2997	Serge	51	#define ARUBA_RLC_UCODE_SIZE 1536
1128	serge	52
1221	serge	53	/* Firmware Names */
		54	MODULE_FIRMWARE("radeon/R600_pfp.bin");
		55	MODULE_FIRMWARE("radeon/R600_me.bin");
		56	MODULE_FIRMWARE("radeon/RV610_pfp.bin");
		57	MODULE_FIRMWARE("radeon/RV610_me.bin");
		58	MODULE_FIRMWARE("radeon/RV630_pfp.bin");
		59	MODULE_FIRMWARE("radeon/RV630_me.bin");
		60	MODULE_FIRMWARE("radeon/RV620_pfp.bin");
		61	MODULE_FIRMWARE("radeon/RV620_me.bin");
		62	MODULE_FIRMWARE("radeon/RV635_pfp.bin");
		63	MODULE_FIRMWARE("radeon/RV635_me.bin");
		64	MODULE_FIRMWARE("radeon/RV670_pfp.bin");
		65	MODULE_FIRMWARE("radeon/RV670_me.bin");
		66	MODULE_FIRMWARE("radeon/RS780_pfp.bin");
		67	MODULE_FIRMWARE("radeon/RS780_me.bin");
		68	MODULE_FIRMWARE("radeon/RV770_pfp.bin");
		69	MODULE_FIRMWARE("radeon/RV770_me.bin");
		70	MODULE_FIRMWARE("radeon/RV730_pfp.bin");
		71	MODULE_FIRMWARE("radeon/RV730_me.bin");
		72	MODULE_FIRMWARE("radeon/RV710_pfp.bin");
		73	MODULE_FIRMWARE("radeon/RV710_me.bin");
1321	serge	74	MODULE_FIRMWARE("radeon/R600_rlc.bin");
		75	MODULE_FIRMWARE("radeon/R700_rlc.bin");
1963	serge	76	MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
		77	MODULE_FIRMWARE("radeon/CEDAR_me.bin");
		78	MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
		79	MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
		80	MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
		81	MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
		82	MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
		83	MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
		84	MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
		85	MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
		86	MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
		87	MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
		88	MODULE_FIRMWARE("radeon/PALM_pfp.bin");
		89	MODULE_FIRMWARE("radeon/PALM_me.bin");
		90	MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
		91	MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
		92	MODULE_FIRMWARE("radeon/SUMO_me.bin");
		93	MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
		94	MODULE_FIRMWARE("radeon/SUMO2_me.bin");
1221	serge	95
3764	Serge	96	static const u32 crtc_offsets[2] =
		97	{
		98	0,
		99	AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
		100	};
		101
1221	serge	102	int r600_debugfs_mc_info_init(struct radeon_device *rdev);
		103
		104	/* r600,rv610,rv630,rv620,rv635,rv670 */
1128	serge	105	int r600_mc_wait_for_idle(struct radeon_device *rdev);
2997	Serge	106	static void r600_gpu_init(struct radeon_device *rdev);
1221	serge	107	void r600_fini(struct radeon_device *rdev);
1963	serge	108	void r600_irq_disable(struct radeon_device *rdev);
		109	static void r600_pcie_gen2_enable(struct radeon_device *rdev);
1128	serge	110
3764	Serge	111	/**
		112	* r600_get_xclk - get the xclk
		113	*
		114	* @rdev: radeon_device pointer
		115	*
		116	* Returns the reference clock used by the gfx engine
		117	* (r6xx, IGPs, APUs).
		118	*/
		119	u32 r600_get_xclk(struct radeon_device *rdev)
		120	{
		121	return rdev->clock.spll.reference_freq;
		122	}
		123
1963	serge	124	/* get temperature in millidegrees */
		125	int rv6xx_get_temp(struct radeon_device *rdev)
		126	{
		127	u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
		128	ASIC_T_SHIFT;
		129	int actual_temp = temp & 0xff;
		130
		131	if (temp & 0x100)
		132	actual_temp -= 256;
		133
		134	return actual_temp * 1000;
		135	}
		136
		137
		138
		139
		140
		141
		142	bool r600_gui_idle(struct radeon_device *rdev)
		143	{
		144	if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
		145	return false;
		146	else
		147	return true;
		148	}
		149
1321	serge	150	/* hpd for digital panel detect/disconnect */
		151	bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
		152	{
		153	bool connected = false;
		154
		155	if (ASIC_IS_DCE3(rdev)) {
		156	switch (hpd) {
		157	case RADEON_HPD_1:
		158	if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
		159	connected = true;
		160	break;
		161	case RADEON_HPD_2:
		162	if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
		163	connected = true;
		164	break;
		165	case RADEON_HPD_3:
		166	if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
		167	connected = true;
		168	break;
		169	case RADEON_HPD_4:
		170	if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
		171	connected = true;
		172	break;
		173	/* DCE 3.2 */
		174	case RADEON_HPD_5:
		175	if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
		176	connected = true;
		177	break;
		178	case RADEON_HPD_6:
		179	if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
		180	connected = true;
		181	break;
		182	default:
		183	break;
		184	}
		185	} else {
		186	switch (hpd) {
		187	case RADEON_HPD_1:
		188	if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		189	connected = true;
		190	break;
		191	case RADEON_HPD_2:
		192	if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		193	connected = true;
		194	break;
		195	case RADEON_HPD_3:
		196	if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
		197	connected = true;
		198	break;
		199	default:
		200	break;
		201	}
		202	}
		203	return connected;
		204	}
		205
		206	void r600_hpd_set_polarity(struct radeon_device *rdev,
		207	enum radeon_hpd_id hpd)
		208	{
		209	u32 tmp;
		210	bool connected = r600_hpd_sense(rdev, hpd);
		211
		212	if (ASIC_IS_DCE3(rdev)) {
		213	switch (hpd) {
		214	case RADEON_HPD_1:
		215	tmp = RREG32(DC_HPD1_INT_CONTROL);
		216	if (connected)
		217	tmp &= ~DC_HPDx_INT_POLARITY;
		218	else
		219	tmp \|= DC_HPDx_INT_POLARITY;
		220	WREG32(DC_HPD1_INT_CONTROL, tmp);
		221	break;
		222	case RADEON_HPD_2:
		223	tmp = RREG32(DC_HPD2_INT_CONTROL);
		224	if (connected)
		225	tmp &= ~DC_HPDx_INT_POLARITY;
		226	else
		227	tmp \|= DC_HPDx_INT_POLARITY;
		228	WREG32(DC_HPD2_INT_CONTROL, tmp);
		229	break;
		230	case RADEON_HPD_3:
		231	tmp = RREG32(DC_HPD3_INT_CONTROL);
		232	if (connected)
		233	tmp &= ~DC_HPDx_INT_POLARITY;
		234	else
		235	tmp \|= DC_HPDx_INT_POLARITY;
		236	WREG32(DC_HPD3_INT_CONTROL, tmp);
		237	break;
		238	case RADEON_HPD_4:
		239	tmp = RREG32(DC_HPD4_INT_CONTROL);
		240	if (connected)
		241	tmp &= ~DC_HPDx_INT_POLARITY;
		242	else
		243	tmp \|= DC_HPDx_INT_POLARITY;
		244	WREG32(DC_HPD4_INT_CONTROL, tmp);
		245	break;
		246	case RADEON_HPD_5:
		247	tmp = RREG32(DC_HPD5_INT_CONTROL);
		248	if (connected)
		249	tmp &= ~DC_HPDx_INT_POLARITY;
		250	else
		251	tmp \|= DC_HPDx_INT_POLARITY;
		252	WREG32(DC_HPD5_INT_CONTROL, tmp);
		253	break;
		254	/* DCE 3.2 */
		255	case RADEON_HPD_6:
		256	tmp = RREG32(DC_HPD6_INT_CONTROL);
		257	if (connected)
		258	tmp &= ~DC_HPDx_INT_POLARITY;
		259	else
		260	tmp \|= DC_HPDx_INT_POLARITY;
		261	WREG32(DC_HPD6_INT_CONTROL, tmp);
		262	break;
		263	default:
		264	break;
		265	}
		266	} else {
		267	switch (hpd) {
		268	case RADEON_HPD_1:
		269	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		270	if (connected)
		271	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		272	else
		273	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		274	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		275	break;
		276	case RADEON_HPD_2:
		277	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		278	if (connected)
		279	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		280	else
		281	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		282	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		283	break;
		284	case RADEON_HPD_3:
		285	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		286	if (connected)
		287	tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
		288	else
		289	tmp \|= DC_HOT_PLUG_DETECTx_INT_POLARITY;
		290	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		291	break;
		292	default:
		293	break;
		294	}
		295	}
		296	}
		297
		298	void r600_hpd_init(struct radeon_device *rdev)
		299	{
		300	struct drm_device *dev = rdev->ddev;
		301	struct drm_connector *connector;
2997	Serge	302	unsigned enable = 0;
1321	serge	303
2997	Serge	304	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		305	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
		306
		307	if (connector->connector_type == DRM_MODE_CONNECTOR_eDP \|\|
		308	connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
		309	/* don't try to enable hpd on eDP or LVDS avoid breaking the
		310	* aux dp channel on imac and help (but not completely fix)
		311	* https://bugzilla.redhat.com/show_bug.cgi?id=726143
		312	*/
		313	continue;
		314	}
1321	serge	315	if (ASIC_IS_DCE3(rdev)) {
		316	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) \| DC_HPDx_RX_INT_TIMER(0xfa);
		317	if (ASIC_IS_DCE32(rdev))
		318	tmp \|= DC_HPDx_EN;
		319
		320	switch (radeon_connector->hpd.hpd) {
		321	case RADEON_HPD_1:
		322	WREG32(DC_HPD1_CONTROL, tmp);
		323	break;
		324	case RADEON_HPD_2:
		325	WREG32(DC_HPD2_CONTROL, tmp);
		326	break;
		327	case RADEON_HPD_3:
		328	WREG32(DC_HPD3_CONTROL, tmp);
		329	break;
		330	case RADEON_HPD_4:
		331	WREG32(DC_HPD4_CONTROL, tmp);
		332	break;
		333	/* DCE 3.2 */
		334	case RADEON_HPD_5:
		335	WREG32(DC_HPD5_CONTROL, tmp);
		336	break;
		337	case RADEON_HPD_6:
		338	WREG32(DC_HPD6_CONTROL, tmp);
		339	break;
		340	default:
		341	break;
		342	}
		343	} else {
		344	switch (radeon_connector->hpd.hpd) {
		345	case RADEON_HPD_1:
		346	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		347	break;
		348	case RADEON_HPD_2:
		349	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		350	break;
		351	case RADEON_HPD_3:
		352	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
		353	break;
		354	default:
		355	break;
		356	}
		357	}
2997	Serge	358	enable \|= 1 << radeon_connector->hpd.hpd;
		359	radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1321	serge	360	}
2997	Serge	361	// radeon_irq_kms_enable_hpd(rdev, enable);
1321	serge	362	}
		363
		364	void r600_hpd_fini(struct radeon_device *rdev)
		365	{
		366	struct drm_device *dev = rdev->ddev;
		367	struct drm_connector *connector;
2997	Serge	368	unsigned disable = 0;
1321	serge	369
		370	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
		371	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
2997	Serge	372	if (ASIC_IS_DCE3(rdev)) {
1321	serge	373	switch (radeon_connector->hpd.hpd) {
		374	case RADEON_HPD_1:
		375	WREG32(DC_HPD1_CONTROL, 0);
		376	break;
		377	case RADEON_HPD_2:
		378	WREG32(DC_HPD2_CONTROL, 0);
		379	break;
		380	case RADEON_HPD_3:
		381	WREG32(DC_HPD3_CONTROL, 0);
		382	break;
		383	case RADEON_HPD_4:
		384	WREG32(DC_HPD4_CONTROL, 0);
		385	break;
		386	/* DCE 3.2 */
		387	case RADEON_HPD_5:
		388	WREG32(DC_HPD5_CONTROL, 0);
		389	break;
		390	case RADEON_HPD_6:
		391	WREG32(DC_HPD6_CONTROL, 0);
		392	break;
		393	default:
		394	break;
		395	}
		396	} else {
		397	switch (radeon_connector->hpd.hpd) {
		398	case RADEON_HPD_1:
		399	WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
		400	break;
		401	case RADEON_HPD_2:
		402	WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
		403	break;
		404	case RADEON_HPD_3:
		405	WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
		406	break;
		407	default:
		408	break;
		409	}
		410	}
2997	Serge	411	disable \|= 1 << radeon_connector->hpd.hpd;
1321	serge	412	}
2997	Serge	413	// radeon_irq_kms_disable_hpd(rdev, disable);
1321	serge	414	}
		415
1128	serge	416	/*
1221	serge	417	* R600 PCIE GART
1128	serge	418	*/
1221	serge	419	void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
		420	{
		421	unsigned i;
		422	u32 tmp;
1128	serge	423
1430	serge	424	/* flush hdp cache so updates hit vram */
1963	serge	425	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		426	!(rdev->flags & RADEON_IS_AGP)) {
2997	Serge	427	void __iomem ptr = (void )rdev->gart.ptr;
1963	serge	428	u32 tmp;
		429
		430	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		431	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
		432	* This seems to cause problems on some AGP cards. Just use the old
		433	* method for them.
		434	*/
		435	WREG32(HDP_DEBUG1, 0);
		436	tmp = readl((void __iomem *)ptr);
		437	} else
1430	serge	438	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		439
1221	serge	440	WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
		441	WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
		442	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
		443	for (i = 0; i < rdev->usec_timeout; i++) {
		444	/* read MC_STATUS */
		445	tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
		446	tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
		447	if (tmp == 2) {
		448	printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
		449	return;
		450	}
		451	if (tmp) {
		452	return;
		453	}
		454	udelay(1);
1128	serge	455	}
1221	serge	456	}
1128	serge	457
1221	serge	458	int r600_pcie_gart_init(struct radeon_device *rdev)
		459	{
		460	int r;
		461
2997	Serge	462	if (rdev->gart.robj) {
1963	serge	463	WARN(1, "R600 PCIE GART already initialized\n");
1221	serge	464	return 0;
		465	}
		466	/* Initialize common gart structure */
		467	r = radeon_gart_init(rdev);
		468	if (r)
		469	return r;
		470	rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
		471	return radeon_gart_table_vram_alloc(rdev);
		472	}
		473
2997	Serge	474	static int r600_pcie_gart_enable(struct radeon_device *rdev)
1221	serge	475	{
		476	u32 tmp;
		477	int r, i;
		478
2997	Serge	479	if (rdev->gart.robj == NULL) {
1221	serge	480	dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		481	return -EINVAL;
		482	}
		483	r = radeon_gart_table_vram_pin(rdev);
		484	if (r)
		485	return r;
1430	serge	486	radeon_gart_restore(rdev);
1221	serge	487
		488	/* Setup L2 cache */
		489	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		490	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		491	EFFECTIVE_L2_QUEUE_SIZE(7));
		492	WREG32(VM_L2_CNTL2, 0);
		493	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		494	/* Setup TLB control */
		495	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		496	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		497	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		498	ENABLE_WAIT_L2_QUERY;
		499	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		500	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		501	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		502	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		503	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		504	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		505	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		506	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		507	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		508	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		509	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		510	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		511	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		512	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		513	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
		514	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
		515	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
		516	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT \| PAGE_TABLE_DEPTH(0) \|
		517	RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
		518	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
		519	(u32)(rdev->dummy_page.addr >> 12));
		520	for (i = 1; i < 7; i++)
		521	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		522
		523	r600_pcie_gart_tlb_flush(rdev);
2997	Serge	524	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
		525	(unsigned)(rdev->mc.gtt_size >> 20),
		526	(unsigned long long)rdev->gart.table_addr);
1221	serge	527	rdev->gart.ready = true;
1128	serge	528	return 0;
		529	}
		530
2997	Serge	531	static void r600_pcie_gart_disable(struct radeon_device *rdev)
1128	serge	532	{
1221	serge	533	u32 tmp;
2997	Serge	534	int i;
1221	serge	535
		536	/* Disable all tables */
		537	for (i = 0; i < 7; i++)
		538	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
		539
		540	/* Disable L2 cache */
		541	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING \|
		542	EFFECTIVE_L2_QUEUE_SIZE(7));
		543	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		544	/* Setup L1 TLB control */
		545	tmp = EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		546	ENABLE_WAIT_L2_QUERY;
		547	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		548	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		549	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		550	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		551	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		552	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		553	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		554	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		555	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
		556	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
		557	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		558	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		559	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
		560	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
2997	Serge	561	radeon_gart_table_vram_unpin(rdev);
1128	serge	562	}
		563
2997	Serge	564	static void r600_pcie_gart_fini(struct radeon_device *rdev)
1221	serge	565	{
1963	serge	566	radeon_gart_fini(rdev);
1221	serge	567	r600_pcie_gart_disable(rdev);
		568	radeon_gart_table_vram_free(rdev);
		569	}
1128	serge	570
2997	Serge	571	static void r600_agp_enable(struct radeon_device *rdev)
1128	serge	572	{
1221	serge	573	u32 tmp;
		574	int i;
		575
		576	/* Setup L2 cache */
		577	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE \| ENABLE_L2_FRAGMENT_PROCESSING \|
		578	ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE \|
		579	EFFECTIVE_L2_QUEUE_SIZE(7));
		580	WREG32(VM_L2_CNTL2, 0);
		581	WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) \| BANK_SELECT_1(1));
		582	/* Setup TLB control */
		583	tmp = ENABLE_L1_TLB \| ENABLE_L1_FRAGMENT_PROCESSING \|
		584	SYSTEM_ACCESS_MODE_NOT_IN_SYS \|
		585	EFFECTIVE_L1_TLB_SIZE(5) \| EFFECTIVE_L1_QUEUE_SIZE(5) \|
		586	ENABLE_WAIT_L2_QUERY;
		587	WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
		588	WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
		589	WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp \| ENABLE_L1_STRICT_ORDERING);
		590	WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
		591	WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
		592	WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
		593	WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
		594	WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
		595	WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
		596	WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
		597	WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
		598	WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
		599	WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		600	WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp \| ENABLE_SEMAPHORE_MODE);
		601	for (i = 0; i < 7; i++)
		602	WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1128	serge	603	}
		604
		605	int r600_mc_wait_for_idle(struct radeon_device *rdev)
		606	{
1221	serge	607	unsigned i;
		608	u32 tmp;
		609
		610	for (i = 0; i < rdev->usec_timeout; i++) {
		611	/* read MC_STATUS */
		612	tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
		613	if (!tmp)
1128	serge	614	return 0;
1221	serge	615	udelay(1);
		616	}
		617	return -1;
1128	serge	618	}
		619
3764	Serge	620	uint32_t rs780_mc_rreg(struct radeon_device *rdev, uint32_t reg)
		621	{
		622	uint32_t r;
		623
		624	WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg));
		625	r = RREG32(R_0028FC_MC_DATA);
		626	WREG32(R_0028F8_MC_INDEX, ~C_0028F8_MC_IND_ADDR);
		627	return r;
		628	}
		629
		630	void rs780_mc_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v)
		631	{
		632	WREG32(R_0028F8_MC_INDEX, S_0028F8_MC_IND_ADDR(reg) \|
		633	S_0028F8_MC_IND_WR_EN(1));
		634	WREG32(R_0028FC_MC_DATA, v);
		635	WREG32(R_0028F8_MC_INDEX, 0x7F);
		636	}
		637
1221	serge	638	static void r600_mc_program(struct radeon_device *rdev)
1128	serge	639	{
1221	serge	640	struct rv515_mc_save save;
		641	u32 tmp;
		642	int i, j;
		643
		644	/* Initialize HDP */
		645	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		646	WREG32((0x2c14 + j), 0x00000000);
		647	WREG32((0x2c18 + j), 0x00000000);
		648	WREG32((0x2c1c + j), 0x00000000);
		649	WREG32((0x2c20 + j), 0x00000000);
		650	WREG32((0x2c24 + j), 0x00000000);
		651	}
		652	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
		653
		654	rv515_mc_stop(rdev, &save);
		655	if (r600_mc_wait_for_idle(rdev)) {
		656	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		657	}
		658	/* Lockout access through VGA aperture (doesn't exist before R600) */
		659	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
		660	/* Update configuration */
		661	if (rdev->flags & RADEON_IS_AGP) {
		662	if (rdev->mc.vram_start < rdev->mc.gtt_start) {
		663	/* VRAM before AGP */
		664	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		665	rdev->mc.vram_start >> 12);
		666	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		667	rdev->mc.gtt_end >> 12);
		668	} else {
		669	/* VRAM after AGP */
		670	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
		671	rdev->mc.gtt_start >> 12);
		672	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
		673	rdev->mc.vram_end >> 12);
		674	}
		675	} else {
		676	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
		677	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
		678	}
2997	Serge	679	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1221	serge	680	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
		681	tmp \|= ((rdev->mc.vram_start >> 24) & 0xFFFF);
		682	WREG32(MC_VM_FB_LOCATION, tmp);
		683	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
		684	WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1963	serge	685	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1221	serge	686	if (rdev->flags & RADEON_IS_AGP) {
		687	WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
		688	WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
		689	WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
		690	} else {
		691	WREG32(MC_VM_AGP_BASE, 0);
		692	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
		693	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
		694	}
		695	if (r600_mc_wait_for_idle(rdev)) {
		696	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		697	}
		698	rv515_mc_resume(rdev, &save);
		699	/* we need to own VRAM, so turn off the VGA renderer here
		700	* to stop it overwriting our objects */
		701	rv515_vga_render_disable(rdev);
1128	serge	702	}
		703
1430	serge	704	/**
		705	* r600_vram_gtt_location - try to find VRAM & GTT location
		706	* @rdev: radeon device structure holding all necessary informations
		707	* @mc: memory controller structure holding memory informations
		708	*
		709	* Function will place try to place VRAM at same place as in CPU (PCI)
		710	* address space as some GPU seems to have issue when we reprogram at
		711	* different address space.
		712	*
		713	* If there is not enough space to fit the unvisible VRAM after the
		714	* aperture then we limit the VRAM size to the aperture.
		715	*
		716	* If we are using AGP then place VRAM adjacent to AGP aperture are we need
		717	* them to be in one from GPU point of view so that we can program GPU to
		718	* catch access outside them (weird GPU policy see ??).
		719	*
		720	* This function will never fails, worst case are limiting VRAM or GTT.
		721	*
		722	* Note: GTT start, end, size should be initialized before calling this
		723	* function on AGP platform.
		724	*/
1963	serge	725	static void r600_vram_gtt_location(struct radeon_device rdev, struct radeon_mc mc)
1430	serge	726	{
		727	u64 size_bf, size_af;
		728
		729	if (mc->mc_vram_size > 0xE0000000) {
		730	/* leave room for at least 512M GTT */
		731	dev_warn(rdev->dev, "limiting VRAM\n");
		732	mc->real_vram_size = 0xE0000000;
		733	mc->mc_vram_size = 0xE0000000;
		734	}
		735	if (rdev->flags & RADEON_IS_AGP) {
		736	size_bf = mc->gtt_start;
3764	Serge	737	size_af = mc->mc_mask - mc->gtt_end;
1430	serge	738	if (size_bf > size_af) {
		739	if (mc->mc_vram_size > size_bf) {
		740	dev_warn(rdev->dev, "limiting VRAM\n");
		741	mc->real_vram_size = size_bf;
		742	mc->mc_vram_size = size_bf;
		743	}
		744	mc->vram_start = mc->gtt_start - mc->mc_vram_size;
		745	} else {
		746	if (mc->mc_vram_size > size_af) {
		747	dev_warn(rdev->dev, "limiting VRAM\n");
		748	mc->real_vram_size = size_af;
		749	mc->mc_vram_size = size_af;
		750	}
2997	Serge	751	mc->vram_start = mc->gtt_end + 1;
1430	serge	752	}
		753	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
		754	dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
		755	mc->mc_vram_size >> 20, mc->vram_start,
		756	mc->vram_end, mc->real_vram_size >> 20);
		757	} else {
		758	u64 base = 0;
1963	serge	759	if (rdev->flags & RADEON_IS_IGP) {
		760	base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
		761	base <<= 24;
		762	}
1430	serge	763	radeon_vram_location(rdev, &rdev->mc, base);
1963	serge	764	rdev->mc.gtt_base_align = 0;
1430	serge	765	radeon_gtt_location(rdev, mc);
		766	}
		767	}
		768
2997	Serge	769	static int r600_mc_init(struct radeon_device *rdev)
1128	serge	770	{
1221	serge	771	u32 tmp;
1268	serge	772	int chansize, numchan;
3764	Serge	773	uint32_t h_addr, l_addr;
		774	unsigned long long k8_addr;
1128	serge	775
1221	serge	776	/* Get VRAM informations */
1128	serge	777	rdev->mc.vram_is_ddr = true;
1221	serge	778	tmp = RREG32(RAMCFG);
		779	if (tmp & CHANSIZE_OVERRIDE) {
1128	serge	780	chansize = 16;
1221	serge	781	} else if (tmp & CHANSIZE_MASK) {
1128	serge	782	chansize = 64;
		783	} else {
		784	chansize = 32;
		785	}
1268	serge	786	tmp = RREG32(CHMAP);
		787	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
		788	case 0:
		789	default:
		790	numchan = 1;
		791	break;
		792	case 1:
		793	numchan = 2;
		794	break;
		795	case 2:
		796	numchan = 4;
		797	break;
		798	case 3:
		799	numchan = 8;
		800	break;
1128	serge	801	}
1268	serge	802	rdev->mc.vram_width = numchan * chansize;
1221	serge	803	/* Could aper size report 0 ? */
1963	serge	804	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
		805	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1221	serge	806	/* Setup GPU memory space */
		807	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
		808	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1430	serge	809	rdev->mc.visible_vram_size = rdev->mc.aper_size;
		810	r600_vram_gtt_location(rdev, &rdev->mc);
1963	serge	811
		812	if (rdev->flags & RADEON_IS_IGP) {
		813	rs690_pm_info(rdev);
1403	serge	814	rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
3764	Serge	815
		816	if (rdev->family == CHIP_RS780 \|\| rdev->family == CHIP_RS880) {
		817	/* Use K8 direct mapping for fast fb access. */
		818	rdev->fastfb_working = false;
		819	h_addr = G_000012_K8_ADDR_EXT(RREG32_MC(R_000012_MC_MISC_UMA_CNTL));
		820	l_addr = RREG32_MC(R_000011_K8_FB_LOCATION);
		821	k8_addr = ((unsigned long long)h_addr) << 32 \| l_addr;
		822	#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
		823	if (k8_addr + rdev->mc.visible_vram_size < 0x100000000ULL)
		824	#endif
		825	{
		826	/* FastFB shall be used with UMA memory. Here it is simply disabled when sideport
		827	* memory is present.
		828	*/
		829	if (rdev->mc.igp_sideport_enabled == false && radeon_fastfb == 1) {
		830	DRM_INFO("Direct mapping: aper base at 0x%llx, replaced by direct mapping base 0x%llx.\n",
		831	(unsigned long long)rdev->mc.aper_base, k8_addr);
		832	rdev->mc.aper_base = (resource_size_t)k8_addr;
		833	rdev->fastfb_working = true;
1963	serge	834	}
3764	Serge	835	}
		836	}
		837	}
		838
1963	serge	839	radeon_update_bandwidth_info(rdev);
1221	serge	840	return 0;
1128	serge	841	}
		842
2997	Serge	843	int r600_vram_scratch_init(struct radeon_device *rdev)
		844	{
		845	int r;
		846
		847	if (rdev->vram_scratch.robj == NULL) {
		848	r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
		849	PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
		850	NULL, &rdev->vram_scratch.robj);
		851	if (r) {
		852	return r;
		853	}
		854	}
		855
		856	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
		857	if (unlikely(r != 0))
		858	return r;
		859	r = radeon_bo_pin(rdev->vram_scratch.robj,
		860	RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
		861	if (r) {
		862	radeon_bo_unreserve(rdev->vram_scratch.robj);
		863	return r;
		864	}
		865	r = radeon_bo_kmap(rdev->vram_scratch.robj,
		866	(void **)&rdev->vram_scratch.ptr);
		867	if (r)
		868	radeon_bo_unpin(rdev->vram_scratch.robj);
		869	radeon_bo_unreserve(rdev->vram_scratch.robj);
		870
		871	return r;
		872	}
3764	Serge	873
		874	void r600_vram_scratch_fini(struct radeon_device *rdev)
1128	serge	875	{
3764	Serge	876	int r;
1128	serge	877
3764	Serge	878	if (rdev->vram_scratch.robj == NULL) {
3192	Serge	879	return;
3764	Serge	880	}
		881	r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
		882	if (likely(r == 0)) {
		883	radeon_bo_kunmap(rdev->vram_scratch.robj);
		884	radeon_bo_unpin(rdev->vram_scratch.robj);
		885	radeon_bo_unreserve(rdev->vram_scratch.robj);
		886	}
		887	radeon_bo_unref(&rdev->vram_scratch.robj);
		888	}
1963	serge	889
3764	Serge	890	void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
		891	{
		892	u32 tmp = RREG32(R600_BIOS_3_SCRATCH);
		893
		894	if (hung)
		895	tmp \|= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
		896	else
		897	tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
		898
		899	WREG32(R600_BIOS_3_SCRATCH, tmp);
		900	}
		901
		902	static void r600_print_gpu_status_regs(struct radeon_device *rdev)
		903	{
3192	Serge	904	dev_info(rdev->dev, " R_008010_GRBM_STATUS = 0x%08X\n",
1221	serge	905	RREG32(R_008010_GRBM_STATUS));
3192	Serge	906	dev_info(rdev->dev, " R_008014_GRBM_STATUS2 = 0x%08X\n",
1221	serge	907	RREG32(R_008014_GRBM_STATUS2));
3192	Serge	908	dev_info(rdev->dev, " R_000E50_SRBM_STATUS = 0x%08X\n",
1221	serge	909	RREG32(R_000E50_SRBM_STATUS));
2997	Serge	910	dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
		911	RREG32(CP_STALLED_STAT1));
		912	dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
		913	RREG32(CP_STALLED_STAT2));
		914	dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
		915	RREG32(CP_BUSY_STAT));
		916	dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
		917	RREG32(CP_STAT));
3764	Serge	918	dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
		919	RREG32(DMA_STATUS_REG));
		920	}
3192	Serge	921
3764	Serge	922	static bool r600_is_display_hung(struct radeon_device *rdev)
		923	{
		924	u32 crtc_hung = 0;
		925	u32 crtc_status[2];
		926	u32 i, j, tmp;
		927
		928	for (i = 0; i < rdev->num_crtc; i++) {
		929	if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) {
		930	crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
		931	crtc_hung \|= (1 << i);
		932	}
		933	}
		934
		935	for (j = 0; j < 10; j++) {
		936	for (i = 0; i < rdev->num_crtc; i++) {
		937	if (crtc_hung & (1 << i)) {
		938	tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
		939	if (tmp != crtc_status[i])
		940	crtc_hung &= ~(1 << i);
		941	}
		942	}
		943	if (crtc_hung == 0)
		944	return false;
		945	udelay(100);
		946	}
		947
		948	return true;
		949	}
		950
		951	static u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
		952	{
		953	u32 reset_mask = 0;
		954	u32 tmp;
		955
		956	/* GRBM_STATUS */
		957	tmp = RREG32(R_008010_GRBM_STATUS);
		958	if (rdev->family >= CHIP_RV770) {
		959	if (G_008010_PA_BUSY(tmp) \| G_008010_SC_BUSY(tmp) \|
		960	G_008010_SH_BUSY(tmp) \| G_008010_SX_BUSY(tmp) \|
		961	G_008010_TA_BUSY(tmp) \| G_008010_VGT_BUSY(tmp) \|
		962	G_008010_DB03_BUSY(tmp) \| G_008010_CB03_BUSY(tmp) \|
		963	G_008010_SPI03_BUSY(tmp) \| G_008010_VGT_BUSY_NO_DMA(tmp))
		964	reset_mask \|= RADEON_RESET_GFX;
		965	} else {
		966	if (G_008010_PA_BUSY(tmp) \| G_008010_SC_BUSY(tmp) \|
		967	G_008010_SH_BUSY(tmp) \| G_008010_SX_BUSY(tmp) \|
		968	G_008010_TA03_BUSY(tmp) \| G_008010_VGT_BUSY(tmp) \|
		969	G_008010_DB03_BUSY(tmp) \| G_008010_CB03_BUSY(tmp) \|
		970	G_008010_SPI03_BUSY(tmp) \| G_008010_VGT_BUSY_NO_DMA(tmp))
		971	reset_mask \|= RADEON_RESET_GFX;
		972	}
		973
		974	if (G_008010_CF_RQ_PENDING(tmp) \| G_008010_PF_RQ_PENDING(tmp) \|
		975	G_008010_CP_BUSY(tmp) \| G_008010_CP_COHERENCY_BUSY(tmp))
		976	reset_mask \|= RADEON_RESET_CP;
		977
		978	if (G_008010_GRBM_EE_BUSY(tmp))
		979	reset_mask \|= RADEON_RESET_GRBM \| RADEON_RESET_GFX \| RADEON_RESET_CP;
		980
		981	/* DMA_STATUS_REG */
		982	tmp = RREG32(DMA_STATUS_REG);
		983	if (!(tmp & DMA_IDLE))
		984	reset_mask \|= RADEON_RESET_DMA;
		985
		986	/* SRBM_STATUS */
		987	tmp = RREG32(R_000E50_SRBM_STATUS);
		988	if (G_000E50_RLC_RQ_PENDING(tmp) \| G_000E50_RLC_BUSY(tmp))
		989	reset_mask \|= RADEON_RESET_RLC;
		990
		991	if (G_000E50_IH_BUSY(tmp))
		992	reset_mask \|= RADEON_RESET_IH;
		993
		994	if (G_000E50_SEM_BUSY(tmp))
		995	reset_mask \|= RADEON_RESET_SEM;
		996
		997	if (G_000E50_GRBM_RQ_PENDING(tmp))
		998	reset_mask \|= RADEON_RESET_GRBM;
		999
		1000	if (G_000E50_VMC_BUSY(tmp))
		1001	reset_mask \|= RADEON_RESET_VMC;
		1002
		1003	if (G_000E50_MCB_BUSY(tmp) \| G_000E50_MCDZ_BUSY(tmp) \|
		1004	G_000E50_MCDY_BUSY(tmp) \| G_000E50_MCDX_BUSY(tmp) \|
		1005	G_000E50_MCDW_BUSY(tmp))
		1006	reset_mask \|= RADEON_RESET_MC;
		1007
		1008	if (r600_is_display_hung(rdev))
		1009	reset_mask \|= RADEON_RESET_DISPLAY;
		1010
		1011	/* Skip MC reset as it's mostly likely not hung, just busy */
		1012	if (reset_mask & RADEON_RESET_MC) {
		1013	DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
		1014	reset_mask &= ~RADEON_RESET_MC;
		1015	}
		1016
		1017	return reset_mask;
		1018	}
		1019
		1020	static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
		1021	{
		1022	struct rv515_mc_save save;
		1023	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
		1024	u32 tmp;
		1025
		1026	if (reset_mask == 0)
		1027	return;
		1028
		1029	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
		1030
		1031	r600_print_gpu_status_regs(rdev);
		1032
1221	serge	1033	/* Disable CP parsing/prefetching */
3764	Serge	1034	if (rdev->family >= CHIP_RV770)
		1035	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) \| S_0086D8_CP_PFP_HALT(1));
		1036	else
1963	serge	1037	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
3192	Serge	1038
3764	Serge	1039	/* disable the RLC */
		1040	WREG32(RLC_CNTL, 0);
		1041
		1042	if (reset_mask & RADEON_RESET_DMA) {
		1043	/* Disable DMA */
		1044	tmp = RREG32(DMA_RB_CNTL);
		1045	tmp &= ~DMA_RB_ENABLE;
		1046	WREG32(DMA_RB_CNTL, tmp);
		1047	}
		1048
		1049	mdelay(50);
		1050
		1051	rv515_mc_stop(rdev, &save);
		1052	if (r600_mc_wait_for_idle(rdev)) {
		1053	dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
		1054	}
		1055
		1056	if (reset_mask & (RADEON_RESET_GFX \| RADEON_RESET_COMPUTE)) {
		1057	if (rdev->family >= CHIP_RV770)
		1058	grbm_soft_reset \|= S_008020_SOFT_RESET_DB(1) \|
		1059	S_008020_SOFT_RESET_CB(1) \|
		1060	S_008020_SOFT_RESET_PA(1) \|
		1061	S_008020_SOFT_RESET_SC(1) \|
		1062	S_008020_SOFT_RESET_SPI(1) \|
		1063	S_008020_SOFT_RESET_SX(1) \|
		1064	S_008020_SOFT_RESET_SH(1) \|
		1065	S_008020_SOFT_RESET_TC(1) \|
		1066	S_008020_SOFT_RESET_TA(1) \|
		1067	S_008020_SOFT_RESET_VC(1) \|
		1068	S_008020_SOFT_RESET_VGT(1);
		1069	else
		1070	grbm_soft_reset \|= S_008020_SOFT_RESET_CR(1) \|
1221	serge	1071	S_008020_SOFT_RESET_DB(1) \|
		1072	S_008020_SOFT_RESET_CB(1) \|
		1073	S_008020_SOFT_RESET_PA(1) \|
		1074	S_008020_SOFT_RESET_SC(1) \|
		1075	S_008020_SOFT_RESET_SMX(1) \|
		1076	S_008020_SOFT_RESET_SPI(1) \|
		1077	S_008020_SOFT_RESET_SX(1) \|
		1078	S_008020_SOFT_RESET_SH(1) \|
		1079	S_008020_SOFT_RESET_TC(1) \|
		1080	S_008020_SOFT_RESET_TA(1) \|
		1081	S_008020_SOFT_RESET_VC(1) \|
		1082	S_008020_SOFT_RESET_VGT(1);
		1083	}
3764	Serge	1084
		1085	if (reset_mask & RADEON_RESET_CP) {
		1086	grbm_soft_reset \|= S_008020_SOFT_RESET_CP(1) \|
		1087	S_008020_SOFT_RESET_VGT(1);
		1088
		1089	srbm_soft_reset \|= S_000E60_SOFT_RESET_GRBM(1);
		1090	}
		1091
		1092	if (reset_mask & RADEON_RESET_DMA) {
		1093	if (rdev->family >= CHIP_RV770)
		1094	srbm_soft_reset \|= RV770_SOFT_RESET_DMA;
		1095	else
		1096	srbm_soft_reset \|= SOFT_RESET_DMA;
		1097	}
		1098
		1099	if (reset_mask & RADEON_RESET_RLC)
		1100	srbm_soft_reset \|= S_000E60_SOFT_RESET_RLC(1);
		1101
		1102	if (reset_mask & RADEON_RESET_SEM)
		1103	srbm_soft_reset \|= S_000E60_SOFT_RESET_SEM(1);
		1104
		1105	if (reset_mask & RADEON_RESET_IH)
		1106	srbm_soft_reset \|= S_000E60_SOFT_RESET_IH(1);
		1107
		1108	if (reset_mask & RADEON_RESET_GRBM)
		1109	srbm_soft_reset \|= S_000E60_SOFT_RESET_GRBM(1);
		1110
		1111	if (!(rdev->flags & RADEON_IS_IGP)) {
		1112	if (reset_mask & RADEON_RESET_MC)
		1113	srbm_soft_reset \|= S_000E60_SOFT_RESET_MC(1);
		1114	}
		1115
		1116	if (reset_mask & RADEON_RESET_VMC)
		1117	srbm_soft_reset \|= S_000E60_SOFT_RESET_VMC(1);
		1118
		1119	if (grbm_soft_reset) {
		1120	tmp = RREG32(R_008020_GRBM_SOFT_RESET);
		1121	tmp \|= grbm_soft_reset;
1221	serge	1122	dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
		1123	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
3764	Serge	1124	tmp = RREG32(R_008020_GRBM_SOFT_RESET);
3192	Serge	1125
3764	Serge	1126	udelay(50);
3192	Serge	1127
3764	Serge	1128	tmp &= ~grbm_soft_reset;
		1129	WREG32(R_008020_GRBM_SOFT_RESET, tmp);
		1130	tmp = RREG32(R_008020_GRBM_SOFT_RESET);
		1131	}
3192	Serge	1132
3764	Serge	1133	if (srbm_soft_reset) {
		1134	tmp = RREG32(SRBM_SOFT_RESET);
		1135	tmp \|= srbm_soft_reset;
		1136	dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
		1137	WREG32(SRBM_SOFT_RESET, tmp);
		1138	tmp = RREG32(SRBM_SOFT_RESET);
3192	Serge	1139
3764	Serge	1140	udelay(50);
3192	Serge	1141
3764	Serge	1142	tmp &= ~srbm_soft_reset;
		1143	WREG32(SRBM_SOFT_RESET, tmp);
		1144	tmp = RREG32(SRBM_SOFT_RESET);
		1145	}
3192	Serge	1146
3764	Serge	1147	/* Wait a little for things to settle down */
		1148	mdelay(1);
3192	Serge	1149
3764	Serge	1150	rv515_mc_resume(rdev, &save);
3192	Serge	1151	udelay(50);
		1152
3764	Serge	1153	r600_print_gpu_status_regs(rdev);
3192	Serge	1154	}
		1155
3764	Serge	1156	int r600_asic_reset(struct radeon_device *rdev)
3192	Serge	1157	{
3764	Serge	1158	u32 reset_mask;
3192	Serge	1159
3764	Serge	1160	reset_mask = r600_gpu_check_soft_reset(rdev);
3192	Serge	1161
3764	Serge	1162	if (reset_mask)
		1163	r600_set_bios_scratch_engine_hung(rdev, true);
3192	Serge	1164
3764	Serge	1165	r600_gpu_soft_reset(rdev, reset_mask);
3192	Serge	1166
3764	Serge	1167	reset_mask = r600_gpu_check_soft_reset(rdev);
3192	Serge	1168
3764	Serge	1169	if (!reset_mask)
		1170	r600_set_bios_scratch_engine_hung(rdev, false);
3192	Serge	1171
1221	serge	1172	return 0;
1128	serge	1173	}
		1174
3764	Serge	1175	/**
		1176	* r600_gfx_is_lockup - Check if the GFX engine is locked up
		1177	*
		1178	* @rdev: radeon_device pointer
		1179	* @ring: radeon_ring structure holding ring information
		1180	*
		1181	* Check if the GFX engine is locked up.
		1182	* Returns true if the engine appears to be locked up, false if not.
		1183	*/
		1184	bool r600_gfx_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
1221	serge	1185	{
3764	Serge	1186	u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1963	serge	1187
3764	Serge	1188	if (!(reset_mask & (RADEON_RESET_GFX \|
		1189	RADEON_RESET_COMPUTE \|
		1190	RADEON_RESET_CP))) {
2997	Serge	1191	radeon_ring_lockup_update(ring);
1963	serge	1192	return false;
		1193	}
		1194	/* force CP activities */
2997	Serge	1195	radeon_ring_force_activity(rdev, ring);
		1196	return radeon_ring_test_lockup(rdev, ring);
1963	serge	1197	}
		1198
3192	Serge	1199	/**
		1200	* r600_dma_is_lockup - Check if the DMA engine is locked up
		1201	*
		1202	* @rdev: radeon_device pointer
		1203	* @ring: radeon_ring structure holding ring information
		1204	*
3764	Serge	1205	* Check if the async DMA engine is locked up.
3192	Serge	1206	* Returns true if the engine appears to be locked up, false if not.
		1207	*/
		1208	bool r600_dma_is_lockup(struct radeon_device rdev, struct radeon_ring ring)
		1209	{
3764	Serge	1210	u32 reset_mask = r600_gpu_check_soft_reset(rdev);
3192	Serge	1211
3764	Serge	1212	if (!(reset_mask & RADEON_RESET_DMA)) {
3192	Serge	1213	radeon_ring_lockup_update(ring);
		1214	return false;
		1215	}
		1216	/* force ring activities */
		1217	radeon_ring_force_activity(rdev, ring);
		1218	return radeon_ring_test_lockup(rdev, ring);
		1219	}
		1220
2997	Serge	1221	u32 r6xx_remap_render_backend(struct radeon_device *rdev,
		1222	u32 tiling_pipe_num,
		1223	u32 max_rb_num,
		1224	u32 total_max_rb_num,
		1225	u32 disabled_rb_mask)
1221	serge	1226	{
2997	Serge	1227	u32 rendering_pipe_num, rb_num_width, req_rb_num;
3764	Serge	1228	u32 pipe_rb_ratio, pipe_rb_remain, tmp;
2997	Serge	1229	u32 data = 0, mask = 1 << (max_rb_num - 1);
		1230	unsigned i, j;
1221	serge	1231
2997	Serge	1232	/* mask out the RBs that don't exist on that asic */
3764	Serge	1233	tmp = disabled_rb_mask \| ((0xff << max_rb_num) & 0xff);
		1234	/* make sure at least one RB is available */
		1235	if ((tmp & 0xff) != 0xff)
		1236	disabled_rb_mask = tmp;
1221	serge	1237
2997	Serge	1238	rendering_pipe_num = 1 << tiling_pipe_num;
		1239	req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
		1240	BUG_ON(rendering_pipe_num < req_rb_num);
1221	serge	1241
2997	Serge	1242	pipe_rb_ratio = rendering_pipe_num / req_rb_num;
		1243	pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1221	serge	1244
2997	Serge	1245	if (rdev->family <= CHIP_RV740) {
		1246	/* r6xx/r7xx */
		1247	rb_num_width = 2;
		1248	} else {
		1249	/* eg+ */
		1250	rb_num_width = 4;
		1251	}
1221	serge	1252
2997	Serge	1253	for (i = 0; i < max_rb_num; i++) {
		1254	if (!(mask & disabled_rb_mask)) {
		1255	for (j = 0; j < pipe_rb_ratio; j++) {
		1256	data <<= rb_num_width;
		1257	data \|= max_rb_num - i - 1;
1221	serge	1258	}
2997	Serge	1259	if (pipe_rb_remain) {
		1260	data <<= rb_num_width;
		1261	data \|= max_rb_num - i - 1;
		1262	pipe_rb_remain--;
1221	serge	1263	}
2997	Serge	1264	}
		1265	mask >>= 1;
		1266	}
1221	serge	1267
2997	Serge	1268	return data;
1221	serge	1269	}
		1270
		1271	int r600_count_pipe_bits(uint32_t val)
		1272	{
3192	Serge	1273	return hweight32(val);
1221	serge	1274	}
		1275
2997	Serge	1276	static void r600_gpu_init(struct radeon_device *rdev)
1221	serge	1277	{
		1278	u32 tiling_config;
		1279	u32 ramcfg;
1430	serge	1280	u32 cc_rb_backend_disable;
		1281	u32 cc_gc_shader_pipe_config;
1221	serge	1282	u32 tmp;
		1283	int i, j;
		1284	u32 sq_config;
		1285	u32 sq_gpr_resource_mgmt_1 = 0;
		1286	u32 sq_gpr_resource_mgmt_2 = 0;
		1287	u32 sq_thread_resource_mgmt = 0;
		1288	u32 sq_stack_resource_mgmt_1 = 0;
		1289	u32 sq_stack_resource_mgmt_2 = 0;
2997	Serge	1290	u32 disabled_rb_mask;
1221	serge	1291
2997	Serge	1292	rdev->config.r600.tiling_group_size = 256;
1221	serge	1293	switch (rdev->family) {
		1294	case CHIP_R600:
		1295	rdev->config.r600.max_pipes = 4;
		1296	rdev->config.r600.max_tile_pipes = 8;
		1297	rdev->config.r600.max_simds = 4;
		1298	rdev->config.r600.max_backends = 4;
		1299	rdev->config.r600.max_gprs = 256;
		1300	rdev->config.r600.max_threads = 192;
		1301	rdev->config.r600.max_stack_entries = 256;
		1302	rdev->config.r600.max_hw_contexts = 8;
		1303	rdev->config.r600.max_gs_threads = 16;
		1304	rdev->config.r600.sx_max_export_size = 128;
		1305	rdev->config.r600.sx_max_export_pos_size = 16;
		1306	rdev->config.r600.sx_max_export_smx_size = 128;
		1307	rdev->config.r600.sq_num_cf_insts = 2;
		1308	break;
		1309	case CHIP_RV630:
		1310	case CHIP_RV635:
		1311	rdev->config.r600.max_pipes = 2;
		1312	rdev->config.r600.max_tile_pipes = 2;
		1313	rdev->config.r600.max_simds = 3;
		1314	rdev->config.r600.max_backends = 1;
		1315	rdev->config.r600.max_gprs = 128;
		1316	rdev->config.r600.max_threads = 192;
		1317	rdev->config.r600.max_stack_entries = 128;
		1318	rdev->config.r600.max_hw_contexts = 8;
		1319	rdev->config.r600.max_gs_threads = 4;
		1320	rdev->config.r600.sx_max_export_size = 128;
		1321	rdev->config.r600.sx_max_export_pos_size = 16;
		1322	rdev->config.r600.sx_max_export_smx_size = 128;
		1323	rdev->config.r600.sq_num_cf_insts = 2;
		1324	break;
		1325	case CHIP_RV610:
		1326	case CHIP_RV620:
		1327	case CHIP_RS780:
		1328	case CHIP_RS880:
		1329	rdev->config.r600.max_pipes = 1;
		1330	rdev->config.r600.max_tile_pipes = 1;
		1331	rdev->config.r600.max_simds = 2;
		1332	rdev->config.r600.max_backends = 1;
		1333	rdev->config.r600.max_gprs = 128;
		1334	rdev->config.r600.max_threads = 192;
		1335	rdev->config.r600.max_stack_entries = 128;
		1336	rdev->config.r600.max_hw_contexts = 4;
		1337	rdev->config.r600.max_gs_threads = 4;
		1338	rdev->config.r600.sx_max_export_size = 128;
		1339	rdev->config.r600.sx_max_export_pos_size = 16;
		1340	rdev->config.r600.sx_max_export_smx_size = 128;
		1341	rdev->config.r600.sq_num_cf_insts = 1;
		1342	break;
		1343	case CHIP_RV670:
		1344	rdev->config.r600.max_pipes = 4;
		1345	rdev->config.r600.max_tile_pipes = 4;
		1346	rdev->config.r600.max_simds = 4;
		1347	rdev->config.r600.max_backends = 4;
		1348	rdev->config.r600.max_gprs = 192;
		1349	rdev->config.r600.max_threads = 192;
		1350	rdev->config.r600.max_stack_entries = 256;
		1351	rdev->config.r600.max_hw_contexts = 8;
		1352	rdev->config.r600.max_gs_threads = 16;
		1353	rdev->config.r600.sx_max_export_size = 128;
		1354	rdev->config.r600.sx_max_export_pos_size = 16;
		1355	rdev->config.r600.sx_max_export_smx_size = 128;
		1356	rdev->config.r600.sq_num_cf_insts = 2;
		1357	break;
		1358	default:
		1359	break;
		1360	}
		1361
		1362	/* Initialize HDP */
		1363	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
		1364	WREG32((0x2c14 + j), 0x00000000);
		1365	WREG32((0x2c18 + j), 0x00000000);
		1366	WREG32((0x2c1c + j), 0x00000000);
		1367	WREG32((0x2c20 + j), 0x00000000);
		1368	WREG32((0x2c24 + j), 0x00000000);
		1369	}
		1370
		1371	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
		1372
		1373	/* Setup tiling */
		1374	tiling_config = 0;
		1375	ramcfg = RREG32(RAMCFG);
		1376	switch (rdev->config.r600.max_tile_pipes) {
		1377	case 1:
		1378	tiling_config \|= PIPE_TILING(0);
		1379	break;
		1380	case 2:
		1381	tiling_config \|= PIPE_TILING(1);
		1382	break;
		1383	case 4:
		1384	tiling_config \|= PIPE_TILING(2);
		1385	break;
		1386	case 8:
		1387	tiling_config \|= PIPE_TILING(3);
		1388	break;
		1389	default:
		1390	break;
		1391	}
1430	serge	1392	rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
		1393	rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1221	serge	1394	tiling_config \|= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1963	serge	1395	tiling_config \|= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
2997	Serge	1396
1221	serge	1397	tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
		1398	if (tmp > 3) {
		1399	tiling_config \|= ROW_TILING(3);
		1400	tiling_config \|= SAMPLE_SPLIT(3);
		1401	} else {
		1402	tiling_config \|= ROW_TILING(tmp);
		1403	tiling_config \|= SAMPLE_SPLIT(tmp);
		1404	}
		1405	tiling_config \|= BANK_SWAPS(1);
1430	serge	1406
		1407	cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
2997	Serge	1408	tmp = R6XX_MAX_BACKENDS -
		1409	r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
		1410	if (tmp < rdev->config.r600.max_backends) {
		1411	rdev->config.r600.max_backends = tmp;
		1412	}
1430	serge	1413
2997	Serge	1414	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
		1415	tmp = R6XX_MAX_PIPES -
		1416	r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
		1417	if (tmp < rdev->config.r600.max_pipes) {
		1418	rdev->config.r600.max_pipes = tmp;
		1419	}
		1420	tmp = R6XX_MAX_SIMDS -
		1421	r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
		1422	if (tmp < rdev->config.r600.max_simds) {
		1423	rdev->config.r600.max_simds = tmp;
		1424	}
1430	serge	1425
2997	Serge	1426	disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
		1427	tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
		1428	tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
		1429	R6XX_MAX_BACKENDS, disabled_rb_mask);
		1430	tiling_config \|= tmp << 16;
		1431	rdev->config.r600.backend_map = tmp;
		1432
1963	serge	1433	rdev->config.r600.tile_config = tiling_config;
1221	serge	1434	WREG32(GB_TILING_CONFIG, tiling_config);
		1435	WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
		1436	WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
3192	Serge	1437	WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
1221	serge	1438
1430	serge	1439	tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1221	serge	1440	WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
		1441	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
		1442
		1443	/* Setup some CP states */
		1444	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) \| ROQ_IB2_START(0x2b)));
		1445	WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) \| ROQ_END(0x40)));
		1446
		1447	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO \| SYNC_GRADIENT \|
		1448	SYNC_WALKER \| SYNC_ALIGNER));
		1449	/* Setup various GPU states */
		1450	if (rdev->family == CHIP_RV670)
		1451	WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
		1452
		1453	tmp = RREG32(SX_DEBUG_1);
		1454	tmp \|= SMX_EVENT_RELEASE;
		1455	if ((rdev->family > CHIP_R600))
		1456	tmp \|= ENABLE_NEW_SMX_ADDRESS;
		1457	WREG32(SX_DEBUG_1, tmp);
		1458
		1459	if (((rdev->family) == CHIP_R600) \|\|
		1460	((rdev->family) == CHIP_RV630) \|\|
		1461	((rdev->family) == CHIP_RV610) \|\|
		1462	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1463	((rdev->family) == CHIP_RS780) \|\|
		1464	((rdev->family) == CHIP_RS880)) {
1221	serge	1465	WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
		1466	} else {
		1467	WREG32(DB_DEBUG, 0);
		1468	}
		1469	WREG32(DB_WATERMARKS, (DEPTH_FREE(4) \| DEPTH_CACHELINE_FREE(16) \|
		1470	DEPTH_FLUSH(16) \| DEPTH_PENDING_FREE(4)));
		1471
		1472	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1473	WREG32(VGT_NUM_INSTANCES, 0);
		1474
		1475	WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
		1476	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
		1477
		1478	tmp = RREG32(SQ_MS_FIFO_SIZES);
		1479	if (((rdev->family) == CHIP_RV610) \|\|
		1480	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1481	((rdev->family) == CHIP_RS780) \|\|
		1482	((rdev->family) == CHIP_RS880)) {
1221	serge	1483	tmp = (CACHE_FIFO_SIZE(0xa) \|
		1484	FETCH_FIFO_HIWATER(0xa) \|
		1485	DONE_FIFO_HIWATER(0xe0) \|
		1486	ALU_UPDATE_FIFO_HIWATER(0x8));
		1487	} else if (((rdev->family) == CHIP_R600) \|\|
		1488	((rdev->family) == CHIP_RV630)) {
		1489	tmp &= ~DONE_FIFO_HIWATER(0xff);
		1490	tmp \|= DONE_FIFO_HIWATER(0x4);
		1491	}
		1492	WREG32(SQ_MS_FIFO_SIZES, tmp);
		1493
		1494	/* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
		1495	* should be adjusted as needed by the 2D/3D drivers. This just sets default values
		1496	*/
		1497	sq_config = RREG32(SQ_CONFIG);
		1498	sq_config &= ~(PS_PRIO(3) \|
		1499	VS_PRIO(3) \|
		1500	GS_PRIO(3) \|
		1501	ES_PRIO(3));
		1502	sq_config \|= (DX9_CONSTS \|
		1503	VC_ENABLE \|
		1504	PS_PRIO(0) \|
		1505	VS_PRIO(1) \|
		1506	GS_PRIO(2) \|
		1507	ES_PRIO(3));
		1508
		1509	if ((rdev->family) == CHIP_R600) {
		1510	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) \|
		1511	NUM_VS_GPRS(124) \|
		1512	NUM_CLAUSE_TEMP_GPRS(4));
		1513	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) \|
		1514	NUM_ES_GPRS(0));
		1515	sq_thread_resource_mgmt = (NUM_PS_THREADS(136) \|
		1516	NUM_VS_THREADS(48) \|
		1517	NUM_GS_THREADS(4) \|
		1518	NUM_ES_THREADS(4));
		1519	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) \|
		1520	NUM_VS_STACK_ENTRIES(128));
		1521	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) \|
		1522	NUM_ES_STACK_ENTRIES(0));
		1523	} else if (((rdev->family) == CHIP_RV610) \|\|
		1524	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1525	((rdev->family) == CHIP_RS780) \|\|
		1526	((rdev->family) == CHIP_RS880)) {
1221	serge	1527	/* no vertex cache */
		1528	sq_config &= ~VC_ENABLE;
		1529
		1530	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1531	NUM_VS_GPRS(44) \|
		1532	NUM_CLAUSE_TEMP_GPRS(2));
		1533	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1534	NUM_ES_GPRS(17));
		1535	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1536	NUM_VS_THREADS(78) \|
		1537	NUM_GS_THREADS(4) \|
		1538	NUM_ES_THREADS(31));
		1539	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1540	NUM_VS_STACK_ENTRIES(40));
		1541	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1542	NUM_ES_STACK_ENTRIES(16));
		1543	} else if (((rdev->family) == CHIP_RV630) \|\|
		1544	((rdev->family) == CHIP_RV635)) {
		1545	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1546	NUM_VS_GPRS(44) \|
		1547	NUM_CLAUSE_TEMP_GPRS(2));
		1548	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) \|
		1549	NUM_ES_GPRS(18));
		1550	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1551	NUM_VS_THREADS(78) \|
		1552	NUM_GS_THREADS(4) \|
		1553	NUM_ES_THREADS(31));
		1554	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) \|
		1555	NUM_VS_STACK_ENTRIES(40));
		1556	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) \|
		1557	NUM_ES_STACK_ENTRIES(16));
		1558	} else if ((rdev->family) == CHIP_RV670) {
		1559	sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) \|
		1560	NUM_VS_GPRS(44) \|
		1561	NUM_CLAUSE_TEMP_GPRS(2));
		1562	sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) \|
		1563	NUM_ES_GPRS(17));
		1564	sq_thread_resource_mgmt = (NUM_PS_THREADS(79) \|
		1565	NUM_VS_THREADS(78) \|
		1566	NUM_GS_THREADS(4) \|
		1567	NUM_ES_THREADS(31));
		1568	sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) \|
		1569	NUM_VS_STACK_ENTRIES(64));
		1570	sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) \|
		1571	NUM_ES_STACK_ENTRIES(64));
		1572	}
		1573
		1574	WREG32(SQ_CONFIG, sq_config);
		1575	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
		1576	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
		1577	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
		1578	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
		1579	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
		1580
		1581	if (((rdev->family) == CHIP_RV610) \|\|
		1582	((rdev->family) == CHIP_RV620) \|\|
1268	serge	1583	((rdev->family) == CHIP_RS780) \|\|
		1584	((rdev->family) == CHIP_RS880)) {
1221	serge	1585	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
		1586	} else {
		1587	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
		1588	}
		1589
		1590	/* More default values. 2D/3D driver should adjust as needed */
		1591	WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) \| S0_Y(0x4) \|
		1592	S1_X(0x4) \| S1_Y(0xc)));
		1593	WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) \| S0_Y(0xe) \|
		1594	S1_X(0x2) \| S1_Y(0x2) \|
		1595	S2_X(0xa) \| S2_Y(0x6) \|
		1596	S3_X(0x6) \| S3_Y(0xa)));
		1597	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) \| S0_Y(0xb) \|
		1598	S1_X(0x4) \| S1_Y(0xc) \|
		1599	S2_X(0x1) \| S2_Y(0x6) \|
		1600	S3_X(0xa) \| S3_Y(0xe)));
		1601	WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) \| S4_Y(0x1) \|
		1602	S5_X(0x0) \| S5_Y(0x0) \|
		1603	S6_X(0xb) \| S6_Y(0x4) \|
		1604	S7_X(0x7) \| S7_Y(0x8)));
		1605
		1606	WREG32(VGT_STRMOUT_EN, 0);
		1607	tmp = rdev->config.r600.max_pipes * 16;
		1608	switch (rdev->family) {
		1609	case CHIP_RV610:
1268	serge	1610	case CHIP_RV620:
1221	serge	1611	case CHIP_RS780:
1268	serge	1612	case CHIP_RS880:
1221	serge	1613	tmp += 32;
		1614	break;
		1615	case CHIP_RV670:
		1616	tmp += 128;
		1617	break;
		1618	default:
		1619	break;
		1620	}
		1621	if (tmp > 256) {
		1622	tmp = 256;
		1623	}
		1624	WREG32(VGT_ES_PER_GS, 128);
		1625	WREG32(VGT_GS_PER_ES, tmp);
		1626	WREG32(VGT_GS_PER_VS, 2);
		1627	WREG32(VGT_GS_VERTEX_REUSE, 16);
		1628
		1629	/* more default values. 2D/3D driver should adjust as needed */
		1630	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
		1631	WREG32(VGT_STRMOUT_EN, 0);
		1632	WREG32(SX_MISC, 0);
		1633	WREG32(PA_SC_MODE_CNTL, 0);
		1634	WREG32(PA_SC_AA_CONFIG, 0);
		1635	WREG32(PA_SC_LINE_STIPPLE, 0);
		1636	WREG32(SPI_INPUT_Z, 0);
		1637	WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
		1638	WREG32(CB_COLOR7_FRAG, 0);
		1639
		1640	/* Clear render buffer base addresses */
		1641	WREG32(CB_COLOR0_BASE, 0);
		1642	WREG32(CB_COLOR1_BASE, 0);
		1643	WREG32(CB_COLOR2_BASE, 0);
		1644	WREG32(CB_COLOR3_BASE, 0);
		1645	WREG32(CB_COLOR4_BASE, 0);
		1646	WREG32(CB_COLOR5_BASE, 0);
		1647	WREG32(CB_COLOR6_BASE, 0);
		1648	WREG32(CB_COLOR7_BASE, 0);
		1649	WREG32(CB_COLOR7_FRAG, 0);
		1650
		1651	switch (rdev->family) {
		1652	case CHIP_RV610:
1268	serge	1653	case CHIP_RV620:
1221	serge	1654	case CHIP_RS780:
1268	serge	1655	case CHIP_RS880:
1221	serge	1656	tmp = TC_L2_SIZE(8);
		1657	break;
		1658	case CHIP_RV630:
		1659	case CHIP_RV635:
		1660	tmp = TC_L2_SIZE(4);
		1661	break;
		1662	case CHIP_R600:
		1663	tmp = TC_L2_SIZE(0) \| L2_DISABLE_LATE_HIT;
		1664	break;
		1665	default:
		1666	tmp = TC_L2_SIZE(0);
		1667	break;
		1668	}
		1669	WREG32(TC_CNTL, tmp);
		1670
		1671	tmp = RREG32(HDP_HOST_PATH_CNTL);
		1672	WREG32(HDP_HOST_PATH_CNTL, tmp);
		1673
		1674	tmp = RREG32(ARB_POP);
		1675	tmp \|= ENABLE_TC128;
		1676	WREG32(ARB_POP, tmp);
		1677
		1678	WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
		1679	WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA \|
		1680	NUM_CLIP_SEQ(3)));
		1681	WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
2997	Serge	1682	WREG32(VC_ENHANCE, 0);
1221	serge	1683	}
		1684
		1685
1128	serge	1686	/*
		1687	* Indirect registers accessor
		1688	*/
1221	serge	1689	u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
1128	serge	1690	{
1221	serge	1691	u32 r;
1128	serge	1692
1221	serge	1693	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1694	(void)RREG32(PCIE_PORT_INDEX);
		1695	r = RREG32(PCIE_PORT_DATA);
1128	serge	1696	return r;
		1697	}
		1698
1221	serge	1699	void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
1128	serge	1700	{
1221	serge	1701	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
		1702	(void)RREG32(PCIE_PORT_INDEX);
		1703	WREG32(PCIE_PORT_DATA, (v));
		1704	(void)RREG32(PCIE_PORT_DATA);
1128	serge	1705	}
1221	serge	1706
		1707	/*
		1708	* CP & Ring
		1709	*/
		1710	void r600_cp_stop(struct radeon_device *rdev)
		1711	{
3192	Serge	1712	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1221	serge	1713	WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1963	serge	1714	WREG32(SCRATCH_UMSK, 0);
3192	Serge	1715	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1221	serge	1716	}
1413	serge	1717
		1718	int r600_init_microcode(struct radeon_device *rdev)
		1719	{
		1720	struct platform_device *pdev;
		1721	const char *chip_name;
		1722	const char *rlc_chip_name;
		1723	size_t pfp_req_size, me_req_size, rlc_req_size;
		1724	char fw_name[30];
		1725	int err;
		1726
		1727	DRM_DEBUG("\n");
		1728
		1729	pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
		1730	err = IS_ERR(pdev);
		1731	if (err) {
		1732	printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
		1733	return -EINVAL;
		1734	}
		1735
		1736	switch (rdev->family) {
		1737	case CHIP_R600:
		1738	chip_name = "R600";
		1739	rlc_chip_name = "R600";
		1740	break;
		1741	case CHIP_RV610:
		1742	chip_name = "RV610";
		1743	rlc_chip_name = "R600";
		1744	break;
		1745	case CHIP_RV630:
		1746	chip_name = "RV630";
		1747	rlc_chip_name = "R600";
		1748	break;
		1749	case CHIP_RV620:
		1750	chip_name = "RV620";
		1751	rlc_chip_name = "R600";
		1752	break;
		1753	case CHIP_RV635:
		1754	chip_name = "RV635";
		1755	rlc_chip_name = "R600";
		1756	break;
		1757	case CHIP_RV670:
		1758	chip_name = "RV670";
		1759	rlc_chip_name = "R600";
		1760	break;
		1761	case CHIP_RS780:
		1762	case CHIP_RS880:
		1763	chip_name = "RS780";
		1764	rlc_chip_name = "R600";
		1765	break;
		1766	case CHIP_RV770:
		1767	chip_name = "RV770";
		1768	rlc_chip_name = "R700";
		1769	break;
		1770	case CHIP_RV730:
		1771	case CHIP_RV740:
		1772	chip_name = "RV730";
		1773	rlc_chip_name = "R700";
		1774	break;
		1775	case CHIP_RV710:
		1776	chip_name = "RV710";
		1777	rlc_chip_name = "R700";
		1778	break;
1963	serge	1779	case CHIP_CEDAR:
		1780	chip_name = "CEDAR";
		1781	rlc_chip_name = "CEDAR";
		1782	break;
		1783	case CHIP_REDWOOD:
		1784	chip_name = "REDWOOD";
		1785	rlc_chip_name = "REDWOOD";
		1786	break;
		1787	case CHIP_JUNIPER:
		1788	chip_name = "JUNIPER";
		1789	rlc_chip_name = "JUNIPER";
		1790	break;
		1791	case CHIP_CYPRESS:
		1792	case CHIP_HEMLOCK:
		1793	chip_name = "CYPRESS";
		1794	rlc_chip_name = "CYPRESS";
		1795	break;
		1796	case CHIP_PALM:
		1797	chip_name = "PALM";
		1798	rlc_chip_name = "SUMO";
		1799	break;
1986	serge	1800	case CHIP_SUMO:
		1801	chip_name = "SUMO";
		1802	rlc_chip_name = "SUMO";
		1803	break;
		1804	case CHIP_SUMO2:
		1805	chip_name = "SUMO2";
		1806	rlc_chip_name = "SUMO";
		1807	break;
1413	serge	1808	default: BUG();
		1809	}
		1810
1963	serge	1811	if (rdev->family >= CHIP_CEDAR) {
		1812	pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
		1813	me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
		1814	rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
		1815	} else if (rdev->family >= CHIP_RV770) {
1413	serge	1816	pfp_req_size = R700_PFP_UCODE_SIZE * 4;
		1817	me_req_size = R700_PM4_UCODE_SIZE * 4;
		1818	rlc_req_size = R700_RLC_UCODE_SIZE * 4;
		1819	} else {
		1820	pfp_req_size = PFP_UCODE_SIZE * 4;
		1821	me_req_size = PM4_UCODE_SIZE * 12;
		1822	rlc_req_size = RLC_UCODE_SIZE * 4;
		1823	}
		1824
		1825	DRM_INFO("Loading %s Microcode\n", chip_name);
		1826
		1827	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
		1828	err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
		1829	if (err)
		1830	goto out;
		1831	if (rdev->pfp_fw->size != pfp_req_size) {
		1832	printk(KERN_ERR
		1833	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1834	rdev->pfp_fw->size, fw_name);
		1835	err = -EINVAL;
		1836	goto out;
		1837	}
		1838
		1839	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
		1840	err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
		1841	if (err)
		1842	goto out;
		1843	if (rdev->me_fw->size != me_req_size) {
		1844	printk(KERN_ERR
		1845	"r600_cp: Bogus length %zu in firmware \"%s\"\n",
		1846	rdev->me_fw->size, fw_name);
		1847	err = -EINVAL;
		1848	}
		1849
		1850	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
		1851	err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
		1852	if (err)
		1853	goto out;
		1854	if (rdev->rlc_fw->size != rlc_req_size) {
		1855	printk(KERN_ERR
		1856	"r600_rlc: Bogus length %zu in firmware \"%s\"\n",
		1857	rdev->rlc_fw->size, fw_name);
		1858	err = -EINVAL;
		1859	}
		1860
		1861	out:
		1862	platform_device_unregister(pdev);
		1863
		1864	if (err) {
		1865	if (err != -EINVAL)
		1866	printk(KERN_ERR
		1867	"r600_cp: Failed to load firmware \"%s\"\n",
		1868	fw_name);
		1869	release_firmware(rdev->pfp_fw);
		1870	rdev->pfp_fw = NULL;
		1871	release_firmware(rdev->me_fw);
		1872	rdev->me_fw = NULL;
		1873	release_firmware(rdev->rlc_fw);
		1874	rdev->rlc_fw = NULL;
		1875	}
		1876	return err;
		1877	}
		1878
		1879	static int r600_cp_load_microcode(struct radeon_device *rdev)
		1880	{
		1881	const __be32 *fw_data;
		1882	int i;
		1883
		1884	if (!rdev->me_fw \|\| !rdev->pfp_fw)
		1885	return -EINVAL;
		1886
		1887	r600_cp_stop(rdev);
		1888
1963	serge	1889	WREG32(CP_RB_CNTL,
		1890	#ifdef __BIG_ENDIAN
		1891	BUF_SWAP_32BIT \|
		1892	#endif
		1893	RB_NO_UPDATE \| RB_BLKSZ(15) \| RB_BUFSZ(3));
1413	serge	1894
		1895	/* Reset cp */
		1896	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1897	RREG32(GRBM_SOFT_RESET);
		1898	mdelay(15);
		1899	WREG32(GRBM_SOFT_RESET, 0);
		1900
		1901	WREG32(CP_ME_RAM_WADDR, 0);
		1902
		1903	fw_data = (const __be32 *)rdev->me_fw->data;
		1904	WREG32(CP_ME_RAM_WADDR, 0);
		1905	for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
		1906	WREG32(CP_ME_RAM_DATA,
		1907	be32_to_cpup(fw_data++));
		1908
		1909	fw_data = (const __be32 *)rdev->pfp_fw->data;
		1910	WREG32(CP_PFP_UCODE_ADDR, 0);
		1911	for (i = 0; i < PFP_UCODE_SIZE; i++)
		1912	WREG32(CP_PFP_UCODE_DATA,
		1913	be32_to_cpup(fw_data++));
		1914
		1915	WREG32(CP_PFP_UCODE_ADDR, 0);
		1916	WREG32(CP_ME_RAM_WADDR, 0);
		1917	WREG32(CP_ME_RAM_RADDR, 0);
		1918	return 0;
		1919	}
		1920
1221	serge	1921	int r600_cp_start(struct radeon_device *rdev)
		1922	{
2997	Serge	1923	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1221	serge	1924	int r;
		1925	uint32_t cp_me;
		1926
2997	Serge	1927	r = radeon_ring_lock(rdev, ring, 7);
1221	serge	1928	if (r) {
		1929	DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
		1930	return r;
		1931	}
2997	Serge	1932	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
		1933	radeon_ring_write(ring, 0x1);
1963	serge	1934	if (rdev->family >= CHIP_RV770) {
2997	Serge	1935	radeon_ring_write(ring, 0x0);
		1936	radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
1963	serge	1937	} else {
2997	Serge	1938	radeon_ring_write(ring, 0x3);
		1939	radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
1221	serge	1940	}
2997	Serge	1941	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
		1942	radeon_ring_write(ring, 0);
		1943	radeon_ring_write(ring, 0);
		1944	radeon_ring_unlock_commit(rdev, ring);
1221	serge	1945
		1946	cp_me = 0xff;
		1947	WREG32(R_0086D8_CP_ME_CNTL, cp_me);
		1948	return 0;
		1949	}
1413	serge	1950
		1951	int r600_cp_resume(struct radeon_device *rdev)
		1952	{
2997	Serge	1953	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1413	serge	1954	u32 tmp;
		1955	u32 rb_bufsz;
		1956	int r;
		1957
		1958	/* Reset cp */
		1959	WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
		1960	RREG32(GRBM_SOFT_RESET);
		1961	mdelay(15);
		1962	WREG32(GRBM_SOFT_RESET, 0);
		1963
		1964	/* Set ring buffer size */
2997	Serge	1965	rb_bufsz = drm_order(ring->ring_size / 8);
1963	serge	1966	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) \| rb_bufsz;
1413	serge	1967	#ifdef __BIG_ENDIAN
		1968	tmp \|= BUF_SWAP_32BIT;
		1969	#endif
		1970	WREG32(CP_RB_CNTL, tmp);
2997	Serge	1971	WREG32(CP_SEM_WAIT_TIMER, 0x0);
1413	serge	1972
		1973	/* Set the write pointer delay */
		1974	WREG32(CP_RB_WPTR_DELAY, 0);
		1975
		1976	/* Initialize the ring buffer's read and write pointers */
		1977	WREG32(CP_RB_CNTL, tmp \| RB_RPTR_WR_ENA);
		1978	WREG32(CP_RB_RPTR_WR, 0);
2997	Serge	1979	ring->wptr = 0;
		1980	WREG32(CP_RB_WPTR, ring->wptr);
1963	serge	1981
		1982	/* set the wb address whether it's enabled or not */
		1983	WREG32(CP_RB_RPTR_ADDR,
		1984	((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
		1985	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
		1986	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
		1987
		1988	if (rdev->wb.enabled)
		1989	WREG32(SCRATCH_UMSK, 0xff);
		1990	else {
		1991	tmp \|= RB_NO_UPDATE;
		1992	WREG32(SCRATCH_UMSK, 0);
		1993	}
		1994
1413	serge	1995	mdelay(1);
		1996	WREG32(CP_RB_CNTL, tmp);
		1997
2997	Serge	1998	WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
1413	serge	1999	WREG32(CP_DEBUG, (1 << 27) \| (1 << 28));
		2000
2997	Serge	2001	ring->rptr = RREG32(CP_RB_RPTR);
1413	serge	2002
		2003	r600_cp_start(rdev);
2997	Serge	2004	ring->ready = true;
		2005	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
1413	serge	2006	if (r) {
2997	Serge	2007	ring->ready = false;
1413	serge	2008	return r;
		2009	}
		2010	return 0;
		2011	}
		2012
2997	Serge	2013	void r600_ring_init(struct radeon_device rdev, struct radeon_ring ring, unsigned ring_size)
1221	serge	2014	{
1233	serge	2015	u32 rb_bufsz;
2997	Serge	2016	int r;
1221	serge	2017
1233	serge	2018	/* Align ring size */
		2019	rb_bufsz = drm_order(ring_size / 8);
		2020	ring_size = (1 << (rb_bufsz + 1)) * 4;
2997	Serge	2021	ring->ring_size = ring_size;
		2022	ring->align_mask = 16 - 1;
		2023
		2024	if (radeon_ring_supports_scratch_reg(rdev, ring)) {
		2025	r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
		2026	if (r) {
		2027	DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
		2028	ring->rptr_save_reg = 0;
		2029	}
		2030	}
1233	serge	2031	}
		2032
1963	serge	2033	void r600_cp_fini(struct radeon_device *rdev)
		2034	{
2997	Serge	2035	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1963	serge	2036	r600_cp_stop(rdev);
2997	Serge	2037	radeon_ring_fini(rdev, ring);
		2038	radeon_scratch_free(rdev, ring->rptr_save_reg);
1963	serge	2039	}
1233	serge	2040
3192	Serge	2041	/*
		2042	* DMA
		2043	* Starting with R600, the GPU has an asynchronous
		2044	* DMA engine. The programming model is very similar
		2045	* to the 3D engine (ring buffer, IBs, etc.), but the
		2046	* DMA controller has it's own packet format that is
		2047	* different form the PM4 format used by the 3D engine.
		2048	* It supports copying data, writing embedded data,
		2049	* solid fills, and a number of other things. It also
		2050	* has support for tiling/detiling of buffers.
		2051	*/
		2052	/**
		2053	* r600_dma_stop - stop the async dma engine
		2054	*
		2055	* @rdev: radeon_device pointer
		2056	*
		2057	* Stop the async dma engine (r6xx-evergreen).
		2058	*/
		2059	void r600_dma_stop(struct radeon_device *rdev)
		2060	{
		2061	u32 rb_cntl = RREG32(DMA_RB_CNTL);
1963	serge	2062
3192	Serge	2063	radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
		2064
		2065	rb_cntl &= ~DMA_RB_ENABLE;
		2066	WREG32(DMA_RB_CNTL, rb_cntl);
		2067
		2068	rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
		2069	}
		2070
		2071	/**
		2072	* r600_dma_resume - setup and start the async dma engine
		2073	*
		2074	* @rdev: radeon_device pointer
		2075	*
		2076	* Set up the DMA ring buffer and enable it. (r6xx-evergreen).
		2077	* Returns 0 for success, error for failure.
		2078	*/
		2079	int r600_dma_resume(struct radeon_device *rdev)
		2080	{
		2081	struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
3764	Serge	2082	u32 rb_cntl, dma_cntl, ib_cntl;
3192	Serge	2083	u32 rb_bufsz;
		2084	int r;
		2085
		2086	/* Reset dma */
		2087	if (rdev->family >= CHIP_RV770)
		2088	WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
		2089	else
		2090	WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
		2091	RREG32(SRBM_SOFT_RESET);
		2092	udelay(50);
		2093	WREG32(SRBM_SOFT_RESET, 0);
		2094
		2095	WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0);
		2096	WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
		2097
		2098	/* Set ring buffer size in dwords */
		2099	rb_bufsz = drm_order(ring->ring_size / 4);
		2100	rb_cntl = rb_bufsz << 1;
		2101	#ifdef __BIG_ENDIAN
		2102	rb_cntl \|= DMA_RB_SWAP_ENABLE \| DMA_RPTR_WRITEBACK_SWAP_ENABLE;
		2103	#endif
		2104	WREG32(DMA_RB_CNTL, rb_cntl);
		2105
		2106	/* Initialize the ring buffer's read and write pointers */
		2107	WREG32(DMA_RB_RPTR, 0);
		2108	WREG32(DMA_RB_WPTR, 0);
		2109
		2110	/* set the wb address whether it's enabled or not */
		2111	WREG32(DMA_RB_RPTR_ADDR_HI,
		2112	upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF);
		2113	WREG32(DMA_RB_RPTR_ADDR_LO,
		2114	((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC));
		2115
		2116	if (rdev->wb.enabled)
		2117	rb_cntl \|= DMA_RPTR_WRITEBACK_ENABLE;
		2118
		2119	WREG32(DMA_RB_BASE, ring->gpu_addr >> 8);
		2120
		2121	/* enable DMA IBs */
3764	Serge	2122	ib_cntl = DMA_IB_ENABLE;
		2123	#ifdef __BIG_ENDIAN
		2124	ib_cntl \|= DMA_IB_SWAP_ENABLE;
		2125	#endif
		2126	WREG32(DMA_IB_CNTL, ib_cntl);
3192	Serge	2127
		2128	dma_cntl = RREG32(DMA_CNTL);
		2129	dma_cntl &= ~CTXEMPTY_INT_ENABLE;
		2130	WREG32(DMA_CNTL, dma_cntl);
		2131
		2132	if (rdev->family >= CHIP_RV770)
		2133	WREG32(DMA_MODE, 1);
		2134
		2135	ring->wptr = 0;
		2136	WREG32(DMA_RB_WPTR, ring->wptr << 2);
		2137
		2138	ring->rptr = RREG32(DMA_RB_RPTR) >> 2;
		2139
		2140	WREG32(DMA_RB_CNTL, rb_cntl \| DMA_RB_ENABLE);
		2141
		2142	ring->ready = true;
		2143
		2144	r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring);
		2145	if (r) {
		2146	ring->ready = false;
		2147	return r;
		2148	}
		2149
		2150	radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
		2151
		2152	return 0;
		2153	}
		2154
		2155	/**
		2156	* r600_dma_fini - tear down the async dma engine
		2157	*
		2158	* @rdev: radeon_device pointer
		2159	*
		2160	* Stop the async dma engine and free the ring (r6xx-evergreen).
		2161	*/
		2162	void r600_dma_fini(struct radeon_device *rdev)
		2163	{
		2164	r600_dma_stop(rdev);
		2165	radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
		2166	}
		2167
1233	serge	2168	/*
3764	Serge	2169	* UVD
		2170	*/
		2171	int r600_uvd_rbc_start(struct radeon_device *rdev)
		2172	{
		2173	struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
		2174	uint64_t rptr_addr;
		2175	uint32_t rb_bufsz, tmp;
		2176	int r;
		2177
		2178	rptr_addr = rdev->wb.gpu_addr + R600_WB_UVD_RPTR_OFFSET;
		2179
		2180	if (upper_32_bits(rptr_addr) != upper_32_bits(ring->gpu_addr)) {
		2181	DRM_ERROR("UVD ring and rptr not in the same 4GB segment!\n");
		2182	return -EINVAL;
		2183	}
		2184
		2185	/* force RBC into idle state */
		2186	WREG32(UVD_RBC_RB_CNTL, 0x11010101);
		2187
		2188	/* Set the write pointer delay */
		2189	WREG32(UVD_RBC_RB_WPTR_CNTL, 0);
		2190
		2191	/* set the wb address */
		2192	WREG32(UVD_RBC_RB_RPTR_ADDR, rptr_addr >> 2);
		2193
		2194	/* programm the 4GB memory segment for rptr and ring buffer */
		2195	WREG32(UVD_LMI_EXT40_ADDR, upper_32_bits(rptr_addr) \|
		2196	(0x7 << 16) \| (0x1 << 31));
		2197
		2198	/* Initialize the ring buffer's read and write pointers */
		2199	WREG32(UVD_RBC_RB_RPTR, 0x0);
		2200
		2201	ring->wptr = ring->rptr = RREG32(UVD_RBC_RB_RPTR);
		2202	WREG32(UVD_RBC_RB_WPTR, ring->wptr);
		2203
		2204	/* set the ring address */
		2205	WREG32(UVD_RBC_RB_BASE, ring->gpu_addr);
		2206
		2207	/* Set ring buffer size */
		2208	rb_bufsz = drm_order(ring->ring_size);
		2209	rb_bufsz = (0x1 << 8) \| rb_bufsz;
		2210	WREG32(UVD_RBC_RB_CNTL, rb_bufsz);
		2211
		2212	ring->ready = true;
		2213	r = radeon_ring_test(rdev, R600_RING_TYPE_UVD_INDEX, ring);
		2214	if (r) {
		2215	ring->ready = false;
		2216	return r;
		2217	}
		2218
		2219	r = radeon_ring_lock(rdev, ring, 10);
		2220	if (r) {
		2221	DRM_ERROR("radeon: ring failed to lock UVD ring (%d).\n", r);
		2222	return r;
		2223	}
		2224
		2225	tmp = PACKET0(UVD_SEMA_WAIT_FAULT_TIMEOUT_CNTL, 0);
		2226	radeon_ring_write(ring, tmp);
		2227	radeon_ring_write(ring, 0xFFFFF);
		2228
		2229	tmp = PACKET0(UVD_SEMA_WAIT_INCOMPLETE_TIMEOUT_CNTL, 0);
		2230	radeon_ring_write(ring, tmp);
		2231	radeon_ring_write(ring, 0xFFFFF);
		2232
		2233	tmp = PACKET0(UVD_SEMA_SIGNAL_INCOMPLETE_TIMEOUT_CNTL, 0);
		2234	radeon_ring_write(ring, tmp);
		2235	radeon_ring_write(ring, 0xFFFFF);
		2236
		2237	/* Clear timeout status bits */
		2238	radeon_ring_write(ring, PACKET0(UVD_SEMA_TIMEOUT_STATUS, 0));
		2239	radeon_ring_write(ring, 0x8);
		2240
		2241	radeon_ring_write(ring, PACKET0(UVD_SEMA_CNTL, 0));
		2242	radeon_ring_write(ring, 3);
		2243
		2244	radeon_ring_unlock_commit(rdev, ring);
		2245
		2246	return 0;
		2247	}
		2248
		2249	void r600_uvd_rbc_stop(struct radeon_device *rdev)
		2250	{
		2251	struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
		2252
		2253	/* force RBC into idle state */
		2254	WREG32(UVD_RBC_RB_CNTL, 0x11010101);
		2255	ring->ready = false;
		2256	}
		2257
		2258	int r600_uvd_init(struct radeon_device *rdev)
		2259	{
		2260	int i, j, r;
		2261	/* disable byte swapping */
		2262	u32 lmi_swap_cntl = 0;
		2263	u32 mp_swap_cntl = 0;
		2264
		2265	/* raise clocks while booting up the VCPU */
		2266	radeon_set_uvd_clocks(rdev, 53300, 40000);
		2267
		2268	/* disable clock gating */
		2269	WREG32(UVD_CGC_GATE, 0);
		2270
		2271	/* disable interupt */
		2272	WREG32_P(UVD_MASTINT_EN, 0, ~(1 << 1));
		2273
		2274	/* put LMI, VCPU, RBC etc... into reset */
		2275	WREG32(UVD_SOFT_RESET, LMI_SOFT_RESET \| VCPU_SOFT_RESET \|
		2276	LBSI_SOFT_RESET \| RBC_SOFT_RESET \| CSM_SOFT_RESET \|
		2277	CXW_SOFT_RESET \| TAP_SOFT_RESET \| LMI_UMC_SOFT_RESET);
		2278	mdelay(5);
		2279
		2280	/* take UVD block out of reset */
		2281	WREG32_P(SRBM_SOFT_RESET, 0, ~SOFT_RESET_UVD);
		2282	mdelay(5);
		2283
		2284	/* initialize UVD memory controller */
		2285	WREG32(UVD_LMI_CTRL, 0x40 \| (1 << 8) \| (1 << 13) \|
		2286	(1 << 21) \| (1 << 9) \| (1 << 20));
		2287
		2288	#ifdef __BIG_ENDIAN
		2289	/* swap (8 in 32) RB and IB */
		2290	lmi_swap_cntl = 0xa;
		2291	mp_swap_cntl = 0;
		2292	#endif
		2293	WREG32(UVD_LMI_SWAP_CNTL, lmi_swap_cntl);
		2294	WREG32(UVD_MP_SWAP_CNTL, mp_swap_cntl);
		2295
		2296	WREG32(UVD_MPC_SET_MUXA0, 0x40c2040);
		2297	WREG32(UVD_MPC_SET_MUXA1, 0x0);
		2298	WREG32(UVD_MPC_SET_MUXB0, 0x40c2040);
		2299	WREG32(UVD_MPC_SET_MUXB1, 0x0);
		2300	WREG32(UVD_MPC_SET_ALU, 0);
		2301	WREG32(UVD_MPC_SET_MUX, 0x88);
		2302
		2303	/* Stall UMC */
		2304	WREG32_P(UVD_LMI_CTRL2, 1 << 8, ~(1 << 8));
		2305	WREG32_P(UVD_RB_ARB_CTRL, 1 << 3, ~(1 << 3));
		2306
		2307	/* take all subblocks out of reset, except VCPU */
		2308	WREG32(UVD_SOFT_RESET, VCPU_SOFT_RESET);
		2309	mdelay(5);
		2310
		2311	/* enable VCPU clock */
		2312	WREG32(UVD_VCPU_CNTL, 1 << 9);
		2313
		2314	/* enable UMC */
		2315	WREG32_P(UVD_LMI_CTRL2, 0, ~(1 << 8));
		2316
		2317	/* boot up the VCPU */
		2318	WREG32(UVD_SOFT_RESET, 0);
		2319	mdelay(10);
		2320
		2321	WREG32_P(UVD_RB_ARB_CTRL, 0, ~(1 << 3));
		2322
		2323	for (i = 0; i < 10; ++i) {
		2324	uint32_t status;
		2325	for (j = 0; j < 100; ++j) {
		2326	status = RREG32(UVD_STATUS);
		2327	if (status & 2)
		2328	break;
		2329	mdelay(10);
		2330	}
		2331	r = 0;
		2332	if (status & 2)
		2333	break;
		2334
		2335	DRM_ERROR("UVD not responding, trying to reset the VCPU!!!\n");
		2336	WREG32_P(UVD_SOFT_RESET, VCPU_SOFT_RESET, ~VCPU_SOFT_RESET);
		2337	mdelay(10);
		2338	WREG32_P(UVD_SOFT_RESET, 0, ~VCPU_SOFT_RESET);
		2339	mdelay(10);
		2340	r = -1;
		2341	}
		2342
		2343	if (r) {
		2344	DRM_ERROR("UVD not responding, giving up!!!\n");
		2345	radeon_set_uvd_clocks(rdev, 0, 0);
		2346	return r;
		2347	}
		2348
		2349	/* enable interupt */
		2350	WREG32_P(UVD_MASTINT_EN, 3<<1, ~(3 << 1));
		2351
		2352	r = r600_uvd_rbc_start(rdev);
		2353	if (!r)
		2354	DRM_INFO("UVD initialized successfully.\n");
		2355
		2356	/* lower clocks again */
		2357	radeon_set_uvd_clocks(rdev, 0, 0);
		2358
		2359	return r;
		2360	}
		2361
		2362	/*
1233	serge	2363	* GPU scratch registers helpers function.
		2364	*/
		2365	void r600_scratch_init(struct radeon_device *rdev)
		2366	{
		2367	int i;
		2368
		2369	rdev->scratch.num_reg = 7;
1963	serge	2370	rdev->scratch.reg_base = SCRATCH_REG0;
1233	serge	2371	for (i = 0; i < rdev->scratch.num_reg; i++) {
		2372	rdev->scratch.free[i] = true;
1963	serge	2373	rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
1233	serge	2374	}
		2375	}
1413	serge	2376
2997	Serge	2377	int r600_ring_test(struct radeon_device rdev, struct radeon_ring ring)
1413	serge	2378	{
		2379	uint32_t scratch;
		2380	uint32_t tmp = 0;
		2381	unsigned i;
		2382	int r;
		2383
		2384	r = radeon_scratch_get(rdev, &scratch);
		2385	if (r) {
		2386	DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
		2387	return r;
		2388	}
		2389	WREG32(scratch, 0xCAFEDEAD);
2997	Serge	2390	r = radeon_ring_lock(rdev, ring, 3);
1413	serge	2391	if (r) {
2997	Serge	2392	DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
1413	serge	2393	radeon_scratch_free(rdev, scratch);
		2394	return r;
		2395	}
2997	Serge	2396	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2397	radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		2398	radeon_ring_write(ring, 0xDEADBEEF);
		2399	radeon_ring_unlock_commit(rdev, ring);
1413	serge	2400	for (i = 0; i < rdev->usec_timeout; i++) {
		2401	tmp = RREG32(scratch);
		2402	if (tmp == 0xDEADBEEF)
		2403	break;
		2404	DRM_UDELAY(1);
		2405	}
		2406	if (i < rdev->usec_timeout) {
2997	Serge	2407	DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
1413	serge	2408	} else {
2997	Serge	2409	DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
		2410	ring->idx, scratch, tmp);
1413	serge	2411	r = -EINVAL;
		2412	}
		2413	radeon_scratch_free(rdev, scratch);
		2414	return r;
		2415	}
1963	serge	2416
3192	Serge	2417	/**
		2418	* r600_dma_ring_test - simple async dma engine test
		2419	*
		2420	* @rdev: radeon_device pointer
		2421	* @ring: radeon_ring structure holding ring information
		2422	*
		2423	* Test the DMA engine by writing using it to write an
		2424	* value to memory. (r6xx-SI).
		2425	* Returns 0 for success, error for failure.
		2426	*/
		2427	int r600_dma_ring_test(struct radeon_device *rdev,
		2428	struct radeon_ring *ring)
		2429	{
		2430	unsigned i;
		2431	int r;
		2432	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		2433	u32 tmp;
		2434
		2435	if (!ptr) {
		2436	DRM_ERROR("invalid vram scratch pointer\n");
		2437	return -EINVAL;
		2438	}
		2439
		2440	tmp = 0xCAFEDEAD;
		2441	writel(tmp, ptr);
		2442
		2443	r = radeon_ring_lock(rdev, ring, 4);
		2444	if (r) {
		2445	DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
		2446	return r;
		2447	}
		2448	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
		2449	radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
		2450	radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff);
		2451	radeon_ring_write(ring, 0xDEADBEEF);
		2452	radeon_ring_unlock_commit(rdev, ring);
		2453
		2454	for (i = 0; i < rdev->usec_timeout; i++) {
		2455	tmp = readl(ptr);
		2456	if (tmp == 0xDEADBEEF)
		2457	break;
		2458	DRM_UDELAY(1);
		2459	}
		2460
		2461	if (i < rdev->usec_timeout) {
		2462	DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
		2463	} else {
		2464	DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
		2465	ring->idx, tmp);
		2466	r = -EINVAL;
		2467	}
		2468	return r;
		2469	}
		2470
3764	Serge	2471	int r600_uvd_ring_test(struct radeon_device rdev, struct radeon_ring ring)
		2472	{
		2473	uint32_t tmp = 0;
		2474	unsigned i;
		2475	int r;
		2476
		2477	WREG32(UVD_CONTEXT_ID, 0xCAFEDEAD);
		2478	r = radeon_ring_lock(rdev, ring, 3);
		2479	if (r) {
		2480	DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n",
		2481	ring->idx, r);
		2482	return r;
		2483	}
		2484	radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0));
		2485	radeon_ring_write(ring, 0xDEADBEEF);
		2486	radeon_ring_unlock_commit(rdev, ring);
		2487	for (i = 0; i < rdev->usec_timeout; i++) {
		2488	tmp = RREG32(UVD_CONTEXT_ID);
		2489	if (tmp == 0xDEADBEEF)
		2490	break;
		2491	DRM_UDELAY(1);
		2492	}
		2493
		2494	if (i < rdev->usec_timeout) {
		2495	DRM_INFO("ring test on %d succeeded in %d usecs\n",
		2496	ring->idx, i);
		2497	} else {
		2498	DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
		2499	ring->idx, tmp);
		2500	r = -EINVAL;
		2501	}
		2502	return r;
		2503	}
		2504
3192	Serge	2505	/*
		2506	* CP fences/semaphores
		2507	*/
		2508
1413	serge	2509	void r600_fence_ring_emit(struct radeon_device *rdev,
		2510	struct radeon_fence *fence)
		2511	{
2997	Serge	2512	struct radeon_ring *ring = &rdev->ring[fence->ring];
		2513
1963	serge	2514	if (rdev->wb.use_event) {
2997	Serge	2515	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		2516	/* flush read cache over gart */
		2517	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		2518	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA \|
		2519	PACKET3_VC_ACTION_ENA \|
		2520	PACKET3_SH_ACTION_ENA);
		2521	radeon_ring_write(ring, 0xFFFFFFFF);
		2522	radeon_ring_write(ring, 0);
		2523	radeon_ring_write(ring, 10); /* poll interval */
1963	serge	2524	/* EVENT_WRITE_EOP - flush caches, send int */
2997	Serge	2525	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
		2526	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) \| EVENT_INDEX(5));
		2527	radeon_ring_write(ring, addr & 0xffffffff);
		2528	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| DATA_SEL(1) \| INT_SEL(2));
		2529	radeon_ring_write(ring, fence->seq);
		2530	radeon_ring_write(ring, 0);
1963	serge	2531	} else {
2997	Serge	2532	/* flush read cache over gart */
		2533	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
		2534	radeon_ring_write(ring, PACKET3_TC_ACTION_ENA \|
		2535	PACKET3_VC_ACTION_ENA \|
		2536	PACKET3_SH_ACTION_ENA);
		2537	radeon_ring_write(ring, 0xFFFFFFFF);
		2538	radeon_ring_write(ring, 0);
		2539	radeon_ring_write(ring, 10); /* poll interval */
		2540	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
		2541	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) \| EVENT_INDEX(0));
1430	serge	2542	/* wait for 3D idle clean */
2997	Serge	2543	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2544	radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		2545	radeon_ring_write(ring, WAIT_3D_IDLE_bit \| WAIT_3D_IDLECLEAN_bit);
1413	serge	2546	/* Emit fence sequence & fire IRQ */
2997	Serge	2547	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2548	radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		2549	radeon_ring_write(ring, fence->seq);
1413	serge	2550	/* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2997	Serge	2551	radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
		2552	radeon_ring_write(ring, RB_INT_STAT);
1963	serge	2553	}
1413	serge	2554	}
1963	serge	2555
3764	Serge	2556	void r600_uvd_fence_emit(struct radeon_device *rdev,
		2557	struct radeon_fence *fence)
		2558	{
		2559	struct radeon_ring *ring = &rdev->ring[fence->ring];
		2560	uint32_t addr = rdev->fence_drv[fence->ring].gpu_addr;
		2561
		2562	radeon_ring_write(ring, PACKET0(UVD_CONTEXT_ID, 0));
		2563	radeon_ring_write(ring, fence->seq);
		2564	radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0));
		2565	radeon_ring_write(ring, addr & 0xffffffff);
		2566	radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0));
		2567	radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
		2568	radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0));
		2569	radeon_ring_write(ring, 0);
		2570
		2571	radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA0, 0));
		2572	radeon_ring_write(ring, 0);
		2573	radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_DATA1, 0));
		2574	radeon_ring_write(ring, 0);
		2575	radeon_ring_write(ring, PACKET0(UVD_GPCOM_VCPU_CMD, 0));
		2576	radeon_ring_write(ring, 2);
		2577	return;
		2578	}
		2579
2997	Serge	2580	void r600_semaphore_ring_emit(struct radeon_device *rdev,
		2581	struct radeon_ring *ring,
		2582	struct radeon_semaphore *semaphore,
		2583	bool emit_wait)
		2584	{
		2585	uint64_t addr = semaphore->gpu_addr;
		2586	unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
		2587
		2588	if (rdev->family < CHIP_CAYMAN)
		2589	sel \|= PACKET3_SEM_WAIT_ON_SIGNAL;
		2590
		2591	radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
		2592	radeon_ring_write(ring, addr & 0xffffffff);
		2593	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) \| sel);
		2594	}
		2595
3192	Serge	2596	/*
		2597	* DMA fences/semaphores
		2598	*/
		2599
		2600	/**
		2601	* r600_dma_fence_ring_emit - emit a fence on the DMA ring
		2602	*
		2603	* @rdev: radeon_device pointer
		2604	* @fence: radeon fence object
		2605	*
		2606	* Add a DMA fence packet to the ring to write
		2607	* the fence seq number and DMA trap packet to generate
		2608	* an interrupt if needed (r6xx-r7xx).
		2609	*/
		2610	void r600_dma_fence_ring_emit(struct radeon_device *rdev,
		2611	struct radeon_fence *fence)
		2612	{
		2613	struct radeon_ring *ring = &rdev->ring[fence->ring];
		2614	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
		2615
		2616	/* write the fence */
		2617	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0));
		2618	radeon_ring_write(ring, addr & 0xfffffffc);
		2619	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
		2620	radeon_ring_write(ring, lower_32_bits(fence->seq));
		2621	/* generate an interrupt */
		2622	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0));
		2623	}
		2624
		2625	/**
		2626	* r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring
		2627	*
		2628	* @rdev: radeon_device pointer
		2629	* @ring: radeon_ring structure holding ring information
		2630	* @semaphore: radeon semaphore object
		2631	* @emit_wait: wait or signal semaphore
		2632	*
		2633	* Add a DMA semaphore packet to the ring wait on or signal
		2634	* other rings (r6xx-SI).
		2635	*/
		2636	void r600_dma_semaphore_ring_emit(struct radeon_device *rdev,
		2637	struct radeon_ring *ring,
		2638	struct radeon_semaphore *semaphore,
		2639	bool emit_wait)
		2640	{
		2641	u64 addr = semaphore->gpu_addr;
		2642	u32 s = emit_wait ? 0 : 1;
		2643
		2644	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0));
		2645	radeon_ring_write(ring, addr & 0xfffffffc);
		2646	radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
		2647	}
		2648
3764	Serge	2649	void r600_uvd_semaphore_emit(struct radeon_device *rdev,
		2650	struct radeon_ring *ring,
		2651	struct radeon_semaphore *semaphore,
		2652	bool emit_wait)
		2653	{
		2654	uint64_t addr = semaphore->gpu_addr;
		2655
		2656	radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_LOW, 0));
		2657	radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF);
		2658
		2659	radeon_ring_write(ring, PACKET0(UVD_SEMA_ADDR_HIGH, 0));
		2660	radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF);
		2661
		2662	radeon_ring_write(ring, PACKET0(UVD_SEMA_CMD, 0));
		2663	radeon_ring_write(ring, emit_wait ? 1 : 0);
		2664	}
		2665
2005	serge	2666	int r600_copy_blit(struct radeon_device *rdev,
2997	Serge	2667	uint64_t src_offset,
		2668	uint64_t dst_offset,
		2669	unsigned num_gpu_pages,
		2670	struct radeon_fence **fence)
2005	serge	2671	{
2997	Serge	2672	struct radeon_semaphore *sem = NULL;
		2673	struct radeon_sa_bo *vb = NULL;
2005	serge	2674	int r;
1963	serge	2675
2997	Serge	2676	r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem);
2005	serge	2677	if (r) {
		2678	return r;
		2679	}
2997	Serge	2680	r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb);
		2681	r600_blit_done_copy(rdev, fence, vb, sem);
2005	serge	2682	return 0;
		2683	}
		2684
3192	Serge	2685	/**
		2686	* r600_copy_dma - copy pages using the DMA engine
		2687	*
		2688	* @rdev: radeon_device pointer
		2689	* @src_offset: src GPU address
		2690	* @dst_offset: dst GPU address
		2691	* @num_gpu_pages: number of GPU pages to xfer
		2692	* @fence: radeon fence object
		2693	*
		2694	* Copy GPU paging using the DMA engine (r6xx).
		2695	* Used by the radeon ttm implementation to move pages if
		2696	* registered as the asic copy callback.
		2697	*/
		2698	int r600_copy_dma(struct radeon_device *rdev,
		2699	uint64_t src_offset, uint64_t dst_offset,
		2700	unsigned num_gpu_pages,
		2701	struct radeon_fence **fence)
		2702	{
		2703	struct radeon_semaphore *sem = NULL;
		2704	int ring_index = rdev->asic->copy.dma_ring_index;
		2705	struct radeon_ring *ring = &rdev->ring[ring_index];
		2706	u32 size_in_dw, cur_size_in_dw;
		2707	int i, num_loops;
		2708	int r = 0;
		2709
		2710	r = radeon_semaphore_create(rdev, &sem);
		2711	if (r) {
		2712	DRM_ERROR("radeon: moving bo (%d).\n", r);
		2713	return r;
		2714	}
		2715
		2716	size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
		2717	num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE);
		2718	r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8);
		2719	if (r) {
		2720	DRM_ERROR("radeon: moving bo (%d).\n", r);
		2721	radeon_semaphore_free(rdev, &sem, NULL);
		2722	return r;
		2723	}
		2724
		2725	if (radeon_fence_need_sync(*fence, ring->idx)) {
		2726	radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
		2727	ring->idx);
		2728	radeon_fence_note_sync(*fence, ring->idx);
		2729	} else {
		2730	radeon_semaphore_free(rdev, &sem, NULL);
		2731	}
		2732
		2733	for (i = 0; i < num_loops; i++) {
		2734	cur_size_in_dw = size_in_dw;
		2735	if (cur_size_in_dw > 0xFFFE)
		2736	cur_size_in_dw = 0xFFFE;
		2737	size_in_dw -= cur_size_in_dw;
		2738	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
		2739	radeon_ring_write(ring, dst_offset & 0xfffffffc);
		2740	radeon_ring_write(ring, src_offset & 0xfffffffc);
		2741	radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) \|
		2742	(upper_32_bits(src_offset) & 0xff)));
		2743	src_offset += cur_size_in_dw * 4;
		2744	dst_offset += cur_size_in_dw * 4;
		2745	}
		2746
		2747	r = radeon_fence_emit(rdev, fence, ring->idx);
		2748	if (r) {
		2749	radeon_ring_unlock_undo(rdev, ring);
		2750	return r;
		2751	}
		2752
		2753	radeon_ring_unlock_commit(rdev, ring);
		2754	radeon_semaphore_free(rdev, &sem, *fence);
		2755
		2756	return r;
		2757	}
		2758
1221	serge	2759	int r600_set_surface_reg(struct radeon_device *rdev, int reg,
		2760	uint32_t tiling_flags, uint32_t pitch,
		2761	uint32_t offset, uint32_t obj_size)
		2762	{
		2763	/* FIXME: implement */
		2764	return 0;
		2765	}
		2766
		2767	void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
		2768	{
		2769	/* FIXME: implement */
		2770	}
		2771
2997	Serge	2772	static int r600_startup(struct radeon_device *rdev)
1221	serge	2773	{
3192	Serge	2774	struct radeon_ring *ring;
1221	serge	2775	int r;
		2776
1963	serge	2777	/* enable pcie gen2 link */
		2778	r600_pcie_gen2_enable(rdev);
		2779
1413	serge	2780	if (!rdev->me_fw \|\| !rdev->pfp_fw \|\| !rdev->rlc_fw) {
		2781	r = r600_init_microcode(rdev);
		2782	if (r) {
		2783	DRM_ERROR("Failed to load firmware!\n");
		2784	return r;
		2785	}
		2786	}
		2787
3764	Serge	2788	r = r600_vram_scratch_init(rdev);
		2789	if (r)
		2790	return r;
		2791
1221	serge	2792	r600_mc_program(rdev);
		2793	if (rdev->flags & RADEON_IS_AGP) {
		2794	r600_agp_enable(rdev);
		2795	} else {
		2796	r = r600_pcie_gart_enable(rdev);
		2797	if (r)
		2798	return r;
		2799	}
		2800	r600_gpu_init(rdev);
2005	serge	2801	r = r600_blit_init(rdev);
		2802	if (r) {
		2803	// r600_blit_fini(rdev);
2997	Serge	2804	rdev->asic->copy.copy = NULL;
2005	serge	2805	dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
		2806	}
1221	serge	2807
2005	serge	2808	/* allocate wb buffer */
		2809	r = radeon_wb_init(rdev);
		2810	if (r)
		2811	return r;
		2812
3192	Serge	2813	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
		2814	if (r) {
		2815	dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
		2816	return r;
		2817	}
		2818
		2819	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
		2820	if (r) {
		2821	dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
		2822	return r;
		2823	}
		2824
2005	serge	2825	/* Enable IRQ */
3764	Serge	2826	if (!rdev->irq.installed) {
		2827	r = radeon_irq_kms_init(rdev);
		2828	if (r)
		2829	return r;
		2830	}
		2831
2005	serge	2832	r = r600_irq_init(rdev);
		2833	if (r) {
		2834	DRM_ERROR("radeon: IH init failed (%d).\n", r);
		2835	// radeon_irq_kms_fini(rdev);
		2836	return r;
		2837	}
		2838	r600_irq_set(rdev);
		2839
3192	Serge	2840	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2997	Serge	2841	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
		2842	R600_CP_RB_RPTR, R600_CP_RB_WPTR,
		2843	0, 0xfffff, RADEON_CP_PACKET2);
3192	Serge	2844	if (r)
		2845	return r;
2997	Serge	2846
3192	Serge	2847	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
		2848	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
		2849	DMA_RB_RPTR, DMA_RB_WPTR,
		2850	2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
1413	serge	2851	if (r)
		2852	return r;
3192	Serge	2853
1413	serge	2854	r = r600_cp_load_microcode(rdev);
		2855	if (r)
		2856	return r;
		2857	r = r600_cp_resume(rdev);
		2858	if (r)
		2859	return r;
1963	serge	2860
3192	Serge	2861	r = r600_dma_resume(rdev);
		2862	if (r)
		2863	return r;
		2864
		2865	r = radeon_ib_pool_init(rdev);
		2866	if (r) {
		2867	dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
		2868	return r;
		2869	}
1221	serge	2870	return 0;
		2871	}
		2872
		2873	void r600_vga_set_state(struct radeon_device *rdev, bool state)
		2874	{
		2875	uint32_t temp;
		2876
		2877	temp = RREG32(CONFIG_CNTL);
		2878	if (state == false) {
		2879	temp &= ~(1<<0);
		2880	temp \|= (1<<1);
		2881	} else {
		2882	temp &= ~(1<<1);
		2883	}
		2884	WREG32(CONFIG_CNTL, temp);
		2885	}
		2886
		2887
		2888
		2889
		2890
		2891	/* Plan is to move initialization in that function and use
		2892	* helper function so that radeon_device_init pretty much
		2893	* do nothing more than calling asic specific function. This
		2894	* should also allow to remove a bunch of callback function
		2895	* like vram_info.
		2896	*/
		2897	int r600_init(struct radeon_device *rdev)
		2898	{
		2899	int r;
		2900
		2901	if (r600_debugfs_mc_info_init(rdev)) {
		2902	DRM_ERROR("Failed to register debugfs file for mc !\n");
		2903	}
		2904	/* Read BIOS */
		2905	if (!radeon_get_bios(rdev)) {
		2906	if (ASIC_IS_AVIVO(rdev))
		2907	return -EINVAL;
		2908	}
		2909	/* Must be an ATOMBIOS */
		2910	if (!rdev->is_atom_bios) {
		2911	dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
		2912	return -EINVAL;
		2913	}
		2914	r = radeon_atombios_init(rdev);
		2915	if (r)
		2916	return r;
		2917	/* Post card if necessary */
1963	serge	2918	if (!radeon_card_posted(rdev)) {
1321	serge	2919	if (!rdev->bios) {
		2920	dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
		2921	return -EINVAL;
		2922	}
1221	serge	2923	DRM_INFO("GPU not posted. posting now...\n");
		2924	atom_asic_init(rdev->mode_info.atom_context);
		2925	}
		2926	/* Initialize scratch registers */
		2927	r600_scratch_init(rdev);
		2928	/* Initialize surface registers */
		2929	radeon_surface_init(rdev);
1268	serge	2930	/* Initialize clocks */
1221	serge	2931	radeon_get_clock_info(rdev->ddev);
		2932	/* Fence driver */
2004	serge	2933	r = radeon_fence_driver_init(rdev);
		2934	if (r)
		2935	return r;
1403	serge	2936	if (rdev->flags & RADEON_IS_AGP) {
		2937	r = radeon_agp_init(rdev);
		2938	if (r)
		2939	radeon_agp_disable(rdev);
		2940	}
1221	serge	2941	r = r600_mc_init(rdev);
		2942	if (r)
		2943	return r;
		2944	/* Memory manager */
1321	serge	2945	r = radeon_bo_init(rdev);
1221	serge	2946	if (r)
		2947	return r;
1321	serge	2948
2997	Serge	2949	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
		2950	r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
1221	serge	2951
3192	Serge	2952	rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
		2953	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
		2954
2004	serge	2955	rdev->ih.ring_obj = NULL;
		2956	r600_ih_ring_init(rdev, 64 * 1024);
1221	serge	2957
		2958	r = r600_pcie_gart_init(rdev);
		2959	if (r)
		2960	return r;
		2961
1321	serge	2962	rdev->accel_working = true;
1221	serge	2963	r = r600_startup(rdev);
		2964	if (r) {
1428	serge	2965	dev_err(rdev->dev, "disabling GPU acceleration\n");
1221	serge	2966	r600_pcie_gart_fini(rdev);
		2967	rdev->accel_working = false;
		2968	}
2005	serge	2969
1221	serge	2970	return 0;
		2971	}
		2972
2004	serge	2973	/*
		2974	* CS stuff
		2975	*/
		2976	void r600_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		2977	{
2997	Serge	2978	struct radeon_ring *ring = &rdev->ring[ib->ring];
		2979	u32 next_rptr;
		2980
		2981	if (ring->rptr_save_reg) {
		2982	next_rptr = ring->wptr + 3 + 4;
		2983	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
		2984	radeon_ring_write(ring, ((ring->rptr_save_reg -
		2985	PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
		2986	radeon_ring_write(ring, next_rptr);
		2987	} else if (rdev->wb.enabled) {
		2988	next_rptr = ring->wptr + 5 + 4;
		2989	radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
		2990	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		2991	radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) \| (1 << 18));
		2992	radeon_ring_write(ring, next_rptr);
		2993	radeon_ring_write(ring, 0);
		2994	}
		2995
		2996	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
		2997	radeon_ring_write(ring,
2004	serge	2998	#ifdef __BIG_ENDIAN
		2999	(2 << 0) \|
		3000	#endif
		3001	(ib->gpu_addr & 0xFFFFFFFC));
2997	Serge	3002	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
		3003	radeon_ring_write(ring, ib->length_dw);
2004	serge	3004	}
		3005
3764	Serge	3006	void r600_uvd_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		3007	{
		3008	struct radeon_ring *ring = &rdev->ring[ib->ring];
		3009
		3010	radeon_ring_write(ring, PACKET0(UVD_RBC_IB_BASE, 0));
		3011	radeon_ring_write(ring, ib->gpu_addr);
		3012	radeon_ring_write(ring, PACKET0(UVD_RBC_IB_SIZE, 0));
		3013	radeon_ring_write(ring, ib->length_dw);
		3014	}
		3015
2997	Serge	3016	int r600_ib_test(struct radeon_device rdev, struct radeon_ring ring)
2004	serge	3017	{
2997	Serge	3018	struct radeon_ib ib;
2004	serge	3019	uint32_t scratch;
		3020	uint32_t tmp = 0;
		3021	unsigned i;
		3022	int r;
		3023
		3024	r = radeon_scratch_get(rdev, &scratch);
		3025	if (r) {
		3026	DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
		3027	return r;
		3028	}
		3029	WREG32(scratch, 0xCAFEDEAD);
2997	Serge	3030	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
2004	serge	3031	if (r) {
		3032	DRM_ERROR("radeon: failed to get ib (%d).\n", r);
2997	Serge	3033	goto free_scratch;
2004	serge	3034	}
2997	Serge	3035	ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
		3036	ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
		3037	ib.ptr[2] = 0xDEADBEEF;
		3038	ib.length_dw = 3;
		3039	r = radeon_ib_schedule(rdev, &ib, NULL);
2004	serge	3040	if (r) {
		3041	DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
2997	Serge	3042	goto free_ib;
2004	serge	3043	}
2997	Serge	3044	r = radeon_fence_wait(ib.fence, false);
2004	serge	3045	if (r) {
		3046	DRM_ERROR("radeon: fence wait failed (%d).\n", r);
2997	Serge	3047	goto free_ib;
2004	serge	3048	}
		3049	for (i = 0; i < rdev->usec_timeout; i++) {
		3050	tmp = RREG32(scratch);
		3051	if (tmp == 0xDEADBEEF)
		3052	break;
		3053	DRM_UDELAY(1);
		3054	}
		3055	if (i < rdev->usec_timeout) {
2997	Serge	3056	DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
2004	serge	3057	} else {
		3058	DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
		3059	scratch, tmp);
		3060	r = -EINVAL;
		3061	}
2997	Serge	3062	free_ib:
		3063	radeon_ib_free(rdev, &ib);
		3064	free_scratch:
2004	serge	3065	radeon_scratch_free(rdev, scratch);
		3066	return r;
		3067	}
		3068
3192	Serge	3069	/**
		3070	* r600_dma_ib_test - test an IB on the DMA engine
		3071	*
		3072	* @rdev: radeon_device pointer
		3073	* @ring: radeon_ring structure holding ring information
		3074	*
		3075	* Test a simple IB in the DMA ring (r6xx-SI).
		3076	* Returns 0 on success, error on failure.
		3077	*/
		3078	int r600_dma_ib_test(struct radeon_device rdev, struct radeon_ring ring)
		3079	{
		3080	struct radeon_ib ib;
		3081	unsigned i;
		3082	int r;
		3083	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		3084	u32 tmp = 0;
		3085
		3086	if (!ptr) {
		3087	DRM_ERROR("invalid vram scratch pointer\n");
		3088	return -EINVAL;
		3089	}
		3090
		3091	tmp = 0xCAFEDEAD;
		3092	writel(tmp, ptr);
		3093
		3094	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
		3095	if (r) {
		3096	DRM_ERROR("radeon: failed to get ib (%d).\n", r);
		3097	return r;
		3098	}
		3099
		3100	ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1);
		3101	ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
		3102	ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff;
		3103	ib.ptr[3] = 0xDEADBEEF;
		3104	ib.length_dw = 4;
		3105
		3106	r = radeon_ib_schedule(rdev, &ib, NULL);
		3107	if (r) {
		3108	radeon_ib_free(rdev, &ib);
		3109	DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
		3110	return r;
		3111	}
		3112	r = radeon_fence_wait(ib.fence, false);
		3113	if (r) {
		3114	DRM_ERROR("radeon: fence wait failed (%d).\n", r);
		3115	return r;
		3116	}
		3117	for (i = 0; i < rdev->usec_timeout; i++) {
		3118	tmp = readl(ptr);
		3119	if (tmp == 0xDEADBEEF)
		3120	break;
		3121	DRM_UDELAY(1);
		3122	}
		3123	if (i < rdev->usec_timeout) {
		3124	DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
		3125	} else {
		3126	DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
		3127	r = -EINVAL;
		3128	}
		3129	radeon_ib_free(rdev, &ib);
3764	Serge	3130	return r;
		3131	}
3192	Serge	3132
3764	Serge	3133	int r600_uvd_ib_test(struct radeon_device rdev, struct radeon_ring ring)
		3134	{
		3135	struct radeon_fence *fence = NULL;
		3136	int r;
3192	Serge	3137
3764	Serge	3138	r = radeon_set_uvd_clocks(rdev, 53300, 40000);
		3139	if (r) {
		3140	DRM_ERROR("radeon: failed to raise UVD clocks (%d).\n", r);
		3141	return r;
		3142	}
		3143
		3144	// r = radeon_uvd_get_create_msg(rdev, ring->idx, 1, NULL);
		3145	if (r) {
		3146	DRM_ERROR("radeon: failed to get create msg (%d).\n", r);
		3147	goto error;
		3148	}
		3149
		3150	// r = radeon_uvd_get_destroy_msg(rdev, ring->idx, 1, &fence);
		3151	if (r) {
		3152	DRM_ERROR("radeon: failed to get destroy ib (%d).\n", r);
		3153	goto error;
		3154	}
		3155
		3156	r = radeon_fence_wait(fence, false);
		3157	if (r) {
		3158	DRM_ERROR("radeon: fence wait failed (%d).\n", r);
		3159	goto error;
		3160	}
		3161	DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
		3162	error:
		3163	radeon_fence_unref(&fence);
		3164	radeon_set_uvd_clocks(rdev, 0, 0);
3192	Serge	3165	return r;
		3166	}
		3167
		3168	/**
		3169	* r600_dma_ring_ib_execute - Schedule an IB on the DMA engine
		3170	*
		3171	* @rdev: radeon_device pointer
		3172	* @ib: IB object to schedule
		3173	*
		3174	* Schedule an IB in the DMA ring (r6xx-r7xx).
		3175	*/
		3176	void r600_dma_ring_ib_execute(struct radeon_device rdev, struct radeon_ib ib)
		3177	{
		3178	struct radeon_ring *ring = &rdev->ring[ib->ring];
		3179
		3180	if (rdev->wb.enabled) {
		3181	u32 next_rptr = ring->wptr + 4;
		3182	while ((next_rptr & 7) != 5)
		3183	next_rptr++;
		3184	next_rptr += 3;
		3185	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
		3186	radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
		3187	radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
		3188	radeon_ring_write(ring, next_rptr);
		3189	}
		3190
		3191	/* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
		3192	* Pad as necessary with NOPs.
		3193	*/
		3194	while ((ring->wptr & 7) != 5)
		3195	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
		3196	radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0));
		3197	radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
		3198	radeon_ring_write(ring, (ib->length_dw << 16) \| (upper_32_bits(ib->gpu_addr) & 0xFF));
		3199
		3200	}
		3201
2004	serge	3202	/*
		3203	* Interrupts
		3204	*
		3205	* Interrupts use a ring buffer on r6xx/r7xx hardware. It works pretty
		3206	* the same as the CP ring buffer, but in reverse. Rather than the CPU
		3207	* writing to the ring and the GPU consuming, the GPU writes to the ring
		3208	* and host consumes. As the host irq handler processes interrupts, it
		3209	* increments the rptr. When the rptr catches up with the wptr, all the
		3210	* current interrupts have been processed.
		3211	*/
		3212
		3213	void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
		3214	{
		3215	u32 rb_bufsz;
		3216
		3217	/* Align ring size */
		3218	rb_bufsz = drm_order(ring_size / 4);
		3219	ring_size = (1 << rb_bufsz) * 4;
		3220	rdev->ih.ring_size = ring_size;
		3221	rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
		3222	rdev->ih.rptr = 0;
		3223	}
		3224
2997	Serge	3225	int r600_ih_ring_alloc(struct radeon_device *rdev)
2004	serge	3226	{
		3227	int r;
		3228
		3229	/* Allocate ring buffer */
		3230	if (rdev->ih.ring_obj == NULL) {
		3231	r = radeon_bo_create(rdev, rdev->ih.ring_size,
		3232	PAGE_SIZE, true,
		3233	RADEON_GEM_DOMAIN_GTT,
2997	Serge	3234	NULL, &rdev->ih.ring_obj);
2004	serge	3235	if (r) {
		3236	DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
		3237	return r;
		3238	}
		3239	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		3240	if (unlikely(r != 0))
		3241	return r;
		3242	r = radeon_bo_pin(rdev->ih.ring_obj,
		3243	RADEON_GEM_DOMAIN_GTT,
		3244	&rdev->ih.gpu_addr);
		3245	if (r) {
		3246	radeon_bo_unreserve(rdev->ih.ring_obj);
		3247	DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
		3248	return r;
		3249	}
		3250	r = radeon_bo_kmap(rdev->ih.ring_obj,
		3251	(void **)&rdev->ih.ring);
		3252	radeon_bo_unreserve(rdev->ih.ring_obj);
		3253	if (r) {
		3254	DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
		3255	return r;
		3256	}
		3257	}
		3258	return 0;
		3259	}
		3260
2997	Serge	3261	void r600_ih_ring_fini(struct radeon_device *rdev)
2004	serge	3262	{
		3263	int r;
		3264	if (rdev->ih.ring_obj) {
		3265	r = radeon_bo_reserve(rdev->ih.ring_obj, false);
		3266	if (likely(r == 0)) {
		3267	radeon_bo_kunmap(rdev->ih.ring_obj);
		3268	radeon_bo_unpin(rdev->ih.ring_obj);
		3269	radeon_bo_unreserve(rdev->ih.ring_obj);
		3270	}
		3271	radeon_bo_unref(&rdev->ih.ring_obj);
		3272	rdev->ih.ring = NULL;
		3273	rdev->ih.ring_obj = NULL;
		3274	}
		3275	}
		3276
		3277	void r600_rlc_stop(struct radeon_device *rdev)
		3278	{
		3279
		3280	if ((rdev->family >= CHIP_RV770) &&
		3281	(rdev->family <= CHIP_RV740)) {
		3282	/* r7xx asics need to soft reset RLC before halting */
		3283	WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
		3284	RREG32(SRBM_SOFT_RESET);
2997	Serge	3285	mdelay(15);
2004	serge	3286	WREG32(SRBM_SOFT_RESET, 0);
		3287	RREG32(SRBM_SOFT_RESET);
		3288	}
		3289
		3290	WREG32(RLC_CNTL, 0);
		3291	}
		3292
		3293	static void r600_rlc_start(struct radeon_device *rdev)
		3294	{
		3295	WREG32(RLC_CNTL, RLC_ENABLE);
		3296	}
		3297
		3298	static int r600_rlc_init(struct radeon_device *rdev)
		3299	{
		3300	u32 i;
		3301	const __be32 *fw_data;
		3302
		3303	if (!rdev->rlc_fw)
		3304	return -EINVAL;
		3305
		3306	r600_rlc_stop(rdev);
		3307
2997	Serge	3308	WREG32(RLC_HB_CNTL, 0);
		3309
		3310	if (rdev->family == CHIP_ARUBA) {
		3311	WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
		3312	WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
		3313	}
		3314	if (rdev->family <= CHIP_CAYMAN) {
2004	serge	3315	WREG32(RLC_HB_BASE, 0);
		3316	WREG32(RLC_HB_RPTR, 0);
		3317	WREG32(RLC_HB_WPTR, 0);
2997	Serge	3318	}
2004	serge	3319	if (rdev->family <= CHIP_CAICOS) {
		3320	WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
		3321	WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
		3322	}
		3323	WREG32(RLC_MC_CNTL, 0);
		3324	WREG32(RLC_UCODE_CNTL, 0);
		3325
		3326	fw_data = (const __be32 *)rdev->rlc_fw->data;
2997	Serge	3327	if (rdev->family >= CHIP_ARUBA) {
		3328	for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
		3329	WREG32(RLC_UCODE_ADDR, i);
		3330	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		3331	}
		3332	} else if (rdev->family >= CHIP_CAYMAN) {
2004	serge	3333	for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
		3334	WREG32(RLC_UCODE_ADDR, i);
		3335	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		3336	}
		3337	} else if (rdev->family >= CHIP_CEDAR) {
		3338	for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
		3339	WREG32(RLC_UCODE_ADDR, i);
		3340	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		3341	}
		3342	} else if (rdev->family >= CHIP_RV770) {
		3343	for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
		3344	WREG32(RLC_UCODE_ADDR, i);
		3345	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		3346	}
		3347	} else {
		3348	for (i = 0; i < RLC_UCODE_SIZE; i++) {
		3349	WREG32(RLC_UCODE_ADDR, i);
		3350	WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
		3351	}
		3352	}
		3353	WREG32(RLC_UCODE_ADDR, 0);
		3354
		3355	r600_rlc_start(rdev);
		3356
		3357	return 0;
		3358	}
		3359
		3360	static void r600_enable_interrupts(struct radeon_device *rdev)
		3361	{
		3362	u32 ih_cntl = RREG32(IH_CNTL);
		3363	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		3364
		3365	ih_cntl \|= ENABLE_INTR;
		3366	ih_rb_cntl \|= IH_RB_ENABLE;
		3367	WREG32(IH_CNTL, ih_cntl);
		3368	WREG32(IH_RB_CNTL, ih_rb_cntl);
		3369	rdev->ih.enabled = true;
		3370	}
		3371
		3372	void r600_disable_interrupts(struct radeon_device *rdev)
		3373	{
		3374	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
		3375	u32 ih_cntl = RREG32(IH_CNTL);
		3376
		3377	ih_rb_cntl &= ~IH_RB_ENABLE;
		3378	ih_cntl &= ~ENABLE_INTR;
		3379	WREG32(IH_RB_CNTL, ih_rb_cntl);
		3380	WREG32(IH_CNTL, ih_cntl);
		3381	/* set rptr, wptr to 0 */
		3382	WREG32(IH_RB_RPTR, 0);
		3383	WREG32(IH_RB_WPTR, 0);
		3384	rdev->ih.enabled = false;
		3385	rdev->ih.rptr = 0;
		3386	}
		3387
1963	serge	3388	static void r600_disable_interrupt_state(struct radeon_device *rdev)
		3389	{
		3390	u32 tmp;
1221	serge	3391
1963	serge	3392	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE);
3192	Serge	3393	tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
		3394	WREG32(DMA_CNTL, tmp);
1963	serge	3395	WREG32(GRBM_INT_CNTL, 0);
		3396	WREG32(DxMODE_INT_MASK, 0);
		3397	WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
		3398	WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
		3399	if (ASIC_IS_DCE3(rdev)) {
		3400	WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
		3401	WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
		3402	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3403	WREG32(DC_HPD1_INT_CONTROL, tmp);
		3404	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3405	WREG32(DC_HPD2_INT_CONTROL, tmp);
		3406	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3407	WREG32(DC_HPD3_INT_CONTROL, tmp);
		3408	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3409	WREG32(DC_HPD4_INT_CONTROL, tmp);
		3410	if (ASIC_IS_DCE32(rdev)) {
		3411	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3412	WREG32(DC_HPD5_INT_CONTROL, tmp);
		3413	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
		3414	WREG32(DC_HPD6_INT_CONTROL, tmp);
2997	Serge	3415	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3416	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
		3417	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3418	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
		3419	} else {
		3420	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3421	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
		3422	tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3423	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
1963	serge	3424	}
		3425	} else {
		3426	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
		3427	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
		3428	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		3429	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		3430	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		3431	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		3432	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
		3433	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
2997	Serge	3434	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3435	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
		3436	tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3437	WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
1963	serge	3438	}
		3439	}
1221	serge	3440
2004	serge	3441	int r600_irq_init(struct radeon_device *rdev)
		3442	{
		3443	int ret = 0;
		3444	int rb_bufsz;
		3445	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
1221	serge	3446
2004	serge	3447	/* allocate ring */
		3448	ret = r600_ih_ring_alloc(rdev);
		3449	if (ret)
		3450	return ret;
1221	serge	3451
2004	serge	3452	/* disable irqs */
		3453	r600_disable_interrupts(rdev);
1221	serge	3454
2004	serge	3455	/* init rlc */
		3456	ret = r600_rlc_init(rdev);
		3457	if (ret) {
		3458	r600_ih_ring_fini(rdev);
		3459	return ret;
		3460	}
1221	serge	3461
2004	serge	3462	/* setup interrupt control */
		3463	/* set dummy read address to ring address */
		3464	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
		3465	interrupt_cntl = RREG32(INTERRUPT_CNTL);
		3466	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
		3467	* IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
		3468	*/
		3469	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
		3470	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
		3471	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
		3472	WREG32(INTERRUPT_CNTL, interrupt_cntl);
1221	serge	3473
2004	serge	3474	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
		3475	rb_bufsz = drm_order(rdev->ih.ring_size / 4);
1221	serge	3476
2004	serge	3477	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE \|
		3478	IH_WPTR_OVERFLOW_CLEAR \|
		3479	(rb_bufsz << 1));
1963	serge	3480
2004	serge	3481	if (rdev->wb.enabled)
		3482	ih_rb_cntl \|= IH_WPTR_WRITEBACK_ENABLE;
		3483
		3484	/* set the writeback address whether it's enabled or not */
		3485	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
		3486	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
		3487
		3488	WREG32(IH_RB_CNTL, ih_rb_cntl);
		3489
		3490	/* set rptr, wptr to 0 */
		3491	WREG32(IH_RB_RPTR, 0);
		3492	WREG32(IH_RB_WPTR, 0);
		3493
		3494	/* Default settings for IH_CNTL (disabled at first) */
		3495	ih_cntl = MC_WRREQ_CREDIT(0x10) \| MC_WR_CLEAN_CNT(0x10);
		3496	/* RPTR_REARM only works if msi's are enabled */
		3497	if (rdev->msi_enabled)
		3498	ih_cntl \|= RPTR_REARM;
		3499	WREG32(IH_CNTL, ih_cntl);
		3500
		3501	/* force the active interrupt state to all disabled */
		3502	if (rdev->family >= CHIP_CEDAR)
		3503	evergreen_disable_interrupt_state(rdev);
		3504	else
		3505	r600_disable_interrupt_state(rdev);
		3506
2997	Serge	3507	/* at this point everything should be setup correctly to enable master */
		3508	pci_set_master(rdev->pdev);
		3509
2004	serge	3510	/* enable irqs */
		3511	r600_enable_interrupts(rdev);
		3512
		3513	return ret;
		3514	}
		3515	int r600_irq_set(struct radeon_device *rdev)
		3516	{
		3517	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE \| CNTX_EMPTY_INT_ENABLE;
		3518	u32 mode_int = 0;
		3519	u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
		3520	u32 grbm_int_cntl = 0;
2997	Serge	3521	u32 hdmi0, hdmi1;
2004	serge	3522	u32 d1grph = 0, d2grph = 0;
3192	Serge	3523	u32 dma_cntl;
2004	serge	3524
		3525	if (!rdev->irq.installed) {
		3526	WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
		3527	return -EINVAL;
		3528	}
		3529	/* don't enable anything if the ih is disabled */
		3530	if (!rdev->ih.enabled) {
		3531	r600_disable_interrupts(rdev);
		3532	/* force the active interrupt state to all disabled */
		3533	r600_disable_interrupt_state(rdev);
		3534	return 0;
		3535	}
		3536
		3537	if (ASIC_IS_DCE3(rdev)) {
		3538	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3539	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3540	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3541	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3542	if (ASIC_IS_DCE32(rdev)) {
		3543	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3544	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2997	Serge	3545	hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		3546	hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
		3547	} else {
		3548	hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3549	hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3550	}
		3551	} else {
		3552	hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3553	hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
		3554	hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2997	Serge	3555	hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
		3556	hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3557	}
3192	Serge	3558	dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
2004	serge	3559
2997	Serge	3560	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
2004	serge	3561	DRM_DEBUG("r600_irq_set: sw int\n");
		3562	cp_int_cntl \|= RB_INT_ENABLE;
		3563	cp_int_cntl \|= TIME_STAMP_INT_ENABLE;
		3564	}
3192	Serge	3565
		3566	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
		3567	DRM_DEBUG("r600_irq_set: sw int dma\n");
		3568	dma_cntl \|= TRAP_ENABLE;
		3569	}
		3570
2004	serge	3571	if (rdev->irq.crtc_vblank_int[0] \|\|
2997	Serge	3572	atomic_read(&rdev->irq.pflip[0])) {
2004	serge	3573	DRM_DEBUG("r600_irq_set: vblank 0\n");
		3574	mode_int \|= D1MODE_VBLANK_INT_MASK;
		3575	}
		3576	if (rdev->irq.crtc_vblank_int[1] \|\|
2997	Serge	3577	atomic_read(&rdev->irq.pflip[1])) {
2004	serge	3578	DRM_DEBUG("r600_irq_set: vblank 1\n");
		3579	mode_int \|= D2MODE_VBLANK_INT_MASK;
		3580	}
		3581	if (rdev->irq.hpd[0]) {
		3582	DRM_DEBUG("r600_irq_set: hpd 1\n");
		3583	hpd1 \|= DC_HPDx_INT_EN;
		3584	}
		3585	if (rdev->irq.hpd[1]) {
		3586	DRM_DEBUG("r600_irq_set: hpd 2\n");
		3587	hpd2 \|= DC_HPDx_INT_EN;
		3588	}
		3589	if (rdev->irq.hpd[2]) {
		3590	DRM_DEBUG("r600_irq_set: hpd 3\n");
		3591	hpd3 \|= DC_HPDx_INT_EN;
		3592	}
		3593	if (rdev->irq.hpd[3]) {
		3594	DRM_DEBUG("r600_irq_set: hpd 4\n");
		3595	hpd4 \|= DC_HPDx_INT_EN;
		3596	}
		3597	if (rdev->irq.hpd[4]) {
		3598	DRM_DEBUG("r600_irq_set: hpd 5\n");
		3599	hpd5 \|= DC_HPDx_INT_EN;
		3600	}
		3601	if (rdev->irq.hpd[5]) {
		3602	DRM_DEBUG("r600_irq_set: hpd 6\n");
		3603	hpd6 \|= DC_HPDx_INT_EN;
		3604	}
2997	Serge	3605	if (rdev->irq.afmt[0]) {
		3606	DRM_DEBUG("r600_irq_set: hdmi 0\n");
		3607	hdmi0 \|= HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3608	}
2997	Serge	3609	if (rdev->irq.afmt[1]) {
		3610	DRM_DEBUG("r600_irq_set: hdmi 0\n");
		3611	hdmi1 \|= HDMI0_AZ_FORMAT_WTRIG_MASK;
2004	serge	3612	}
		3613
		3614	WREG32(CP_INT_CNTL, cp_int_cntl);
3192	Serge	3615	WREG32(DMA_CNTL, dma_cntl);
2004	serge	3616	WREG32(DxMODE_INT_MASK, mode_int);
		3617	WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
		3618	WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
		3619	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
		3620	if (ASIC_IS_DCE3(rdev)) {
		3621	WREG32(DC_HPD1_INT_CONTROL, hpd1);
		3622	WREG32(DC_HPD2_INT_CONTROL, hpd2);
		3623	WREG32(DC_HPD3_INT_CONTROL, hpd3);
		3624	WREG32(DC_HPD4_INT_CONTROL, hpd4);
		3625	if (ASIC_IS_DCE32(rdev)) {
		3626	WREG32(DC_HPD5_INT_CONTROL, hpd5);
		3627	WREG32(DC_HPD6_INT_CONTROL, hpd6);
2997	Serge	3628	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
		3629	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
		3630	} else {
		3631	WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
		3632	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
2004	serge	3633	}
		3634	} else {
		3635	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
		3636	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
		3637	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
2997	Serge	3638	WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
		3639	WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
2004	serge	3640	}
		3641
		3642	return 0;
		3643	}
		3644
2997	Serge	3645	static void r600_irq_ack(struct radeon_device *rdev)
2004	serge	3646	{
		3647	u32 tmp;
		3648
		3649	if (ASIC_IS_DCE3(rdev)) {
		3650	rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
		3651	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
		3652	rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
2997	Serge	3653	if (ASIC_IS_DCE32(rdev)) {
		3654	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
		3655	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
		3656	} else {
		3657	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
		3658	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
		3659	}
2004	serge	3660	} else {
		3661	rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
		3662	rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
		3663	rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
2997	Serge	3664	rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
		3665	rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
2004	serge	3666	}
		3667	rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
		3668	rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
		3669
		3670	if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		3671	WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		3672	if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
		3673	WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
		3674	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
		3675	WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		3676	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
		3677	WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		3678	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
		3679	WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
		3680	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
		3681	WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
		3682	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		3683	if (ASIC_IS_DCE3(rdev)) {
		3684	tmp = RREG32(DC_HPD1_INT_CONTROL);
		3685	tmp \|= DC_HPDx_INT_ACK;
		3686	WREG32(DC_HPD1_INT_CONTROL, tmp);
		3687	} else {
		3688	tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
		3689	tmp \|= DC_HPDx_INT_ACK;
		3690	WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
		3691	}
		3692	}
		3693	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		3694	if (ASIC_IS_DCE3(rdev)) {
		3695	tmp = RREG32(DC_HPD2_INT_CONTROL);
		3696	tmp \|= DC_HPDx_INT_ACK;
		3697	WREG32(DC_HPD2_INT_CONTROL, tmp);
		3698	} else {
		3699	tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
		3700	tmp \|= DC_HPDx_INT_ACK;
		3701	WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
		3702	}
		3703	}
		3704	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		3705	if (ASIC_IS_DCE3(rdev)) {
		3706	tmp = RREG32(DC_HPD3_INT_CONTROL);
		3707	tmp \|= DC_HPDx_INT_ACK;
		3708	WREG32(DC_HPD3_INT_CONTROL, tmp);
		3709	} else {
		3710	tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
		3711	tmp \|= DC_HPDx_INT_ACK;
		3712	WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
		3713	}
		3714	}
		3715	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		3716	tmp = RREG32(DC_HPD4_INT_CONTROL);
		3717	tmp \|= DC_HPDx_INT_ACK;
		3718	WREG32(DC_HPD4_INT_CONTROL, tmp);
		3719	}
		3720	if (ASIC_IS_DCE32(rdev)) {
		3721	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		3722	tmp = RREG32(DC_HPD5_INT_CONTROL);
		3723	tmp \|= DC_HPDx_INT_ACK;
		3724	WREG32(DC_HPD5_INT_CONTROL, tmp);
		3725	}
		3726	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		3727	tmp = RREG32(DC_HPD5_INT_CONTROL);
		3728	tmp \|= DC_HPDx_INT_ACK;
		3729	WREG32(DC_HPD6_INT_CONTROL, tmp);
		3730	}
2997	Serge	3731	if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
		3732	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
		3733	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		3734	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
		3735	}
		3736	if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
		3737	tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
		3738	tmp \|= AFMT_AZ_FORMAT_WTRIG_ACK;
		3739	WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
2004	serge	3740	}
2997	Serge	3741	} else {
		3742	if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
		3743	tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
		3744	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		3745	WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
2004	serge	3746	}
2997	Serge	3747	if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
2004	serge	3748	if (ASIC_IS_DCE3(rdev)) {
2997	Serge	3749	tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
		3750	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		3751	WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
		3752	} else {
		3753	tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
		3754	tmp \|= HDMI0_AZ_FORMAT_WTRIG_ACK;
		3755	WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
2004	serge	3756	}
		3757	}
		3758	}
		3759	}
		3760
3192	Serge	3761	void r600_irq_disable(struct radeon_device *rdev)
		3762	{
		3763	r600_disable_interrupts(rdev);
		3764	/* Wait and acknowledge irq */
		3765	mdelay(1);
		3766	r600_irq_ack(rdev);
		3767	r600_disable_interrupt_state(rdev);
		3768	}
		3769
2997	Serge	3770	static u32 r600_get_ih_wptr(struct radeon_device *rdev)
2004	serge	3771	{
		3772	u32 wptr, tmp;
		3773
		3774	if (rdev->wb.enabled)
		3775	wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
		3776	else
		3777	wptr = RREG32(IH_RB_WPTR);
		3778
		3779	if (wptr & RB_OVERFLOW) {
		3780	/* When a ring buffer overflow happen start parsing interrupt
		3781	* from the last not overwritten vector (wptr + 16). Hopefully
		3782	* this should allow us to catchup.
		3783	*/
		3784	dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
		3785	wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
		3786	rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
		3787	tmp = RREG32(IH_RB_CNTL);
		3788	tmp \|= IH_WPTR_OVERFLOW_CLEAR;
		3789	WREG32(IH_RB_CNTL, tmp);
		3790	}
		3791	return (wptr & rdev->ih.ptr_mask);
		3792	}
		3793
		3794	/* r600 IV Ring
		3795	* Each IV ring entry is 128 bits:
		3796	* [7:0] - interrupt source id
		3797	* [31:8] - reserved
		3798	* [59:32] - interrupt source data
		3799	* [127:60] - reserved
		3800	*
		3801	* The basic interrupt vector entries
		3802	* are decoded as follows:
		3803	* src_id src_data description
		3804	* 1 0 D1 Vblank
		3805	* 1 1 D1 Vline
		3806	* 5 0 D2 Vblank
		3807	* 5 1 D2 Vline
		3808	* 19 0 FP Hot plug detection A
		3809	* 19 1 FP Hot plug detection B
		3810	* 19 2 DAC A auto-detection
		3811	* 19 3 DAC B auto-detection
		3812	* 21 4 HDMI block A
		3813	* 21 5 HDMI block B
		3814	* 176 - CP_INT RB
		3815	* 177 - CP_INT IB1
		3816	* 178 - CP_INT IB2
		3817	* 181 - EOP Interrupt
		3818	* 233 - GUI Idle
		3819	*
		3820	* Note, these are based on r600 and may need to be
		3821	* adjusted or added to on newer asics
		3822	*/
3764	Serge	3823	#undef DRM_DEBUG
2160	serge	3824	#define DRM_DEBUG(...)
		3825
2004	serge	3826	int r600_irq_process(struct radeon_device *rdev)
		3827	{
		3828	u32 wptr;
		3829	u32 rptr;
		3830	u32 src_id, src_data;
		3831	u32 ring_index;
		3832	bool queue_hotplug = false;
2997	Serge	3833	bool queue_hdmi = false;
2004	serge	3834
		3835	if (!rdev->ih.enabled \|\| rdev->shutdown)
		3836	return IRQ_NONE;
		3837
2160	serge	3838	/* No MSIs, need a dummy read to flush PCI DMAs */
		3839	if (!rdev->msi_enabled)
		3840	RREG32(IH_RB_WPTR);
		3841
2004	serge	3842	wptr = r600_get_ih_wptr(rdev);
		3843
2997	Serge	3844	restart_ih:
		3845	/* is somebody else already processing irqs? */
		3846	if (atomic_xchg(&rdev->ih.lock, 1))
2004	serge	3847	return IRQ_NONE;
		3848
2997	Serge	3849	rptr = rdev->ih.rptr;
		3850	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
		3851
2160	serge	3852	/* Order reading of wptr vs. reading of IH ring data */
		3853	rmb();
		3854
2004	serge	3855	/* display interrupts */
		3856	r600_irq_ack(rdev);
		3857
		3858	while (rptr != wptr) {
		3859	/* wptr/rptr are in bytes! */
		3860	ring_index = rptr / 4;
		3861	src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
		3862	src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
		3863
		3864	switch (src_id) {
		3865	case 1: /* D1 vblank/vline */
		3866	switch (src_data) {
		3867	case 0: /* D1 vblank */
		3868	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
		3869	if (rdev->irq.crtc_vblank_int[0]) {
		3870	// drm_handle_vblank(rdev->ddev, 0);
		3871	rdev->pm.vblank_sync = true;
		3872	// wake_up(&rdev->irq.vblank_queue);
		3873	}
		3874	// if (rdev->irq.pflip[0])
		3875	// radeon_crtc_handle_flip(rdev, 0);
		3876	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
		3877	DRM_DEBUG("IH: D1 vblank\n");
		3878	}
		3879	break;
		3880	case 1: /* D1 vline */
		3881	if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
		3882	rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
		3883	DRM_DEBUG("IH: D1 vline\n");
		3884	}
		3885	break;
		3886	default:
		3887	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3888	break;
		3889	}
		3890	break;
		3891	case 5: /* D2 vblank/vline */
		3892	switch (src_data) {
		3893	case 0: /* D2 vblank */
		3894	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
		3895	if (rdev->irq.crtc_vblank_int[1]) {
		3896	// drm_handle_vblank(rdev->ddev, 1);
		3897	rdev->pm.vblank_sync = true;
		3898	// wake_up(&rdev->irq.vblank_queue);
		3899	}
		3900	// if (rdev->irq.pflip[1])
		3901	// radeon_crtc_handle_flip(rdev, 1);
		3902	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
		3903	DRM_DEBUG("IH: D2 vblank\n");
		3904	}
		3905	break;
		3906	case 1: /* D1 vline */
		3907	if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
		3908	rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
		3909	DRM_DEBUG("IH: D2 vline\n");
		3910	}
		3911	break;
		3912	default:
		3913	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3914	break;
		3915	}
		3916	break;
		3917	case 19: /* HPD/DAC hotplug */
		3918	switch (src_data) {
		3919	case 0:
		3920	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
		3921	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
		3922	queue_hotplug = true;
		3923	DRM_DEBUG("IH: HPD1\n");
		3924	}
		3925	break;
		3926	case 1:
		3927	if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
		3928	rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
		3929	queue_hotplug = true;
		3930	DRM_DEBUG("IH: HPD2\n");
		3931	}
		3932	break;
		3933	case 4:
		3934	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
		3935	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
		3936	queue_hotplug = true;
		3937	DRM_DEBUG("IH: HPD3\n");
		3938	}
		3939	break;
		3940	case 5:
		3941	if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
		3942	rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
		3943	queue_hotplug = true;
		3944	DRM_DEBUG("IH: HPD4\n");
		3945	}
		3946	break;
		3947	case 10:
		3948	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
		3949	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
		3950	queue_hotplug = true;
		3951	DRM_DEBUG("IH: HPD5\n");
		3952	}
		3953	break;
		3954	case 12:
		3955	if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
		3956	rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
		3957	queue_hotplug = true;
		3958	DRM_DEBUG("IH: HPD6\n");
		3959	}
		3960	break;
		3961	default:
		3962	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		3963	break;
		3964	}
		3965	break;
2997	Serge	3966	case 21: /* hdmi */
		3967	switch (src_data) {
		3968	case 4:
		3969	if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
		3970	rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
		3971	queue_hdmi = true;
		3972	DRM_DEBUG("IH: HDMI0\n");
		3973	}
		3974	break;
		3975	case 5:
		3976	if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
		3977	rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
		3978	queue_hdmi = true;
		3979	DRM_DEBUG("IH: HDMI1\n");
		3980	}
		3981	break;
		3982	default:
		3983	DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
		3984	break;
		3985	}
2004	serge	3986	break;
		3987	case 176: /* CP_INT in ring buffer */
		3988	case 177: /* CP_INT in IB1 */
		3989	case 178: /* CP_INT in IB2 */
		3990	DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2997	Serge	3991	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2004	serge	3992	break;
		3993	case 181: /* CP EOP event */
		3994	DRM_DEBUG("IH: CP EOP\n");
2997	Serge	3995	radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
2004	serge	3996	break;
3192	Serge	3997	case 224: /* DMA trap event */
		3998	DRM_DEBUG("IH: DMA trap\n");
		3999	radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
		4000	break;
2004	serge	4001	case 233: /* GUI IDLE */
		4002	DRM_DEBUG("IH: GUI idle\n");
		4003	break;
		4004	default:
		4005	DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
		4006	break;
		4007	}
		4008
		4009	/* wptr/rptr are in bytes! */
		4010	rptr += 16;
		4011	rptr &= rdev->ih.ptr_mask;
		4012	}
2997	Serge	4013	rdev->ih.rptr = rptr;
		4014	WREG32(IH_RB_RPTR, rdev->ih.rptr);
		4015	atomic_set(&rdev->ih.lock, 0);
		4016
2004	serge	4017	/* make sure wptr hasn't changed while processing */
		4018	wptr = r600_get_ih_wptr(rdev);
2997	Serge	4019	if (wptr != rptr)
2004	serge	4020	goto restart_ih;
2997	Serge	4021
2004	serge	4022	return IRQ_HANDLED;
		4023	}
		4024
1221	serge	4025	/*
		4026	* Debugfs info
		4027	*/
		4028	#if defined(CONFIG_DEBUG_FS)
		4029
		4030	static int r600_debugfs_mc_info(struct seq_file m, void data)
		4031	{
		4032	struct drm_info_node node = (struct drm_info_node ) m->private;
		4033	struct drm_device *dev = node->minor->dev;
		4034	struct radeon_device *rdev = dev->dev_private;
		4035
		4036	DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
		4037	DREG32_SYS(m, rdev, VM_L2_STATUS);
		4038	return 0;
		4039	}
		4040
		4041	static struct drm_info_list r600_mc_info_list[] = {
		4042	{"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
		4043	};
		4044	#endif
		4045
		4046	int r600_debugfs_mc_info_init(struct radeon_device *rdev)
		4047	{
		4048	#if defined(CONFIG_DEBUG_FS)
		4049	return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
		4050	#else
		4051	return 0;
		4052	#endif
		4053	}
1404	serge	4054
		4055	/**
		4056	* r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
		4057	* rdev: radeon device structure
		4058	* bo: buffer object struct which userspace is waiting for idle
		4059	*
		4060	* Some R6XX/R7XX doesn't seems to take into account HDP flush performed
		4061	* through ring buffer, this leads to corruption in rendering, see
		4062	* http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
		4063	* directly perform HDP flush by writing register through MMIO.
		4064	*/
		4065	void r600_ioctl_wait_idle(struct radeon_device rdev, struct radeon_bo bo)
		4066	{
1963	serge	4067	/* r7xx hw bug. write to HDP_DEBUG1 followed by fb read
		4068	* rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
		4069	* This seems to cause problems on some AGP cards. Just use the old
		4070	* method for them.
		4071	*/
		4072	if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
		4073	rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
		4074	void __iomem ptr = (void )rdev->vram_scratch.ptr;
		4075	u32 tmp;
		4076
		4077	WREG32(HDP_DEBUG1, 0);
		4078	tmp = readl((void __iomem *)ptr);
		4079	} else
1404	serge	4080	WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
		4081	}
1963	serge	4082
		4083	void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
		4084	{
3764	Serge	4085	u32 link_width_cntl, mask;
1963	serge	4086
		4087	if (rdev->flags & RADEON_IS_IGP)
		4088	return;
		4089
		4090	if (!(rdev->flags & RADEON_IS_PCIE))
		4091	return;
		4092
		4093	/* x2 cards have a special sequence */
		4094	if (ASIC_IS_X2(rdev))
		4095	return;
		4096
3764	Serge	4097	radeon_gui_idle(rdev);
1963	serge	4098
		4099	switch (lanes) {
		4100	case 0:
		4101	mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
		4102	break;
		4103	case 1:
		4104	mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
		4105	break;
		4106	case 2:
		4107	mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
		4108	break;
		4109	case 4:
		4110	mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
		4111	break;
		4112	case 8:
		4113	mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
		4114	break;
		4115	case 12:
3764	Serge	4116	/* not actually supported */
1963	serge	4117	mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
		4118	break;
		4119	case 16:
		4120	mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
		4121	break;
3764	Serge	4122	default:
		4123	DRM_ERROR("invalid pcie lane request: %d\n", lanes);
		4124	return;
1963	serge	4125	}
		4126
3764	Serge	4127	link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
		4128	link_width_cntl &= ~RADEON_PCIE_LC_LINK_WIDTH_MASK;
		4129	link_width_cntl \|= mask << RADEON_PCIE_LC_LINK_WIDTH_SHIFT;
		4130	link_width_cntl \|= (RADEON_PCIE_LC_RECONFIG_NOW \|
1963	serge	4131	R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
		4132
3764	Serge	4133	WREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1963	serge	4134	}
		4135
		4136	int r600_get_pcie_lanes(struct radeon_device *rdev)
		4137	{
		4138	u32 link_width_cntl;
		4139
		4140	if (rdev->flags & RADEON_IS_IGP)
		4141	return 0;
		4142
		4143	if (!(rdev->flags & RADEON_IS_PCIE))
		4144	return 0;
		4145
		4146	/* x2 cards have a special sequence */
		4147	if (ASIC_IS_X2(rdev))
		4148	return 0;
		4149
3764	Serge	4150	radeon_gui_idle(rdev);
1963	serge	4151
3764	Serge	4152	link_width_cntl = RREG32_PCIE_PORT(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
1963	serge	4153
		4154	switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
		4155	case RADEON_PCIE_LC_LINK_WIDTH_X1:
		4156	return 1;
		4157	case RADEON_PCIE_LC_LINK_WIDTH_X2:
		4158	return 2;
		4159	case RADEON_PCIE_LC_LINK_WIDTH_X4:
		4160	return 4;
		4161	case RADEON_PCIE_LC_LINK_WIDTH_X8:
		4162	return 8;
3764	Serge	4163	case RADEON_PCIE_LC_LINK_WIDTH_X12:
		4164	/* not actually supported */
		4165	return 12;
		4166	case RADEON_PCIE_LC_LINK_WIDTH_X0:
1963	serge	4167	case RADEON_PCIE_LC_LINK_WIDTH_X16:
		4168	default:
		4169	return 16;
		4170	}
		4171	}
		4172
		4173	static void r600_pcie_gen2_enable(struct radeon_device *rdev)
		4174	{
		4175	u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
		4176	u16 link_cntl2;
		4177
		4178	if (radeon_pcie_gen2 == 0)
		4179	return;
		4180
		4181	if (rdev->flags & RADEON_IS_IGP)
		4182	return;
		4183
		4184	if (!(rdev->flags & RADEON_IS_PCIE))
		4185	return;
		4186
		4187	/* x2 cards have a special sequence */
		4188	if (ASIC_IS_X2(rdev))
		4189	return;
		4190
		4191	/* only RV6xx+ chips are supported */
		4192	if (rdev->family <= CHIP_R600)
		4193	return;
		4194
3764	Serge	4195	if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
		4196	(rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
2997	Serge	4197	return;
		4198
3764	Serge	4199	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
2997	Serge	4200	if (speed_cntl & LC_CURRENT_DATA_RATE) {
		4201	DRM_INFO("PCIE gen 2 link speeds already enabled\n");
		4202	return;
		4203	}
		4204
		4205	DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
		4206
1963	serge	4207	/* 55 nm r6xx asics */
		4208	if ((rdev->family == CHIP_RV670) \|\|
		4209	(rdev->family == CHIP_RV620) \|\|
		4210	(rdev->family == CHIP_RV635)) {
		4211	/* advertise upconfig capability */
3764	Serge	4212	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1963	serge	4213	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3764	Serge	4214	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
		4215	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1963	serge	4216	if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
		4217	lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
		4218	link_width_cntl &= ~(LC_LINK_WIDTH_MASK \|
		4219	LC_RECONFIG_ARC_MISSING_ESCAPE);
		4220	link_width_cntl \|= lanes \| LC_RECONFIG_NOW \| LC_RENEGOTIATE_EN;
3764	Serge	4221	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1963	serge	4222	} else {
		4223	link_width_cntl \|= LC_UPCONFIGURE_DIS;
3764	Serge	4224	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1963	serge	4225	}
		4226	}
		4227
3764	Serge	4228	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1963	serge	4229	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
		4230	(speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
		4231
		4232	/* 55 nm r6xx asics */
		4233	if ((rdev->family == CHIP_RV670) \|\|
		4234	(rdev->family == CHIP_RV620) \|\|
		4235	(rdev->family == CHIP_RV635)) {
		4236	WREG32(MM_CFGREGS_CNTL, 0x8);
		4237	link_cntl2 = RREG32(0x4088);
		4238	WREG32(MM_CFGREGS_CNTL, 0);
		4239	/* not supported yet */
		4240	if (link_cntl2 & SELECTABLE_DEEMPHASIS)
		4241	return;
		4242	}
		4243
		4244	speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
		4245	speed_cntl \|= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
		4246	speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
		4247	speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
		4248	speed_cntl \|= LC_FORCE_EN_HW_SPEED_CHANGE;
3764	Serge	4249	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1963	serge	4250
		4251	tmp = RREG32(0x541c);
		4252	WREG32(0x541c, tmp \| 0x8);
		4253	WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
		4254	link_cntl2 = RREG16(0x4088);
		4255	link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
		4256	link_cntl2 \|= 0x2;
		4257	WREG16(0x4088, link_cntl2);
		4258	WREG32(MM_CFGREGS_CNTL, 0);
		4259
		4260	if ((rdev->family == CHIP_RV670) \|\|
		4261	(rdev->family == CHIP_RV620) \|\|
		4262	(rdev->family == CHIP_RV635)) {
3764	Serge	4263	training_cntl = RREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL);
1963	serge	4264	training_cntl &= ~LC_POINT_7_PLUS_EN;
3764	Serge	4265	WREG32_PCIE_PORT(PCIE_LC_TRAINING_CNTL, training_cntl);
1963	serge	4266	} else {
3764	Serge	4267	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1963	serge	4268	speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3764	Serge	4269	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1963	serge	4270	}
		4271
3764	Serge	4272	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1963	serge	4273	speed_cntl \|= LC_GEN2_EN_STRAP;
3764	Serge	4274	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1963	serge	4275
		4276	} else {
3764	Serge	4277	link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1963	serge	4278	/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
		4279	if (1)
		4280	link_width_cntl \|= LC_UPCONFIGURE_DIS;
		4281	else
		4282	link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3764	Serge	4283	WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1963	serge	4284	}
		4285	}
2997	Serge	4286
		4287	/**
3764	Serge	4288	* r600_get_gpu_clock_counter - return GPU clock counter snapshot
2997	Serge	4289	*
		4290	* @rdev: radeon_device pointer
		4291	*
		4292	* Fetches a GPU clock counter snapshot (R6xx-cayman).
		4293	* Returns the 64 bit clock counter snapshot.
		4294	*/
3764	Serge	4295	uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)
2997	Serge	4296	{
		4297	uint64_t clock;
		4298
		4299	mutex_lock(&rdev->gpu_clock_mutex);
		4300	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
		4301	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) \|
		4302	((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
		4303	mutex_unlock(&rdev->gpu_clock_mutex);
		4304	return clock;
		4305	}

Subversion Repositories Kolibri OS

(root)/drivers/video/drm/radeon/r600.c – Rev 3764