Subversion Repositories Kolibri OS

Rev

Rev 2004 | Rev 2160 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
1430 serge 1
/*
2
 * Copyright 2010 Advanced Micro Devices, Inc.
3
 *
4
 * Permission is hereby granted, free of charge, to any person obtaining a
5
 * copy of this software and associated documentation files (the "Software"),
6
 * to deal in the Software without restriction, including without limitation
7
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
 * and/or sell copies of the Software, and to permit persons to whom the
9
 * Software is furnished to do so, subject to the following conditions:
10
 *
11
 * The above copyright notice and this permission notice shall be included in
12
 * all copies or substantial portions of the Software.
13
 *
14
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20
 * OTHER DEALINGS IN THE SOFTWARE.
21
 *
22
 * Authors: Alex Deucher
23
 */
24
#include 
25
//#include 
1963 serge 26
#include 
1430 serge 27
#include "drmP.h"
28
#include "radeon.h"
1963 serge 29
#include "radeon_asic.h"
1430 serge 30
#include "radeon_drm.h"
1963 serge 31
#include "evergreend.h"
1430 serge 32
#include "atom.h"
33
#include "avivod.h"
34
#include "evergreen_reg.h"
1986 serge 35
#include "evergreen_blit_shaders.h"
1430 serge 36
 
1963 serge 37
#define EVERGREEN_PFP_UCODE_SIZE 1120
38
#define EVERGREEN_PM4_UCODE_SIZE 1376
39
 
1430 serge 40
static void evergreen_gpu_init(struct radeon_device *rdev);
41
void evergreen_fini(struct radeon_device *rdev);
1986 serge 42
static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
1430 serge 43
 
1990 serge 44
 
45
 
46
u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
47
{
48
	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
49
	u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
50
 
51
	/* Lock the graphics update lock */
52
	tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
53
	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
54
 
55
	/* update the scanout addresses */
56
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
57
	       upper_32_bits(crtc_base));
58
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
59
	       (u32)crtc_base);
60
 
61
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
62
	       upper_32_bits(crtc_base));
63
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
64
	       (u32)crtc_base);
65
 
66
	/* Wait for update_pending to go high. */
67
	while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING));
68
	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
69
 
70
	/* Unlock the lock, so double-buffering can take place inside vblank */
71
	tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
72
	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
73
 
74
	/* Return current update_pending status: */
75
	return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
76
}
77
 
78
/* get temperature in millidegrees */
79
int evergreen_get_temp(struct radeon_device *rdev)
80
{
81
	u32 temp, toffset;
82
	int actual_temp = 0;
83
 
84
	if (rdev->family == CHIP_JUNIPER) {
85
		toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
86
			TOFFSET_SHIFT;
87
		temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
88
			TS0_ADC_DOUT_SHIFT;
89
 
90
		if (toffset & 0x100)
91
			actual_temp = temp / 2 - (0x200 - toffset);
92
		else
93
			actual_temp = temp / 2 + toffset;
94
 
95
		actual_temp = actual_temp * 1000;
96
 
97
	} else {
98
		temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
99
			ASIC_T_SHIFT;
100
 
101
		if (temp & 0x400)
102
			actual_temp = -256;
103
		else if (temp & 0x200)
104
			actual_temp = 255;
105
		else if (temp & 0x100) {
106
			actual_temp = temp & 0x1ff;
107
			actual_temp |= ~0x1ff;
108
		} else
109
			actual_temp = temp & 0xff;
110
 
111
		actual_temp = (actual_temp * 1000) / 2;
112
	}
113
 
114
	return actual_temp;
115
}
116
 
117
int sumo_get_temp(struct radeon_device *rdev)
118
{
119
	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
120
	int actual_temp = temp - 49;
121
 
122
	return actual_temp * 1000;
123
}
124
 
125
void evergreen_pm_misc(struct radeon_device *rdev)
126
{
127
	int req_ps_idx = rdev->pm.requested_power_state_index;
128
	int req_cm_idx = rdev->pm.requested_clock_mode_index;
129
	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
130
	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
131
 
132
	if (voltage->type == VOLTAGE_SW) {
133
		/* 0xff01 is a flag rather then an actual voltage */
134
		if (voltage->voltage == 0xff01)
135
			return;
136
		if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
137
			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
138
			rdev->pm.current_vddc = voltage->voltage;
139
			DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
140
		}
141
		/* 0xff01 is a flag rather then an actual voltage */
142
		if (voltage->vddci == 0xff01)
143
			return;
144
		if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
145
			radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
146
			rdev->pm.current_vddci = voltage->vddci;
147
			DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
148
		}
149
	}
150
}
151
 
152
void evergreen_pm_prepare(struct radeon_device *rdev)
153
{
154
	struct drm_device *ddev = rdev->ddev;
155
	struct drm_crtc *crtc;
156
	struct radeon_crtc *radeon_crtc;
157
	u32 tmp;
158
 
159
	/* disable any active CRTCs */
160
	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
161
		radeon_crtc = to_radeon_crtc(crtc);
162
		if (radeon_crtc->enabled) {
163
			tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
164
			tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
165
			WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
166
		}
167
	}
168
}
169
 
170
void evergreen_pm_finish(struct radeon_device *rdev)
171
{
172
	struct drm_device *ddev = rdev->ddev;
173
	struct drm_crtc *crtc;
174
	struct radeon_crtc *radeon_crtc;
175
	u32 tmp;
176
 
177
	/* enable any active CRTCs */
178
	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
179
		radeon_crtc = to_radeon_crtc(crtc);
180
		if (radeon_crtc->enabled) {
181
			tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
182
			tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
183
			WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
184
		}
185
	}
186
}
187
 
1430 serge 188
bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
189
{
190
	bool connected = false;
1963 serge 191
 
192
	switch (hpd) {
193
	case RADEON_HPD_1:
194
		if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
195
			connected = true;
196
		break;
197
	case RADEON_HPD_2:
198
		if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
199
			connected = true;
200
		break;
201
	case RADEON_HPD_3:
202
		if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
203
			connected = true;
204
		break;
205
	case RADEON_HPD_4:
206
		if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
207
			connected = true;
208
		break;
209
	case RADEON_HPD_5:
210
		if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
211
			connected = true;
212
		break;
213
	case RADEON_HPD_6:
214
		if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
215
			connected = true;
216
			break;
217
	default:
218
		break;
219
	}
220
 
1430 serge 221
	return connected;
222
}
223
 
224
void evergreen_hpd_set_polarity(struct radeon_device *rdev,
225
				enum radeon_hpd_id hpd)
226
{
1963 serge 227
	u32 tmp;
228
	bool connected = evergreen_hpd_sense(rdev, hpd);
229
 
230
	switch (hpd) {
231
	case RADEON_HPD_1:
232
		tmp = RREG32(DC_HPD1_INT_CONTROL);
233
		if (connected)
234
			tmp &= ~DC_HPDx_INT_POLARITY;
235
		else
236
			tmp |= DC_HPDx_INT_POLARITY;
237
		WREG32(DC_HPD1_INT_CONTROL, tmp);
238
		break;
239
	case RADEON_HPD_2:
240
		tmp = RREG32(DC_HPD2_INT_CONTROL);
241
		if (connected)
242
			tmp &= ~DC_HPDx_INT_POLARITY;
243
		else
244
			tmp |= DC_HPDx_INT_POLARITY;
245
		WREG32(DC_HPD2_INT_CONTROL, tmp);
246
		break;
247
	case RADEON_HPD_3:
248
		tmp = RREG32(DC_HPD3_INT_CONTROL);
249
		if (connected)
250
			tmp &= ~DC_HPDx_INT_POLARITY;
251
		else
252
			tmp |= DC_HPDx_INT_POLARITY;
253
		WREG32(DC_HPD3_INT_CONTROL, tmp);
254
		break;
255
	case RADEON_HPD_4:
256
		tmp = RREG32(DC_HPD4_INT_CONTROL);
257
		if (connected)
258
			tmp &= ~DC_HPDx_INT_POLARITY;
259
		else
260
			tmp |= DC_HPDx_INT_POLARITY;
261
		WREG32(DC_HPD4_INT_CONTROL, tmp);
262
		break;
263
	case RADEON_HPD_5:
264
		tmp = RREG32(DC_HPD5_INT_CONTROL);
265
		if (connected)
266
			tmp &= ~DC_HPDx_INT_POLARITY;
267
		else
268
			tmp |= DC_HPDx_INT_POLARITY;
269
		WREG32(DC_HPD5_INT_CONTROL, tmp);
270
			break;
271
	case RADEON_HPD_6:
272
		tmp = RREG32(DC_HPD6_INT_CONTROL);
273
		if (connected)
274
			tmp &= ~DC_HPDx_INT_POLARITY;
275
		else
276
			tmp |= DC_HPDx_INT_POLARITY;
277
		WREG32(DC_HPD6_INT_CONTROL, tmp);
278
		break;
279
	default:
280
		break;
281
	}
1430 serge 282
}
283
 
284
void evergreen_hpd_init(struct radeon_device *rdev)
285
{
1963 serge 286
	struct drm_device *dev = rdev->ddev;
287
	struct drm_connector *connector;
288
	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
289
		DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
290
 
291
	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
292
		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
293
		switch (radeon_connector->hpd.hpd) {
294
		case RADEON_HPD_1:
295
			WREG32(DC_HPD1_CONTROL, tmp);
296
			rdev->irq.hpd[0] = true;
297
			break;
298
		case RADEON_HPD_2:
299
			WREG32(DC_HPD2_CONTROL, tmp);
300
			rdev->irq.hpd[1] = true;
301
			break;
302
		case RADEON_HPD_3:
303
			WREG32(DC_HPD3_CONTROL, tmp);
304
			rdev->irq.hpd[2] = true;
305
			break;
306
		case RADEON_HPD_4:
307
			WREG32(DC_HPD4_CONTROL, tmp);
308
			rdev->irq.hpd[3] = true;
309
			break;
310
		case RADEON_HPD_5:
311
			WREG32(DC_HPD5_CONTROL, tmp);
312
			rdev->irq.hpd[4] = true;
313
			break;
314
		case RADEON_HPD_6:
315
			WREG32(DC_HPD6_CONTROL, tmp);
316
			rdev->irq.hpd[5] = true;
317
			break;
318
		default:
319
			break;
320
		}
321
	}
2005 serge 322
	if (rdev->irq.installed)
323
		evergreen_irq_set(rdev);
1430 serge 324
}
325
 
1963 serge 326
void evergreen_hpd_fini(struct radeon_device *rdev)
327
{
328
	struct drm_device *dev = rdev->ddev;
329
	struct drm_connector *connector;
1430 serge 330
 
1963 serge 331
	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
332
		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
333
		switch (radeon_connector->hpd.hpd) {
334
		case RADEON_HPD_1:
335
			WREG32(DC_HPD1_CONTROL, 0);
336
			rdev->irq.hpd[0] = false;
337
			break;
338
		case RADEON_HPD_2:
339
			WREG32(DC_HPD2_CONTROL, 0);
340
			rdev->irq.hpd[1] = false;
341
			break;
342
		case RADEON_HPD_3:
343
			WREG32(DC_HPD3_CONTROL, 0);
344
			rdev->irq.hpd[2] = false;
345
			break;
346
		case RADEON_HPD_4:
347
			WREG32(DC_HPD4_CONTROL, 0);
348
			rdev->irq.hpd[3] = false;
349
			break;
350
		case RADEON_HPD_5:
351
			WREG32(DC_HPD5_CONTROL, 0);
352
			rdev->irq.hpd[4] = false;
353
			break;
354
		case RADEON_HPD_6:
355
			WREG32(DC_HPD6_CONTROL, 0);
356
			rdev->irq.hpd[5] = false;
357
			break;
358
		default:
359
			break;
360
		}
361
	}
1430 serge 362
}
363
 
1986 serge 364
/* watermark setup */
1963 serge 365
 
1986 serge 366
static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
367
					struct radeon_crtc *radeon_crtc,
368
					struct drm_display_mode *mode,
369
					struct drm_display_mode *other_mode)
370
{
371
	u32 tmp;
372
	/*
373
	 * Line Buffer Setup
374
	 * There are 3 line buffers, each one shared by 2 display controllers.
375
	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
376
	 * the display controllers.  The paritioning is done via one of four
377
	 * preset allocations specified in bits 2:0:
378
	 * first display controller
379
	 *  0 - first half of lb (3840 * 2)
380
	 *  1 - first 3/4 of lb (5760 * 2)
381
	 *  2 - whole lb (7680 * 2), other crtc must be disabled
382
	 *  3 - first 1/4 of lb (1920 * 2)
383
	 * second display controller
384
	 *  4 - second half of lb (3840 * 2)
385
	 *  5 - second 3/4 of lb (5760 * 2)
386
	 *  6 - whole lb (7680 * 2), other crtc must be disabled
387
	 *  7 - last 1/4 of lb (1920 * 2)
388
	 */
389
	/* this can get tricky if we have two large displays on a paired group
390
	 * of crtcs.  Ideally for multiple large displays we'd assign them to
391
	 * non-linked crtcs for maximum line buffer allocation.
392
	 */
393
	if (radeon_crtc->base.enabled && mode) {
394
		if (other_mode)
395
			tmp = 0; /* 1/2 */
396
		else
397
			tmp = 2; /* whole */
398
	} else
399
		tmp = 0;
1963 serge 400
 
1986 serge 401
	/* second controller of the pair uses second half of the lb */
402
	if (radeon_crtc->crtc_id % 2)
403
		tmp += 4;
404
	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
405
 
406
	if (radeon_crtc->base.enabled && mode) {
407
		switch (tmp) {
408
		case 0:
409
		case 4:
410
		default:
411
			if (ASIC_IS_DCE5(rdev))
412
				return 4096 * 2;
413
			else
414
				return 3840 * 2;
415
		case 1:
416
		case 5:
417
			if (ASIC_IS_DCE5(rdev))
418
				return 6144 * 2;
419
			else
420
				return 5760 * 2;
421
		case 2:
422
		case 6:
423
			if (ASIC_IS_DCE5(rdev))
424
				return 8192 * 2;
425
			else
426
				return 7680 * 2;
427
		case 3:
428
		case 7:
429
			if (ASIC_IS_DCE5(rdev))
430
				return 2048 * 2;
431
			else
432
				return 1920 * 2;
433
		}
434
	}
435
 
436
	/* controller not enabled, so no lb used */
437
	return 0;
438
}
439
 
440
static u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
441
{
442
	u32 tmp = RREG32(MC_SHARED_CHMAP);
443
 
444
	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
445
	case 0:
446
	default:
447
		return 1;
448
	case 1:
449
		return 2;
450
	case 2:
451
		return 4;
452
	case 3:
453
		return 8;
454
	}
455
}
456
 
457
struct evergreen_wm_params {
458
	u32 dram_channels; /* number of dram channels */
459
	u32 yclk;          /* bandwidth per dram data pin in kHz */
460
	u32 sclk;          /* engine clock in kHz */
461
	u32 disp_clk;      /* display clock in kHz */
462
	u32 src_width;     /* viewport width */
463
	u32 active_time;   /* active display time in ns */
464
	u32 blank_time;    /* blank time in ns */
465
	bool interlaced;    /* mode is interlaced */
466
	fixed20_12 vsc;    /* vertical scale ratio */
467
	u32 num_heads;     /* number of active crtcs */
468
	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
469
	u32 lb_size;       /* line buffer allocated to pipe */
470
	u32 vtaps;         /* vertical scaler taps */
471
};
472
 
473
static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
474
{
475
	/* Calculate DRAM Bandwidth and the part allocated to display. */
476
	fixed20_12 dram_efficiency; /* 0.7 */
477
	fixed20_12 yclk, dram_channels, bandwidth;
478
	fixed20_12 a;
479
 
480
	a.full = dfixed_const(1000);
481
	yclk.full = dfixed_const(wm->yclk);
482
	yclk.full = dfixed_div(yclk, a);
483
	dram_channels.full = dfixed_const(wm->dram_channels * 4);
484
	a.full = dfixed_const(10);
485
	dram_efficiency.full = dfixed_const(7);
486
	dram_efficiency.full = dfixed_div(dram_efficiency, a);
487
	bandwidth.full = dfixed_mul(dram_channels, yclk);
488
	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
489
 
490
	return dfixed_trunc(bandwidth);
491
}
492
 
493
static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
494
{
495
	/* Calculate DRAM Bandwidth and the part allocated to display. */
496
	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
497
	fixed20_12 yclk, dram_channels, bandwidth;
498
	fixed20_12 a;
499
 
500
	a.full = dfixed_const(1000);
501
	yclk.full = dfixed_const(wm->yclk);
502
	yclk.full = dfixed_div(yclk, a);
503
	dram_channels.full = dfixed_const(wm->dram_channels * 4);
504
	a.full = dfixed_const(10);
505
	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
506
	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
507
	bandwidth.full = dfixed_mul(dram_channels, yclk);
508
	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
509
 
510
	return dfixed_trunc(bandwidth);
511
}
512
 
513
static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
514
{
515
	/* Calculate the display Data return Bandwidth */
516
	fixed20_12 return_efficiency; /* 0.8 */
517
	fixed20_12 sclk, bandwidth;
518
	fixed20_12 a;
519
 
520
	a.full = dfixed_const(1000);
521
	sclk.full = dfixed_const(wm->sclk);
522
	sclk.full = dfixed_div(sclk, a);
523
	a.full = dfixed_const(10);
524
	return_efficiency.full = dfixed_const(8);
525
	return_efficiency.full = dfixed_div(return_efficiency, a);
526
	a.full = dfixed_const(32);
527
	bandwidth.full = dfixed_mul(a, sclk);
528
	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
529
 
530
	return dfixed_trunc(bandwidth);
531
}
532
 
533
static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
534
{
535
	/* Calculate the DMIF Request Bandwidth */
536
	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
537
	fixed20_12 disp_clk, bandwidth;
538
	fixed20_12 a;
539
 
540
	a.full = dfixed_const(1000);
541
	disp_clk.full = dfixed_const(wm->disp_clk);
542
	disp_clk.full = dfixed_div(disp_clk, a);
543
	a.full = dfixed_const(10);
544
	disp_clk_request_efficiency.full = dfixed_const(8);
545
	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
546
	a.full = dfixed_const(32);
547
	bandwidth.full = dfixed_mul(a, disp_clk);
548
	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
549
 
550
	return dfixed_trunc(bandwidth);
551
}
552
 
553
static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
554
{
555
	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
556
	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
557
	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
558
	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
559
 
560
	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
561
}
562
 
563
static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
564
{
565
	/* Calculate the display mode Average Bandwidth
566
	 * DisplayMode should contain the source and destination dimensions,
567
	 * timing, etc.
568
	 */
569
	fixed20_12 bpp;
570
	fixed20_12 line_time;
571
	fixed20_12 src_width;
572
	fixed20_12 bandwidth;
573
	fixed20_12 a;
574
 
575
	a.full = dfixed_const(1000);
576
	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
577
	line_time.full = dfixed_div(line_time, a);
578
	bpp.full = dfixed_const(wm->bytes_per_pixel);
579
	src_width.full = dfixed_const(wm->src_width);
580
	bandwidth.full = dfixed_mul(src_width, bpp);
581
	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
582
	bandwidth.full = dfixed_div(bandwidth, line_time);
583
 
584
	return dfixed_trunc(bandwidth);
585
}
586
 
587
static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
588
{
589
	/* First calcualte the latency in ns */
590
	u32 mc_latency = 2000; /* 2000 ns. */
591
	u32 available_bandwidth = evergreen_available_bandwidth(wm);
592
	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
593
	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
594
	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
595
	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
596
		(wm->num_heads * cursor_line_pair_return_time);
597
	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
598
	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
599
	fixed20_12 a, b, c;
600
 
601
	if (wm->num_heads == 0)
602
		return 0;
603
 
604
	a.full = dfixed_const(2);
605
	b.full = dfixed_const(1);
606
	if ((wm->vsc.full > a.full) ||
607
	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
608
	    (wm->vtaps >= 5) ||
609
	    ((wm->vsc.full >= a.full) && wm->interlaced))
610
		max_src_lines_per_dst_line = 4;
611
	else
612
		max_src_lines_per_dst_line = 2;
613
 
614
	a.full = dfixed_const(available_bandwidth);
615
	b.full = dfixed_const(wm->num_heads);
616
	a.full = dfixed_div(a, b);
617
 
618
	b.full = dfixed_const(1000);
619
	c.full = dfixed_const(wm->disp_clk);
620
	b.full = dfixed_div(c, b);
621
	c.full = dfixed_const(wm->bytes_per_pixel);
622
	b.full = dfixed_mul(b, c);
623
 
624
	lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
625
 
626
	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
627
	b.full = dfixed_const(1000);
628
	c.full = dfixed_const(lb_fill_bw);
629
	b.full = dfixed_div(c, b);
630
	a.full = dfixed_div(a, b);
631
	line_fill_time = dfixed_trunc(a);
632
 
633
	if (line_fill_time < wm->active_time)
634
		return latency;
635
	else
636
		return latency + (line_fill_time - wm->active_time);
637
 
638
}
639
 
640
static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
641
{
642
	if (evergreen_average_bandwidth(wm) <=
643
	    (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
644
		return true;
645
	else
646
		return false;
647
};
648
 
649
static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
650
{
651
	if (evergreen_average_bandwidth(wm) <=
652
	    (evergreen_available_bandwidth(wm) / wm->num_heads))
653
		return true;
654
	else
655
		return false;
656
};
657
 
658
static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
659
{
660
	u32 lb_partitions = wm->lb_size / wm->src_width;
661
	u32 line_time = wm->active_time + wm->blank_time;
662
	u32 latency_tolerant_lines;
663
	u32 latency_hiding;
664
	fixed20_12 a;
665
 
666
	a.full = dfixed_const(1);
667
	if (wm->vsc.full > a.full)
668
		latency_tolerant_lines = 1;
669
	else {
670
		if (lb_partitions <= (wm->vtaps + 1))
671
			latency_tolerant_lines = 1;
672
		else
673
			latency_tolerant_lines = 2;
674
	}
675
 
676
	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
677
 
678
	if (evergreen_latency_watermark(wm) <= latency_hiding)
679
		return true;
680
	else
681
		return false;
682
}
683
 
684
static void evergreen_program_watermarks(struct radeon_device *rdev,
685
					 struct radeon_crtc *radeon_crtc,
686
					 u32 lb_size, u32 num_heads)
687
{
688
	struct drm_display_mode *mode = &radeon_crtc->base.mode;
689
	struct evergreen_wm_params wm;
690
	u32 pixel_period;
691
	u32 line_time = 0;
692
	u32 latency_watermark_a = 0, latency_watermark_b = 0;
693
	u32 priority_a_mark = 0, priority_b_mark = 0;
694
	u32 priority_a_cnt = PRIORITY_OFF;
695
	u32 priority_b_cnt = PRIORITY_OFF;
696
	u32 pipe_offset = radeon_crtc->crtc_id * 16;
697
	u32 tmp, arb_control3;
698
	fixed20_12 a, b, c;
699
 
700
	if (radeon_crtc->base.enabled && num_heads && mode) {
701
		pixel_period = 1000000 / (u32)mode->clock;
702
		line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
703
		priority_a_cnt = 0;
704
		priority_b_cnt = 0;
705
 
706
		wm.yclk = rdev->pm.current_mclk * 10;
707
		wm.sclk = rdev->pm.current_sclk * 10;
708
		wm.disp_clk = mode->clock;
709
		wm.src_width = mode->crtc_hdisplay;
710
		wm.active_time = mode->crtc_hdisplay * pixel_period;
711
		wm.blank_time = line_time - wm.active_time;
712
		wm.interlaced = false;
713
		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
714
			wm.interlaced = true;
715
		wm.vsc = radeon_crtc->vsc;
716
		wm.vtaps = 1;
717
		if (radeon_crtc->rmx_type != RMX_OFF)
718
			wm.vtaps = 2;
719
		wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
720
		wm.lb_size = lb_size;
721
		wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
722
		wm.num_heads = num_heads;
723
 
724
		/* set for high clocks */
725
		latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
726
		/* set for low clocks */
727
		/* wm.yclk = low clk; wm.sclk = low clk */
728
		latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
729
 
730
		/* possibly force display priority to high */
731
		/* should really do this at mode validation time... */
732
		if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
733
		    !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
734
		    !evergreen_check_latency_hiding(&wm) ||
735
		    (rdev->disp_priority == 2)) {
736
			DRM_INFO("force priority to high\n");
737
			priority_a_cnt |= PRIORITY_ALWAYS_ON;
738
			priority_b_cnt |= PRIORITY_ALWAYS_ON;
739
		}
740
 
741
		a.full = dfixed_const(1000);
742
		b.full = dfixed_const(mode->clock);
743
		b.full = dfixed_div(b, a);
744
		c.full = dfixed_const(latency_watermark_a);
745
		c.full = dfixed_mul(c, b);
746
		c.full = dfixed_mul(c, radeon_crtc->hsc);
747
		c.full = dfixed_div(c, a);
748
		a.full = dfixed_const(16);
749
		c.full = dfixed_div(c, a);
750
		priority_a_mark = dfixed_trunc(c);
751
		priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
752
 
753
		a.full = dfixed_const(1000);
754
		b.full = dfixed_const(mode->clock);
755
		b.full = dfixed_div(b, a);
756
		c.full = dfixed_const(latency_watermark_b);
757
		c.full = dfixed_mul(c, b);
758
		c.full = dfixed_mul(c, radeon_crtc->hsc);
759
		c.full = dfixed_div(c, a);
760
		a.full = dfixed_const(16);
761
		c.full = dfixed_div(c, a);
762
		priority_b_mark = dfixed_trunc(c);
763
		priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
764
	}
765
 
766
	/* select wm A */
767
	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
768
	tmp = arb_control3;
769
	tmp &= ~LATENCY_WATERMARK_MASK(3);
770
	tmp |= LATENCY_WATERMARK_MASK(1);
771
	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
772
	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
773
	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
774
		LATENCY_HIGH_WATERMARK(line_time)));
775
	/* select wm B */
776
	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
777
	tmp &= ~LATENCY_WATERMARK_MASK(3);
778
	tmp |= LATENCY_WATERMARK_MASK(2);
779
	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
780
	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
781
	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
782
		LATENCY_HIGH_WATERMARK(line_time)));
783
	/* restore original selection */
784
	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
785
 
786
	/* write the priority marks */
787
	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
788
	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
789
 
790
}
791
 
1963 serge 792
void evergreen_bandwidth_update(struct radeon_device *rdev)
1430 serge 793
{
1986 serge 794
	struct drm_display_mode *mode0 = NULL;
795
	struct drm_display_mode *mode1 = NULL;
796
	u32 num_heads = 0, lb_size;
797
	int i;
798
 
799
	radeon_update_display_priority(rdev);
800
 
801
	for (i = 0; i < rdev->num_crtc; i++) {
802
		if (rdev->mode_info.crtcs[i]->base.enabled)
803
			num_heads++;
804
	}
805
	for (i = 0; i < rdev->num_crtc; i += 2) {
806
		mode0 = &rdev->mode_info.crtcs[i]->base.mode;
807
		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
808
		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
809
		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
810
		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
811
		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
812
	}
1430 serge 813
}
814
 
1963 serge 815
int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1430 serge 816
{
817
	unsigned i;
818
	u32 tmp;
819
 
820
	for (i = 0; i < rdev->usec_timeout; i++) {
821
		/* read MC_STATUS */
822
		tmp = RREG32(SRBM_STATUS) & 0x1F00;
823
		if (!tmp)
824
			return 0;
825
		udelay(1);
826
	}
827
	return -1;
828
}
829
 
830
/*
831
 * GART
832
 */
1963 serge 833
void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
834
{
835
	unsigned i;
836
	u32 tmp;
837
 
838
	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
839
 
840
	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
841
	for (i = 0; i < rdev->usec_timeout; i++) {
842
		/* read MC_STATUS */
843
		tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
844
		tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
845
		if (tmp == 2) {
846
			printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
847
			return;
848
		}
849
		if (tmp) {
850
			return;
851
		}
852
		udelay(1);
853
	}
854
}
855
 
1430 serge 856
int evergreen_pcie_gart_enable(struct radeon_device *rdev)
857
{
858
	u32 tmp;
1963 serge 859
	int r;
1430 serge 860
 
861
	if (rdev->gart.table.vram.robj == NULL) {
862
		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
863
		return -EINVAL;
864
	}
865
	r = radeon_gart_table_vram_pin(rdev);
866
	if (r)
867
		return r;
868
	radeon_gart_restore(rdev);
869
	/* Setup L2 cache */
870
	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
871
				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
872
				EFFECTIVE_L2_QUEUE_SIZE(7));
873
	WREG32(VM_L2_CNTL2, 0);
874
	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
875
	/* Setup TLB control */
876
	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
877
		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
878
		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
879
		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1963 serge 880
	if (rdev->flags & RADEON_IS_IGP) {
881
		WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
882
		WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
883
		WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
884
	} else {
1430 serge 885
	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
886
	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
887
	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1963 serge 888
	}
1430 serge 889
	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
890
	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
891
	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
892
	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
893
	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
894
	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
895
	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
896
	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
897
				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
898
	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
899
			(u32)(rdev->dummy_page.addr >> 12));
1963 serge 900
	WREG32(VM_CONTEXT1_CNTL, 0);
1430 serge 901
 
1963 serge 902
	evergreen_pcie_gart_tlb_flush(rdev);
1430 serge 903
	rdev->gart.ready = true;
904
	return 0;
905
}
906
 
907
void evergreen_pcie_gart_disable(struct radeon_device *rdev)
908
{
909
	u32 tmp;
1963 serge 910
	int r;
1430 serge 911
 
912
	/* Disable all tables */
1963 serge 913
	WREG32(VM_CONTEXT0_CNTL, 0);
914
	WREG32(VM_CONTEXT1_CNTL, 0);
1430 serge 915
 
916
	/* Setup L2 cache */
917
	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
918
				EFFECTIVE_L2_QUEUE_SIZE(7));
919
	WREG32(VM_L2_CNTL2, 0);
920
	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
921
	/* Setup TLB control */
922
	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
923
	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
924
	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
925
	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
926
	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
927
	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
928
	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
929
	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
930
	if (rdev->gart.table.vram.robj) {
931
		r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
932
		if (likely(r == 0)) {
933
			radeon_bo_kunmap(rdev->gart.table.vram.robj);
934
			radeon_bo_unpin(rdev->gart.table.vram.robj);
935
			radeon_bo_unreserve(rdev->gart.table.vram.robj);
936
		}
937
	}
938
}
939
 
940
void evergreen_pcie_gart_fini(struct radeon_device *rdev)
941
{
942
	evergreen_pcie_gart_disable(rdev);
943
	radeon_gart_table_vram_free(rdev);
944
	radeon_gart_fini(rdev);
945
}
946
 
947
 
948
void evergreen_agp_enable(struct radeon_device *rdev)
949
{
950
	u32 tmp;
951
 
952
	/* Setup L2 cache */
953
	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
954
				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
955
				EFFECTIVE_L2_QUEUE_SIZE(7));
956
	WREG32(VM_L2_CNTL2, 0);
957
	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
958
	/* Setup TLB control */
959
	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
960
		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
961
		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
962
		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
963
	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
964
	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
965
	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
966
	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
967
	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
968
	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
969
	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1963 serge 970
	WREG32(VM_CONTEXT0_CNTL, 0);
971
	WREG32(VM_CONTEXT1_CNTL, 0);
1430 serge 972
}
973
 
1963 serge 974
void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
1430 serge 975
{
976
	save->vga_control[0] = RREG32(D1VGA_CONTROL);
977
	save->vga_control[1] = RREG32(D2VGA_CONTROL);
978
	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
979
	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
980
	save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
981
	save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
2005 serge 982
	if (rdev->num_crtc >= 4) {
983
		save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL);
984
		save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL);
1430 serge 985
	save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
986
	save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
2005 serge 987
	}
988
	if (rdev->num_crtc >= 6) {
989
		save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL);
990
		save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL);
1430 serge 991
	save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
992
	save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
1963 serge 993
	}
1430 serge 994
 
995
	/* Stop all video */
996
	WREG32(VGA_RENDER_CONTROL, 0);
997
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
998
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
2005 serge 999
	if (rdev->num_crtc >= 4) {
1430 serge 1000
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
1001
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
2005 serge 1002
	}
1003
	if (rdev->num_crtc >= 6) {
1430 serge 1004
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1005
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1963 serge 1006
	}
1430 serge 1007
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1008
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005 serge 1009
	if (rdev->num_crtc >= 4) {
1430 serge 1010
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1011
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005 serge 1012
	}
1013
	if (rdev->num_crtc >= 6) {
1430 serge 1014
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1015
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963 serge 1016
	}
1430 serge 1017
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1018
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005 serge 1019
	if (rdev->num_crtc >= 4) {
1430 serge 1020
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1021
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005 serge 1022
	}
1023
	if (rdev->num_crtc >= 6) {
1430 serge 1024
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1025
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963 serge 1026
	}
1430 serge 1027
 
1028
	WREG32(D1VGA_CONTROL, 0);
1029
	WREG32(D2VGA_CONTROL, 0);
2005 serge 1030
	if (rdev->num_crtc >= 4) {
1430 serge 1031
	WREG32(EVERGREEN_D3VGA_CONTROL, 0);
1032
	WREG32(EVERGREEN_D4VGA_CONTROL, 0);
2005 serge 1033
	}
1034
	if (rdev->num_crtc >= 6) {
1430 serge 1035
	WREG32(EVERGREEN_D5VGA_CONTROL, 0);
1036
	WREG32(EVERGREEN_D6VGA_CONTROL, 0);
2005 serge 1037
	}
1430 serge 1038
}
1039
 
1963 serge 1040
void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
1430 serge 1041
{
1042
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
1043
	       upper_32_bits(rdev->mc.vram_start));
1044
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
1045
	       upper_32_bits(rdev->mc.vram_start));
1046
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1047
	       (u32)rdev->mc.vram_start);
1048
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1049
	       (u32)rdev->mc.vram_start);
1050
 
1051
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
1052
	       upper_32_bits(rdev->mc.vram_start));
1053
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
1054
	       upper_32_bits(rdev->mc.vram_start));
1055
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1056
	       (u32)rdev->mc.vram_start);
1057
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1058
	       (u32)rdev->mc.vram_start);
1059
 
2005 serge 1060
	if (rdev->num_crtc >= 4) {
1430 serge 1061
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
1062
	       upper_32_bits(rdev->mc.vram_start));
1063
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
1064
	       upper_32_bits(rdev->mc.vram_start));
1065
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1066
	       (u32)rdev->mc.vram_start);
1067
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1068
	       (u32)rdev->mc.vram_start);
1069
 
1070
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
1071
	       upper_32_bits(rdev->mc.vram_start));
1072
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
1073
	       upper_32_bits(rdev->mc.vram_start));
1074
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1075
	       (u32)rdev->mc.vram_start);
1076
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1077
	       (u32)rdev->mc.vram_start);
2005 serge 1078
	}
1079
	if (rdev->num_crtc >= 6) {
1430 serge 1080
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
1081
	       upper_32_bits(rdev->mc.vram_start));
1082
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
1083
	       upper_32_bits(rdev->mc.vram_start));
1084
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1085
	       (u32)rdev->mc.vram_start);
1086
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1087
	       (u32)rdev->mc.vram_start);
1088
 
1089
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
1090
	       upper_32_bits(rdev->mc.vram_start));
1091
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
1092
	       upper_32_bits(rdev->mc.vram_start));
1093
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1094
	       (u32)rdev->mc.vram_start);
1095
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1096
	       (u32)rdev->mc.vram_start);
1963 serge 1097
	}
1430 serge 1098
 
1099
	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
1100
	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
1101
	/* Unlock host access */
1102
	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
1103
	mdelay(1);
1104
	/* Restore video state */
1105
	WREG32(D1VGA_CONTROL, save->vga_control[0]);
1106
	WREG32(D2VGA_CONTROL, save->vga_control[1]);
2005 serge 1107
	if (rdev->num_crtc >= 4) {
1430 serge 1108
	WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]);
1109
	WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]);
2005 serge 1110
	}
1111
	if (rdev->num_crtc >= 6) {
1430 serge 1112
	WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]);
1113
	WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
2005 serge 1114
	}
1430 serge 1115
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
1116
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
2005 serge 1117
	if (rdev->num_crtc >= 4) {
1430 serge 1118
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
1119
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
2005 serge 1120
	}
1121
	if (rdev->num_crtc >= 6) {
1430 serge 1122
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1123
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1963 serge 1124
	}
1430 serge 1125
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]);
1126
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]);
2005 serge 1127
	if (rdev->num_crtc >= 4) {
1430 serge 1128
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
1129
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
2005 serge 1130
	}
1131
	if (rdev->num_crtc >= 6) {
1430 serge 1132
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
1133
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
1963 serge 1134
	}
1430 serge 1135
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1136
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005 serge 1137
	if (rdev->num_crtc >= 4) {
1430 serge 1138
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1139
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005 serge 1140
	}
1141
	if (rdev->num_crtc >= 6) {
1430 serge 1142
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1143
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963 serge 1144
	}
1430 serge 1145
	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
1146
}
1147
 
1963 serge 1148
void evergreen_mc_program(struct radeon_device *rdev)
1430 serge 1149
{
1150
	struct evergreen_mc_save save;
1151
	u32 tmp;
1152
	int i, j;
1153
 
1154
	/* Initialize HDP */
1155
	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1156
		WREG32((0x2c14 + j), 0x00000000);
1157
		WREG32((0x2c18 + j), 0x00000000);
1158
		WREG32((0x2c1c + j), 0x00000000);
1159
		WREG32((0x2c20 + j), 0x00000000);
1160
		WREG32((0x2c24 + j), 0x00000000);
1161
	}
1162
	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1163
 
1164
	evergreen_mc_stop(rdev, &save);
1165
	if (evergreen_mc_wait_for_idle(rdev)) {
1166
		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1167
	}
1168
	/* Lockout access through VGA aperture*/
1169
	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1170
	/* Update configuration */
1171
	if (rdev->flags & RADEON_IS_AGP) {
1172
		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1173
			/* VRAM before AGP */
1174
			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1175
				rdev->mc.vram_start >> 12);
1176
			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1177
				rdev->mc.gtt_end >> 12);
1178
		} else {
1179
			/* VRAM after AGP */
1180
			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1181
				rdev->mc.gtt_start >> 12);
1182
			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1183
				rdev->mc.vram_end >> 12);
1184
		}
1185
	} else {
1186
		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1187
			rdev->mc.vram_start >> 12);
1188
		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1189
			rdev->mc.vram_end >> 12);
1190
	}
1191
	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
1963 serge 1192
	if (rdev->flags & RADEON_IS_IGP) {
1193
		tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
1194
		tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
1195
		tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
1196
		WREG32(MC_FUS_VM_FB_OFFSET, tmp);
1197
	}
1430 serge 1198
	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1199
	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1200
	WREG32(MC_VM_FB_LOCATION, tmp);
1201
	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1963 serge 1202
	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
1203
	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1430 serge 1204
	if (rdev->flags & RADEON_IS_AGP) {
1205
		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1206
		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1207
		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1208
	} else {
1209
		WREG32(MC_VM_AGP_BASE, 0);
1210
		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1211
		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1212
	}
1213
	if (evergreen_mc_wait_for_idle(rdev)) {
1214
		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1215
	}
1216
	evergreen_mc_resume(rdev, &save);
1217
	/* we need to own VRAM, so turn off the VGA renderer here
1218
	 * to stop it overwriting our objects */
1219
	rv515_vga_render_disable(rdev);
1220
}
1221
 
1222
/*
1223
 * CP.
1224
 */
1986 serge 1225
void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1226
{
1227
	/* set to DX10/11 mode */
1228
	radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
1229
	radeon_ring_write(rdev, 1);
1230
	/* FIXME: implement */
1231
	radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1232
	radeon_ring_write(rdev,
1233
#ifdef __BIG_ENDIAN
1234
			  (2 << 0) |
1235
#endif
1236
			  (ib->gpu_addr & 0xFFFFFFFC));
1237
	radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
1238
	radeon_ring_write(rdev, ib->length_dw);
1239
}
1963 serge 1240
 
1986 serge 1241
 
1963 serge 1242
static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1430 serge 1243
{
1963 serge 1244
	const __be32 *fw_data;
1245
	int i;
1246
 
1247
	if (!rdev->me_fw || !rdev->pfp_fw)
1248
		return -EINVAL;
1249
 
1250
	r700_cp_stop(rdev);
1251
	WREG32(CP_RB_CNTL,
1252
#ifdef __BIG_ENDIAN
1253
	       BUF_SWAP_32BIT |
1254
#endif
1255
	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1256
 
1257
	fw_data = (const __be32 *)rdev->pfp_fw->data;
1258
	WREG32(CP_PFP_UCODE_ADDR, 0);
1259
	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
1260
		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1261
	WREG32(CP_PFP_UCODE_ADDR, 0);
1262
 
1263
	fw_data = (const __be32 *)rdev->me_fw->data;
1264
	WREG32(CP_ME_RAM_WADDR, 0);
1265
	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
1266
		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1267
 
1268
	WREG32(CP_PFP_UCODE_ADDR, 0);
1269
	WREG32(CP_ME_RAM_WADDR, 0);
1270
	WREG32(CP_ME_RAM_RADDR, 0);
1271
	return 0;
1430 serge 1272
}
1273
 
1963 serge 1274
static int evergreen_cp_start(struct radeon_device *rdev)
1275
{
1276
	int r, i;
1277
	uint32_t cp_me;
1430 serge 1278
 
1963 serge 1279
	r = radeon_ring_lock(rdev, 7);
1280
	if (r) {
1281
		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1282
		return r;
1283
	}
1284
	radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
1285
	radeon_ring_write(rdev, 0x1);
1286
	radeon_ring_write(rdev, 0x0);
1287
	radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
1288
	radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1289
	radeon_ring_write(rdev, 0);
1290
	radeon_ring_write(rdev, 0);
1291
	radeon_ring_unlock_commit(rdev);
1292
 
1293
	cp_me = 0xff;
1294
	WREG32(CP_ME_CNTL, cp_me);
1295
 
1296
	r = radeon_ring_lock(rdev, evergreen_default_size + 19);
1297
	if (r) {
1298
		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1299
		return r;
1300
	}
1301
 
1302
	/* setup clear context state */
1303
	radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1304
	radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1305
 
1306
	for (i = 0; i < evergreen_default_size; i++)
1307
		radeon_ring_write(rdev, evergreen_default_state[i]);
1308
 
1309
	radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1310
	radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
1311
 
1312
	/* set clear context state */
1313
	radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
1314
	radeon_ring_write(rdev, 0);
1315
 
1316
	/* SQ_VTX_BASE_VTX_LOC */
1317
	radeon_ring_write(rdev, 0xc0026f00);
1318
	radeon_ring_write(rdev, 0x00000000);
1319
	radeon_ring_write(rdev, 0x00000000);
1320
	radeon_ring_write(rdev, 0x00000000);
1321
 
1322
	/* Clear consts */
1323
	radeon_ring_write(rdev, 0xc0036f00);
1324
	radeon_ring_write(rdev, 0x00000bc4);
1325
	radeon_ring_write(rdev, 0xffffffff);
1326
	radeon_ring_write(rdev, 0xffffffff);
1327
	radeon_ring_write(rdev, 0xffffffff);
1328
 
1329
	radeon_ring_write(rdev, 0xc0026900);
1330
	radeon_ring_write(rdev, 0x00000316);
1331
	radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1332
	radeon_ring_write(rdev, 0x00000010); /*  */
1333
 
1334
	radeon_ring_unlock_commit(rdev);
1335
 
1336
	return 0;
1337
}
1338
 
1339
int evergreen_cp_resume(struct radeon_device *rdev)
1430 serge 1340
{
1963 serge 1341
	u32 tmp;
1342
	u32 rb_bufsz;
1343
	int r;
1430 serge 1344
 
1963 serge 1345
	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1346
	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1347
				 SOFT_RESET_PA |
1348
				 SOFT_RESET_SH |
1349
				 SOFT_RESET_VGT |
1350
				 SOFT_RESET_SX));
1351
	RREG32(GRBM_SOFT_RESET);
1352
	mdelay(15);
1353
	WREG32(GRBM_SOFT_RESET, 0);
1354
	RREG32(GRBM_SOFT_RESET);
1355
 
1356
	/* Set ring buffer size */
1357
	rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1358
	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1359
#ifdef __BIG_ENDIAN
1360
	tmp |= BUF_SWAP_32BIT;
1361
#endif
1362
	WREG32(CP_RB_CNTL, tmp);
1363
	WREG32(CP_SEM_WAIT_TIMER, 0x4);
1364
 
1365
	/* Set the write pointer delay */
1366
	WREG32(CP_RB_WPTR_DELAY, 0);
1367
 
1368
	/* Initialize the ring buffer's read and write pointers */
1369
	WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1370
	WREG32(CP_RB_RPTR_WR, 0);
1371
	WREG32(CP_RB_WPTR, 0);
1372
 
1373
	/* set the wb address wether it's enabled or not */
1374
	WREG32(CP_RB_RPTR_ADDR,
1375
#ifdef __BIG_ENDIAN
1376
	       RB_RPTR_SWAP(2) |
1377
#endif
1378
	       ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
1379
	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
1380
	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1381
 
1382
	if (rdev->wb.enabled)
1383
		WREG32(SCRATCH_UMSK, 0xff);
1384
	else {
1385
		tmp |= RB_NO_UPDATE;
1386
		WREG32(SCRATCH_UMSK, 0);
1387
	}
1388
 
1389
	mdelay(1);
1390
	WREG32(CP_RB_CNTL, tmp);
1391
 
1392
	WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
1393
	WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1394
 
1395
	rdev->cp.rptr = RREG32(CP_RB_RPTR);
1396
	rdev->cp.wptr = RREG32(CP_RB_WPTR);
1397
 
1398
	evergreen_cp_start(rdev);
1399
	rdev->cp.ready = true;
1400
	r = radeon_ring_test(rdev);
1401
	if (r) {
1402
		rdev->cp.ready = false;
1403
		return r;
1404
	}
1430 serge 1405
	return 0;
1406
}
1407
 
1408
/*
1409
 * Core functions
1410
 */
1963 serge 1411
static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
1412
						  u32 num_tile_pipes,
1430 serge 1413
						  u32 num_backends,
1414
						  u32 backend_disable_mask)
1415
{
1416
	u32 backend_map = 0;
1963 serge 1417
	u32 enabled_backends_mask = 0;
1418
	u32 enabled_backends_count = 0;
1419
	u32 cur_pipe;
1420
	u32 swizzle_pipe[EVERGREEN_MAX_PIPES];
1421
	u32 cur_backend = 0;
1422
	u32 i;
1423
	bool force_no_swizzle;
1430 serge 1424
 
1963 serge 1425
	if (num_tile_pipes > EVERGREEN_MAX_PIPES)
1426
		num_tile_pipes = EVERGREEN_MAX_PIPES;
1427
	if (num_tile_pipes < 1)
1428
		num_tile_pipes = 1;
1429
	if (num_backends > EVERGREEN_MAX_BACKENDS)
1430
		num_backends = EVERGREEN_MAX_BACKENDS;
1431
	if (num_backends < 1)
1432
		num_backends = 1;
1433
 
1434
	for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
1435
		if (((backend_disable_mask >> i) & 1) == 0) {
1436
			enabled_backends_mask |= (1 << i);
1437
			++enabled_backends_count;
1438
		}
1439
		if (enabled_backends_count == num_backends)
1440
			break;
1441
	}
1442
 
1443
	if (enabled_backends_count == 0) {
1444
		enabled_backends_mask = 1;
1445
		enabled_backends_count = 1;
1446
	}
1447
 
1448
	if (enabled_backends_count != num_backends)
1449
		num_backends = enabled_backends_count;
1450
 
1451
	memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * EVERGREEN_MAX_PIPES);
1452
	switch (rdev->family) {
1453
	case CHIP_CEDAR:
1454
	case CHIP_REDWOOD:
1455
	case CHIP_PALM:
1456
	case CHIP_SUMO:
1457
	case CHIP_SUMO2:
1458
	case CHIP_TURKS:
1459
	case CHIP_CAICOS:
1460
		force_no_swizzle = false;
1461
		break;
1462
	case CHIP_CYPRESS:
1463
	case CHIP_HEMLOCK:
1464
	case CHIP_JUNIPER:
1465
	case CHIP_BARTS:
1466
	default:
1467
		force_no_swizzle = true;
1468
		break;
1469
	}
1470
	if (force_no_swizzle) {
1471
		bool last_backend_enabled = false;
1472
 
1473
		force_no_swizzle = false;
1474
		for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
1475
			if (((enabled_backends_mask >> i) & 1) == 1) {
1476
				if (last_backend_enabled)
1477
					force_no_swizzle = true;
1478
				last_backend_enabled = true;
1479
			} else
1480
				last_backend_enabled = false;
1481
		}
1482
	}
1483
 
1484
	switch (num_tile_pipes) {
1485
	case 1:
1486
	case 3:
1487
	case 5:
1488
	case 7:
1489
		DRM_ERROR("odd number of pipes!\n");
1490
		break;
1491
	case 2:
1492
		swizzle_pipe[0] = 0;
1493
		swizzle_pipe[1] = 1;
1494
		break;
1495
	case 4:
1496
		if (force_no_swizzle) {
1497
			swizzle_pipe[0] = 0;
1498
			swizzle_pipe[1] = 1;
1499
			swizzle_pipe[2] = 2;
1500
			swizzle_pipe[3] = 3;
1501
		} else {
1502
			swizzle_pipe[0] = 0;
1503
			swizzle_pipe[1] = 2;
1504
			swizzle_pipe[2] = 1;
1505
			swizzle_pipe[3] = 3;
1506
		}
1507
		break;
1508
	case 6:
1509
		if (force_no_swizzle) {
1510
			swizzle_pipe[0] = 0;
1511
			swizzle_pipe[1] = 1;
1512
			swizzle_pipe[2] = 2;
1513
			swizzle_pipe[3] = 3;
1514
			swizzle_pipe[4] = 4;
1515
			swizzle_pipe[5] = 5;
1516
		} else {
1517
			swizzle_pipe[0] = 0;
1518
			swizzle_pipe[1] = 2;
1519
			swizzle_pipe[2] = 4;
1520
			swizzle_pipe[3] = 1;
1521
			swizzle_pipe[4] = 3;
1522
			swizzle_pipe[5] = 5;
1523
		}
1524
		break;
1525
	case 8:
1526
		if (force_no_swizzle) {
1527
			swizzle_pipe[0] = 0;
1528
			swizzle_pipe[1] = 1;
1529
			swizzle_pipe[2] = 2;
1530
			swizzle_pipe[3] = 3;
1531
			swizzle_pipe[4] = 4;
1532
			swizzle_pipe[5] = 5;
1533
			swizzle_pipe[6] = 6;
1534
			swizzle_pipe[7] = 7;
1535
		} else {
1536
			swizzle_pipe[0] = 0;
1537
			swizzle_pipe[1] = 2;
1538
			swizzle_pipe[2] = 4;
1539
			swizzle_pipe[3] = 6;
1540
			swizzle_pipe[4] = 1;
1541
			swizzle_pipe[5] = 3;
1542
			swizzle_pipe[6] = 5;
1543
			swizzle_pipe[7] = 7;
1544
		}
1545
		break;
1546
	}
1547
 
1548
	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1549
		while (((1 << cur_backend) & enabled_backends_mask) == 0)
1550
			cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
1551
 
1552
		backend_map |= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4)));
1553
 
1554
		cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
1555
	}
1556
 
1430 serge 1557
	return backend_map;
1558
}
1559
 
1986 serge 1560
static void evergreen_program_channel_remap(struct radeon_device *rdev)
1561
{
1562
	u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp;
1563
 
1564
	tmp = RREG32(MC_SHARED_CHMAP);
1565
	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1566
	case 0:
1567
	case 1:
1568
	case 2:
1569
	case 3:
1570
	default:
1571
		/* default mapping */
1572
		mc_shared_chremap = 0x00fac688;
1573
		break;
1574
	}
1575
 
1576
	switch (rdev->family) {
1577
	case CHIP_HEMLOCK:
1578
	case CHIP_CYPRESS:
1579
	case CHIP_BARTS:
1580
		tcp_chan_steer_lo = 0x54763210;
1581
		tcp_chan_steer_hi = 0x0000ba98;
1582
		break;
1583
	case CHIP_JUNIPER:
1584
	case CHIP_REDWOOD:
1585
	case CHIP_CEDAR:
1586
	case CHIP_PALM:
1587
	case CHIP_SUMO:
1588
	case CHIP_SUMO2:
1589
	case CHIP_TURKS:
1590
	case CHIP_CAICOS:
1591
	default:
1592
		tcp_chan_steer_lo = 0x76543210;
1593
		tcp_chan_steer_hi = 0x0000ba98;
1594
		break;
1595
	}
1596
 
1597
	WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo);
1598
	WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi);
1599
	WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
1600
}
1601
 
1430 serge 1602
static void evergreen_gpu_init(struct radeon_device *rdev)
1603
{
1963 serge 1604
	u32 cc_rb_backend_disable = 0;
1605
	u32 cc_gc_shader_pipe_config;
1606
	u32 gb_addr_config = 0;
1607
	u32 mc_shared_chmap, mc_arb_ramcfg;
1608
	u32 gb_backend_map;
1609
	u32 grbm_gfx_index;
1610
	u32 sx_debug_1;
1611
	u32 smx_dc_ctl0;
1612
	u32 sq_config;
1613
	u32 sq_lds_resource_mgmt;
1614
	u32 sq_gpr_resource_mgmt_1;
1615
	u32 sq_gpr_resource_mgmt_2;
1616
	u32 sq_gpr_resource_mgmt_3;
1617
	u32 sq_thread_resource_mgmt;
1618
	u32 sq_thread_resource_mgmt_2;
1619
	u32 sq_stack_resource_mgmt_1;
1620
	u32 sq_stack_resource_mgmt_2;
1621
	u32 sq_stack_resource_mgmt_3;
1622
	u32 vgt_cache_invalidation;
1623
	u32 hdp_host_path_cntl, tmp;
1624
	int i, j, num_shader_engines, ps_thread_count;
1625
 
1626
	switch (rdev->family) {
1627
	case CHIP_CYPRESS:
1628
	case CHIP_HEMLOCK:
1629
		rdev->config.evergreen.num_ses = 2;
1630
		rdev->config.evergreen.max_pipes = 4;
1631
		rdev->config.evergreen.max_tile_pipes = 8;
1632
		rdev->config.evergreen.max_simds = 10;
1633
		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1634
		rdev->config.evergreen.max_gprs = 256;
1635
		rdev->config.evergreen.max_threads = 248;
1636
		rdev->config.evergreen.max_gs_threads = 32;
1637
		rdev->config.evergreen.max_stack_entries = 512;
1638
		rdev->config.evergreen.sx_num_of_sets = 4;
1639
		rdev->config.evergreen.sx_max_export_size = 256;
1640
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1641
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1642
		rdev->config.evergreen.max_hw_contexts = 8;
1643
		rdev->config.evergreen.sq_num_cf_insts = 2;
1644
 
1645
		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1646
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1647
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1648
		break;
1649
	case CHIP_JUNIPER:
1650
		rdev->config.evergreen.num_ses = 1;
1651
		rdev->config.evergreen.max_pipes = 4;
1652
		rdev->config.evergreen.max_tile_pipes = 4;
1653
		rdev->config.evergreen.max_simds = 10;
1654
		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1655
		rdev->config.evergreen.max_gprs = 256;
1656
		rdev->config.evergreen.max_threads = 248;
1657
		rdev->config.evergreen.max_gs_threads = 32;
1658
		rdev->config.evergreen.max_stack_entries = 512;
1659
		rdev->config.evergreen.sx_num_of_sets = 4;
1660
		rdev->config.evergreen.sx_max_export_size = 256;
1661
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1662
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1663
		rdev->config.evergreen.max_hw_contexts = 8;
1664
		rdev->config.evergreen.sq_num_cf_insts = 2;
1665
 
1666
		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1667
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1668
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1669
		break;
1670
	case CHIP_REDWOOD:
1671
		rdev->config.evergreen.num_ses = 1;
1672
		rdev->config.evergreen.max_pipes = 4;
1673
		rdev->config.evergreen.max_tile_pipes = 4;
1674
		rdev->config.evergreen.max_simds = 5;
1675
		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1676
		rdev->config.evergreen.max_gprs = 256;
1677
		rdev->config.evergreen.max_threads = 248;
1678
		rdev->config.evergreen.max_gs_threads = 32;
1679
		rdev->config.evergreen.max_stack_entries = 256;
1680
		rdev->config.evergreen.sx_num_of_sets = 4;
1681
		rdev->config.evergreen.sx_max_export_size = 256;
1682
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1683
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1684
		rdev->config.evergreen.max_hw_contexts = 8;
1685
		rdev->config.evergreen.sq_num_cf_insts = 2;
1686
 
1687
		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1688
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1689
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1690
		break;
1691
	case CHIP_CEDAR:
1692
	default:
1693
		rdev->config.evergreen.num_ses = 1;
1694
		rdev->config.evergreen.max_pipes = 2;
1695
		rdev->config.evergreen.max_tile_pipes = 2;
1696
		rdev->config.evergreen.max_simds = 2;
1697
		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1698
		rdev->config.evergreen.max_gprs = 256;
1699
		rdev->config.evergreen.max_threads = 192;
1700
		rdev->config.evergreen.max_gs_threads = 16;
1701
		rdev->config.evergreen.max_stack_entries = 256;
1702
		rdev->config.evergreen.sx_num_of_sets = 4;
1703
		rdev->config.evergreen.sx_max_export_size = 128;
1704
		rdev->config.evergreen.sx_max_export_pos_size = 32;
1705
		rdev->config.evergreen.sx_max_export_smx_size = 96;
1706
		rdev->config.evergreen.max_hw_contexts = 4;
1707
		rdev->config.evergreen.sq_num_cf_insts = 1;
1708
 
1709
		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1710
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1711
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1712
		break;
1713
	case CHIP_PALM:
1714
		rdev->config.evergreen.num_ses = 1;
1715
		rdev->config.evergreen.max_pipes = 2;
1716
		rdev->config.evergreen.max_tile_pipes = 2;
1717
		rdev->config.evergreen.max_simds = 2;
1718
		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1719
		rdev->config.evergreen.max_gprs = 256;
1720
		rdev->config.evergreen.max_threads = 192;
1721
		rdev->config.evergreen.max_gs_threads = 16;
1722
		rdev->config.evergreen.max_stack_entries = 256;
1723
		rdev->config.evergreen.sx_num_of_sets = 4;
1724
		rdev->config.evergreen.sx_max_export_size = 128;
1725
		rdev->config.evergreen.sx_max_export_pos_size = 32;
1726
		rdev->config.evergreen.sx_max_export_smx_size = 96;
1727
		rdev->config.evergreen.max_hw_contexts = 4;
1728
		rdev->config.evergreen.sq_num_cf_insts = 1;
1729
 
1730
		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1731
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1732
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1733
		break;
1734
	case CHIP_SUMO:
1735
		rdev->config.evergreen.num_ses = 1;
1736
		rdev->config.evergreen.max_pipes = 4;
1737
		rdev->config.evergreen.max_tile_pipes = 2;
1738
		if (rdev->pdev->device == 0x9648)
1739
			rdev->config.evergreen.max_simds = 3;
1740
		else if ((rdev->pdev->device == 0x9647) ||
1741
			 (rdev->pdev->device == 0x964a))
1742
			rdev->config.evergreen.max_simds = 4;
1743
		else
1744
			rdev->config.evergreen.max_simds = 5;
1745
		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1746
		rdev->config.evergreen.max_gprs = 256;
1747
		rdev->config.evergreen.max_threads = 248;
1748
		rdev->config.evergreen.max_gs_threads = 32;
1749
		rdev->config.evergreen.max_stack_entries = 256;
1750
		rdev->config.evergreen.sx_num_of_sets = 4;
1751
		rdev->config.evergreen.sx_max_export_size = 256;
1752
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1753
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1754
		rdev->config.evergreen.max_hw_contexts = 8;
1755
		rdev->config.evergreen.sq_num_cf_insts = 2;
1756
 
1757
		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1758
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1759
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1760
		break;
1761
	case CHIP_SUMO2:
1762
		rdev->config.evergreen.num_ses = 1;
1763
		rdev->config.evergreen.max_pipes = 4;
1764
		rdev->config.evergreen.max_tile_pipes = 4;
1765
		rdev->config.evergreen.max_simds = 2;
1766
		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1767
		rdev->config.evergreen.max_gprs = 256;
1768
		rdev->config.evergreen.max_threads = 248;
1769
		rdev->config.evergreen.max_gs_threads = 32;
1770
		rdev->config.evergreen.max_stack_entries = 512;
1771
		rdev->config.evergreen.sx_num_of_sets = 4;
1772
		rdev->config.evergreen.sx_max_export_size = 256;
1773
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1774
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1775
		rdev->config.evergreen.max_hw_contexts = 8;
1776
		rdev->config.evergreen.sq_num_cf_insts = 2;
1777
 
1778
		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1779
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1780
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1781
		break;
1782
	case CHIP_BARTS:
1783
		rdev->config.evergreen.num_ses = 2;
1784
		rdev->config.evergreen.max_pipes = 4;
1785
		rdev->config.evergreen.max_tile_pipes = 8;
1786
		rdev->config.evergreen.max_simds = 7;
1787
		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1788
		rdev->config.evergreen.max_gprs = 256;
1789
		rdev->config.evergreen.max_threads = 248;
1790
		rdev->config.evergreen.max_gs_threads = 32;
1791
		rdev->config.evergreen.max_stack_entries = 512;
1792
		rdev->config.evergreen.sx_num_of_sets = 4;
1793
		rdev->config.evergreen.sx_max_export_size = 256;
1794
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1795
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1796
		rdev->config.evergreen.max_hw_contexts = 8;
1797
		rdev->config.evergreen.sq_num_cf_insts = 2;
1798
 
1799
		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1800
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1801
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1802
		break;
1803
	case CHIP_TURKS:
1804
		rdev->config.evergreen.num_ses = 1;
1805
		rdev->config.evergreen.max_pipes = 4;
1806
		rdev->config.evergreen.max_tile_pipes = 4;
1807
		rdev->config.evergreen.max_simds = 6;
1808
		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1809
		rdev->config.evergreen.max_gprs = 256;
1810
		rdev->config.evergreen.max_threads = 248;
1811
		rdev->config.evergreen.max_gs_threads = 32;
1812
		rdev->config.evergreen.max_stack_entries = 256;
1813
		rdev->config.evergreen.sx_num_of_sets = 4;
1814
		rdev->config.evergreen.sx_max_export_size = 256;
1815
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1816
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1817
		rdev->config.evergreen.max_hw_contexts = 8;
1818
		rdev->config.evergreen.sq_num_cf_insts = 2;
1819
 
1820
		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1821
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1822
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1823
		break;
1824
	case CHIP_CAICOS:
1825
		rdev->config.evergreen.num_ses = 1;
1826
		rdev->config.evergreen.max_pipes = 4;
1827
		rdev->config.evergreen.max_tile_pipes = 2;
1828
		rdev->config.evergreen.max_simds = 2;
1829
		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1830
		rdev->config.evergreen.max_gprs = 256;
1831
		rdev->config.evergreen.max_threads = 192;
1832
		rdev->config.evergreen.max_gs_threads = 16;
1833
		rdev->config.evergreen.max_stack_entries = 256;
1834
		rdev->config.evergreen.sx_num_of_sets = 4;
1835
		rdev->config.evergreen.sx_max_export_size = 128;
1836
		rdev->config.evergreen.sx_max_export_pos_size = 32;
1837
		rdev->config.evergreen.sx_max_export_smx_size = 96;
1838
		rdev->config.evergreen.max_hw_contexts = 4;
1839
		rdev->config.evergreen.sq_num_cf_insts = 1;
1840
 
1841
		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1842
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1843
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1844
		break;
1845
	}
1846
 
1847
	/* Initialize HDP */
1848
	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1849
		WREG32((0x2c14 + j), 0x00000000);
1850
		WREG32((0x2c18 + j), 0x00000000);
1851
		WREG32((0x2c1c + j), 0x00000000);
1852
		WREG32((0x2c20 + j), 0x00000000);
1853
		WREG32((0x2c24 + j), 0x00000000);
1854
	}
1855
 
1856
	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1857
 
1858
	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & ~2;
1859
 
1860
	cc_gc_shader_pipe_config |=
1861
		INACTIVE_QD_PIPES((EVERGREEN_MAX_PIPES_MASK << rdev->config.evergreen.max_pipes)
1862
				  & EVERGREEN_MAX_PIPES_MASK);
1863
	cc_gc_shader_pipe_config |=
1864
		INACTIVE_SIMDS((EVERGREEN_MAX_SIMDS_MASK << rdev->config.evergreen.max_simds)
1865
			       & EVERGREEN_MAX_SIMDS_MASK);
1866
 
1867
	cc_rb_backend_disable =
1868
		BACKEND_DISABLE((EVERGREEN_MAX_BACKENDS_MASK << rdev->config.evergreen.max_backends)
1869
				& EVERGREEN_MAX_BACKENDS_MASK);
1870
 
1871
 
1872
	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1873
	if (rdev->flags & RADEON_IS_IGP)
1874
		mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
1875
	else
1876
	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1877
 
1878
	switch (rdev->config.evergreen.max_tile_pipes) {
1879
	case 1:
1880
	default:
1881
		gb_addr_config |= NUM_PIPES(0);
1882
		break;
1883
	case 2:
1884
		gb_addr_config |= NUM_PIPES(1);
1885
		break;
1886
	case 4:
1887
		gb_addr_config |= NUM_PIPES(2);
1888
		break;
1889
	case 8:
1890
		gb_addr_config |= NUM_PIPES(3);
1891
		break;
1892
	}
1893
 
1894
	gb_addr_config |= PIPE_INTERLEAVE_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1895
	gb_addr_config |= BANK_INTERLEAVE_SIZE(0);
1896
	gb_addr_config |= NUM_SHADER_ENGINES(rdev->config.evergreen.num_ses - 1);
1897
	gb_addr_config |= SHADER_ENGINE_TILE_SIZE(1);
1898
	gb_addr_config |= NUM_GPUS(0); /* Hemlock? */
1899
	gb_addr_config |= MULTI_GPU_TILE_SIZE(2);
1900
 
1901
	if (((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) > 2)
1902
		gb_addr_config |= ROW_SIZE(2);
1903
	else
1904
		gb_addr_config |= ROW_SIZE((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT);
1905
 
1906
	if (rdev->ddev->pdev->device == 0x689e) {
1907
		u32 efuse_straps_4;
1908
		u32 efuse_straps_3;
1909
		u8 efuse_box_bit_131_124;
1910
 
1911
		WREG32(RCU_IND_INDEX, 0x204);
1912
		efuse_straps_4 = RREG32(RCU_IND_DATA);
1913
		WREG32(RCU_IND_INDEX, 0x203);
1914
		efuse_straps_3 = RREG32(RCU_IND_DATA);
1915
		efuse_box_bit_131_124 = (u8)(((efuse_straps_4 & 0xf) << 4) | ((efuse_straps_3 & 0xf0000000) >> 28));
1916
 
1917
		switch(efuse_box_bit_131_124) {
1918
		case 0x00:
1919
			gb_backend_map = 0x76543210;
1920
			break;
1921
		case 0x55:
1922
			gb_backend_map = 0x77553311;
1923
			break;
1924
		case 0x56:
1925
			gb_backend_map = 0x77553300;
1926
			break;
1927
		case 0x59:
1928
			gb_backend_map = 0x77552211;
1929
			break;
1930
		case 0x66:
1931
			gb_backend_map = 0x77443300;
1932
			break;
1933
		case 0x99:
1934
			gb_backend_map = 0x66552211;
1935
			break;
1936
		case 0x5a:
1937
			gb_backend_map = 0x77552200;
1938
			break;
1939
		case 0xaa:
1940
			gb_backend_map = 0x66442200;
1941
			break;
1942
		case 0x95:
1943
			gb_backend_map = 0x66553311;
1944
			break;
1945
		default:
1946
			DRM_ERROR("bad backend map, using default\n");
1947
			gb_backend_map =
1948
				evergreen_get_tile_pipe_to_backend_map(rdev,
1949
								       rdev->config.evergreen.max_tile_pipes,
1950
								       rdev->config.evergreen.max_backends,
1951
								       ((EVERGREEN_MAX_BACKENDS_MASK <<
1952
								   rdev->config.evergreen.max_backends) &
1953
									EVERGREEN_MAX_BACKENDS_MASK));
1954
			break;
1955
		}
1956
	} else if (rdev->ddev->pdev->device == 0x68b9) {
1957
		u32 efuse_straps_3;
1958
		u8 efuse_box_bit_127_124;
1959
 
1960
		WREG32(RCU_IND_INDEX, 0x203);
1961
		efuse_straps_3 = RREG32(RCU_IND_DATA);
1962
		efuse_box_bit_127_124 = (u8)((efuse_straps_3 & 0xF0000000) >> 28);
1963
 
1964
		switch(efuse_box_bit_127_124) {
1965
		case 0x0:
1966
			gb_backend_map = 0x00003210;
1967
			break;
1968
		case 0x5:
1969
		case 0x6:
1970
		case 0x9:
1971
		case 0xa:
1972
			gb_backend_map = 0x00003311;
1973
			break;
1974
		default:
1975
			DRM_ERROR("bad backend map, using default\n");
1976
			gb_backend_map =
1977
				evergreen_get_tile_pipe_to_backend_map(rdev,
1978
								       rdev->config.evergreen.max_tile_pipes,
1979
								       rdev->config.evergreen.max_backends,
1980
								       ((EVERGREEN_MAX_BACKENDS_MASK <<
1981
								   rdev->config.evergreen.max_backends) &
1982
									EVERGREEN_MAX_BACKENDS_MASK));
1983
			break;
1984
		}
1985
	} else {
1986
		switch (rdev->family) {
1987
		case CHIP_CYPRESS:
1988
		case CHIP_HEMLOCK:
1989
		case CHIP_BARTS:
1990
			gb_backend_map = 0x66442200;
1991
			break;
1992
		case CHIP_JUNIPER:
2005 serge 1993
			gb_backend_map = 0x00002200;
1963 serge 1994
			break;
1995
		default:
1996
			gb_backend_map =
1997
				evergreen_get_tile_pipe_to_backend_map(rdev,
1998
								       rdev->config.evergreen.max_tile_pipes,
1999
								       rdev->config.evergreen.max_backends,
2000
								       ((EVERGREEN_MAX_BACKENDS_MASK <<
2001
									 rdev->config.evergreen.max_backends) &
2002
									EVERGREEN_MAX_BACKENDS_MASK));
2003
		}
2004
	}
2005
 
2006
	/* setup tiling info dword.  gb_addr_config is not adequate since it does
2007
	 * not have bank info, so create a custom tiling dword.
2008
	 * bits 3:0   num_pipes
2009
	 * bits 7:4   num_banks
2010
	 * bits 11:8  group_size
2011
	 * bits 15:12 row_size
2012
	 */
2013
	rdev->config.evergreen.tile_config = 0;
2014
	switch (rdev->config.evergreen.max_tile_pipes) {
2015
	case 1:
2016
	default:
2017
		rdev->config.evergreen.tile_config |= (0 << 0);
2018
		break;
2019
	case 2:
2020
		rdev->config.evergreen.tile_config |= (1 << 0);
2021
		break;
2022
	case 4:
2023
		rdev->config.evergreen.tile_config |= (2 << 0);
2024
		break;
2025
	case 8:
2026
		rdev->config.evergreen.tile_config |= (3 << 0);
2027
		break;
2028
	}
1986 serge 2029
	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1963 serge 2030
	if (rdev->flags & RADEON_IS_IGP)
1986 serge 2031
		rdev->config.evergreen.tile_config |= 1 << 4;
1963 serge 2032
	else
2033
		rdev->config.evergreen.tile_config |=
2034
			((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
2035
	rdev->config.evergreen.tile_config |=
2036
		((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT) << 8;
2037
	rdev->config.evergreen.tile_config |=
2038
		((gb_addr_config & 0x30000000) >> 28) << 12;
2039
 
2040
	WREG32(GB_BACKEND_MAP, gb_backend_map);
2041
	WREG32(GB_ADDR_CONFIG, gb_addr_config);
2042
	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2043
	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2044
 
2045
	evergreen_program_channel_remap(rdev);
2046
 
2047
	num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
2048
	grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
2049
 
2050
	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
2051
		u32 rb = cc_rb_backend_disable | (0xf0 << 16);
2052
		u32 sp = cc_gc_shader_pipe_config;
2053
		u32 gfx = grbm_gfx_index | SE_INDEX(i);
2054
 
2055
		if (i == num_shader_engines) {
2056
			rb |= BACKEND_DISABLE(EVERGREEN_MAX_BACKENDS_MASK);
2057
			sp |= INACTIVE_SIMDS(EVERGREEN_MAX_SIMDS_MASK);
2058
		}
2059
 
2060
		WREG32(GRBM_GFX_INDEX, gfx);
2061
		WREG32(RLC_GFX_INDEX, gfx);
2062
 
2063
		WREG32(CC_RB_BACKEND_DISABLE, rb);
2064
		WREG32(CC_SYS_RB_BACKEND_DISABLE, rb);
2065
		WREG32(GC_USER_RB_BACKEND_DISABLE, rb);
2066
		WREG32(CC_GC_SHADER_PIPE_CONFIG, sp);
2067
        }
2068
 
2069
	grbm_gfx_index |= SE_BROADCAST_WRITES;
2070
	WREG32(GRBM_GFX_INDEX, grbm_gfx_index);
2071
	WREG32(RLC_GFX_INDEX, grbm_gfx_index);
2072
 
2073
	WREG32(CGTS_SYS_TCC_DISABLE, 0);
2074
	WREG32(CGTS_TCC_DISABLE, 0);
2075
	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
2076
	WREG32(CGTS_USER_TCC_DISABLE, 0);
2077
 
2078
	/* set HW defaults for 3D engine */
2079
	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2080
				     ROQ_IB2_START(0x2b)));
2081
 
2082
	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
2083
 
2084
	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
2085
			     SYNC_GRADIENT |
2086
			     SYNC_WALKER |
2087
			     SYNC_ALIGNER));
2088
 
2089
	sx_debug_1 = RREG32(SX_DEBUG_1);
2090
	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
2091
	WREG32(SX_DEBUG_1, sx_debug_1);
2092
 
2093
 
2094
	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
2095
	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
2096
	smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
2097
	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
2098
 
2099
	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
2100
					POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
2101
					SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
2102
 
2103
	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
2104
				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
2105
				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
2106
 
2107
	WREG32(VGT_NUM_INSTANCES, 1);
2108
	WREG32(SPI_CONFIG_CNTL, 0);
2109
	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2110
	WREG32(CP_PERFMON_CNTL, 0);
2111
 
2112
	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
2113
				  FETCH_FIFO_HIWATER(0x4) |
2114
				  DONE_FIFO_HIWATER(0xe0) |
2115
				  ALU_UPDATE_FIFO_HIWATER(0x8)));
2116
 
2117
	sq_config = RREG32(SQ_CONFIG);
2118
	sq_config &= ~(PS_PRIO(3) |
2119
		       VS_PRIO(3) |
2120
		       GS_PRIO(3) |
2121
		       ES_PRIO(3));
2122
	sq_config |= (VC_ENABLE |
2123
		      EXPORT_SRC_C |
2124
		      PS_PRIO(0) |
2125
		      VS_PRIO(1) |
2126
		      GS_PRIO(2) |
2127
		      ES_PRIO(3));
2128
 
2129
	switch (rdev->family) {
2130
	case CHIP_CEDAR:
2131
	case CHIP_PALM:
2132
	case CHIP_SUMO:
2133
	case CHIP_SUMO2:
2134
	case CHIP_CAICOS:
2135
		/* no vertex cache */
2136
		sq_config &= ~VC_ENABLE;
2137
		break;
2138
	default:
2139
		break;
2140
	}
2141
 
2142
	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
2143
 
2144
	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
2145
	sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
2146
	sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
2147
	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2148
	sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2149
	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2150
	sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2151
 
2152
	switch (rdev->family) {
2153
	case CHIP_CEDAR:
2154
	case CHIP_PALM:
2155
	case CHIP_SUMO:
2156
	case CHIP_SUMO2:
2157
		ps_thread_count = 96;
2158
		break;
2159
	default:
2160
		ps_thread_count = 128;
2161
		break;
2162
	}
2163
 
2164
	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
2165
	sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2166
	sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2167
	sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2168
	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2169
	sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2170
 
2171
	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2172
	sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2173
	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2174
	sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2175
	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2176
	sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2177
 
2178
	WREG32(SQ_CONFIG, sq_config);
2179
	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2180
	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2181
	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
2182
	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2183
	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
2184
	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2185
	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2186
	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
2187
	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
2188
	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
2189
 
2190
	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2191
					  FORCE_EOV_MAX_REZ_CNT(255)));
2192
 
2193
	switch (rdev->family) {
2194
	case CHIP_CEDAR:
2195
	case CHIP_PALM:
2196
	case CHIP_SUMO:
2197
	case CHIP_SUMO2:
2198
	case CHIP_CAICOS:
2199
		vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
2200
		break;
2201
	default:
2202
		vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
2203
		break;
2204
	}
2205
	vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
2206
	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
2207
 
2208
	WREG32(VGT_GS_VERTEX_REUSE, 16);
2209
	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
2210
	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2211
 
2212
	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
2213
	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
2214
 
2215
	WREG32(CB_PERF_CTR0_SEL_0, 0);
2216
	WREG32(CB_PERF_CTR0_SEL_1, 0);
2217
	WREG32(CB_PERF_CTR1_SEL_0, 0);
2218
	WREG32(CB_PERF_CTR1_SEL_1, 0);
2219
	WREG32(CB_PERF_CTR2_SEL_0, 0);
2220
	WREG32(CB_PERF_CTR2_SEL_1, 0);
2221
	WREG32(CB_PERF_CTR3_SEL_0, 0);
2222
	WREG32(CB_PERF_CTR3_SEL_1, 0);
2223
 
2224
	/* clear render buffer base addresses */
2225
	WREG32(CB_COLOR0_BASE, 0);
2226
	WREG32(CB_COLOR1_BASE, 0);
2227
	WREG32(CB_COLOR2_BASE, 0);
2228
	WREG32(CB_COLOR3_BASE, 0);
2229
	WREG32(CB_COLOR4_BASE, 0);
2230
	WREG32(CB_COLOR5_BASE, 0);
2231
	WREG32(CB_COLOR6_BASE, 0);
2232
	WREG32(CB_COLOR7_BASE, 0);
2233
	WREG32(CB_COLOR8_BASE, 0);
2234
	WREG32(CB_COLOR9_BASE, 0);
2235
	WREG32(CB_COLOR10_BASE, 0);
2236
	WREG32(CB_COLOR11_BASE, 0);
2237
 
2238
	/* set the shader const cache sizes to 0 */
2239
	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
2240
		WREG32(i, 0);
2241
	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
2242
		WREG32(i, 0);
2243
 
2244
	tmp = RREG32(HDP_MISC_CNTL);
2245
	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2246
	WREG32(HDP_MISC_CNTL, tmp);
2247
 
2248
	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2249
	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2250
 
2251
	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2252
 
2253
	udelay(50);
2254
 
1430 serge 2255
}
2256
 
2257
int evergreen_mc_init(struct radeon_device *rdev)
2258
{
2259
	u32 tmp;
2260
	int chansize, numchan;
2261
 
2262
	/* Get VRAM informations */
2263
	rdev->mc.vram_is_ddr = true;
2004 serge 2264
	if (rdev->flags & RADEON_IS_IGP)
2265
		tmp = RREG32(FUS_MC_ARB_RAMCFG);
2266
	else
1430 serge 2267
	tmp = RREG32(MC_ARB_RAMCFG);
2268
	if (tmp & CHANSIZE_OVERRIDE) {
2269
		chansize = 16;
2270
	} else if (tmp & CHANSIZE_MASK) {
2271
		chansize = 64;
2272
	} else {
2273
		chansize = 32;
2274
	}
2275
	tmp = RREG32(MC_SHARED_CHMAP);
2276
	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2277
	case 0:
2278
	default:
2279
		numchan = 1;
2280
		break;
2281
	case 1:
2282
		numchan = 2;
2283
		break;
2284
	case 2:
2285
		numchan = 4;
2286
		break;
2287
	case 3:
2288
		numchan = 8;
2289
		break;
2290
	}
2291
	rdev->mc.vram_width = numchan * chansize;
2292
	/* Could aper size report 0 ? */
1963 serge 2293
	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
2294
	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1430 serge 2295
	/* Setup GPU memory space */
1963 serge 2296
	if (rdev->flags & RADEON_IS_IGP) {
2297
		/* size in bytes on fusion */
2298
		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
2299
		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
2300
	} else {
1430 serge 2301
	/* size in MB on evergreen */
2302
	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2303
	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
1963 serge 2304
	}
1430 serge 2305
	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1963 serge 2306
	r700_vram_gtt_location(rdev, &rdev->mc);
2307
	radeon_update_bandwidth_info(rdev);
2308
 
1430 serge 2309
	return 0;
2310
}
2311
 
1963 serge 2312
bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
1430 serge 2313
{
1986 serge 2314
	u32 srbm_status;
2315
	u32 grbm_status;
2316
	u32 grbm_status_se0, grbm_status_se1;
2317
	struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
2318
	int r;
2319
 
2320
	srbm_status = RREG32(SRBM_STATUS);
2321
	grbm_status = RREG32(GRBM_STATUS);
2322
	grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
2323
	grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
2324
	if (!(grbm_status & GUI_ACTIVE)) {
2325
		r100_gpu_lockup_update(lockup, &rdev->cp);
1963 serge 2326
	return false;
1986 serge 2327
	}
2328
	/* force CP activities */
2329
	r = radeon_ring_lock(rdev, 2);
2330
	if (!r) {
2331
		/* PACKET2 NOP */
2332
		radeon_ring_write(rdev, 0x80000000);
2333
		radeon_ring_write(rdev, 0x80000000);
2334
		radeon_ring_unlock_commit(rdev);
2335
	}
2336
	rdev->cp.rptr = RREG32(CP_RB_RPTR);
2337
	return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
1963 serge 2338
}
2339
 
2340
static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
2341
{
2342
	struct evergreen_mc_save save;
2343
	u32 grbm_reset = 0;
2344
 
2345
	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
2346
		return 0;
2347
 
2348
	dev_info(rdev->dev, "GPU softreset \n");
2349
	dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
2350
		RREG32(GRBM_STATUS));
2351
	dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
2352
		RREG32(GRBM_STATUS_SE0));
2353
	dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
2354
		RREG32(GRBM_STATUS_SE1));
2355
	dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
2356
		RREG32(SRBM_STATUS));
2357
	evergreen_mc_stop(rdev, &save);
2358
	if (evergreen_mc_wait_for_idle(rdev)) {
2359
		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2360
	}
2361
	/* Disable CP parsing/prefetching */
2362
	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
2363
 
2364
	/* reset all the gfx blocks */
2365
	grbm_reset = (SOFT_RESET_CP |
2366
		      SOFT_RESET_CB |
2367
		      SOFT_RESET_DB |
2368
		      SOFT_RESET_PA |
2369
		      SOFT_RESET_SC |
2370
		      SOFT_RESET_SPI |
2371
		      SOFT_RESET_SH |
2372
		      SOFT_RESET_SX |
2373
		      SOFT_RESET_TC |
2374
		      SOFT_RESET_TA |
2375
		      SOFT_RESET_VC |
2376
		      SOFT_RESET_VGT);
2377
 
2378
	dev_info(rdev->dev, "  GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
2379
	WREG32(GRBM_SOFT_RESET, grbm_reset);
2380
	(void)RREG32(GRBM_SOFT_RESET);
2381
	udelay(50);
2382
	WREG32(GRBM_SOFT_RESET, 0);
2383
	(void)RREG32(GRBM_SOFT_RESET);
2384
	/* Wait a little for things to settle down */
2385
	udelay(50);
2386
	dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
2387
		RREG32(GRBM_STATUS));
2388
	dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
2389
		RREG32(GRBM_STATUS_SE0));
2390
	dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
2391
		RREG32(GRBM_STATUS_SE1));
2392
	dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
2393
		RREG32(SRBM_STATUS));
2394
	evergreen_mc_resume(rdev, &save);
1430 serge 2395
	return 0;
2396
}
2397
 
1963 serge 2398
int evergreen_asic_reset(struct radeon_device *rdev)
2399
{
2400
	return evergreen_gpu_soft_reset(rdev);
2401
}
2402
 
2403
/* Interrupts */
2404
 
2405
u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
2406
{
2407
	switch (crtc) {
2408
	case 0:
2409
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC0_REGISTER_OFFSET);
2410
	case 1:
2411
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC1_REGISTER_OFFSET);
2412
	case 2:
2413
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC2_REGISTER_OFFSET);
2414
	case 3:
2415
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC3_REGISTER_OFFSET);
2416
	case 4:
2417
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC4_REGISTER_OFFSET);
2418
	case 5:
2419
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC5_REGISTER_OFFSET);
2420
	default:
2421
	return 0;
2422
	}
2423
}
2424
 
2425
void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2426
{
2427
	u32 tmp;
2428
 
2429
	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2430
	WREG32(GRBM_INT_CNTL, 0);
2431
	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2432
	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005 serge 2433
	if (rdev->num_crtc >= 4) {
1963 serge 2434
	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2435
	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005 serge 2436
	}
2437
	if (rdev->num_crtc >= 6) {
1963 serge 2438
	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2439
	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2440
	}
2441
 
2442
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2443
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005 serge 2444
	if (rdev->num_crtc >= 4) {
1963 serge 2445
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2446
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005 serge 2447
	}
2448
	if (rdev->num_crtc >= 6) {
1963 serge 2449
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2450
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2451
	}
2452
 
2453
	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2454
	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2455
 
2456
	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2457
	WREG32(DC_HPD1_INT_CONTROL, tmp);
2458
	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2459
	WREG32(DC_HPD2_INT_CONTROL, tmp);
2460
	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2461
	WREG32(DC_HPD3_INT_CONTROL, tmp);
2462
	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2463
	WREG32(DC_HPD4_INT_CONTROL, tmp);
2464
	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2465
	WREG32(DC_HPD5_INT_CONTROL, tmp);
2466
	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2467
	WREG32(DC_HPD6_INT_CONTROL, tmp);
2468
 
2469
}
2005 serge 2470
 
2471
int evergreen_irq_set(struct radeon_device *rdev)
2472
{
2473
	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2474
	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
2475
	u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2476
	u32 grbm_int_cntl = 0;
2477
	u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2478
 
2479
	if (!rdev->irq.installed) {
2480
		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
2481
		return -EINVAL;
2482
	}
2483
	/* don't enable anything if the ih is disabled */
2484
	if (!rdev->ih.enabled) {
2485
		r600_disable_interrupts(rdev);
2486
		/* force the active interrupt state to all disabled */
2487
		evergreen_disable_interrupt_state(rdev);
2488
		return 0;
2489
	}
2490
 
2491
	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2492
	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2493
	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2494
	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2495
	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2496
	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2497
 
2498
	if (rdev->irq.sw_int) {
2499
		DRM_DEBUG("evergreen_irq_set: sw int\n");
2500
		cp_int_cntl |= RB_INT_ENABLE;
2501
		cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2502
	}
2503
	if (rdev->irq.crtc_vblank_int[0] ||
2504
	    rdev->irq.pflip[0]) {
2505
		DRM_DEBUG("evergreen_irq_set: vblank 0\n");
2506
		crtc1 |= VBLANK_INT_MASK;
2507
	}
2508
	if (rdev->irq.crtc_vblank_int[1] ||
2509
	    rdev->irq.pflip[1]) {
2510
		DRM_DEBUG("evergreen_irq_set: vblank 1\n");
2511
		crtc2 |= VBLANK_INT_MASK;
2512
	}
2513
	if (rdev->irq.crtc_vblank_int[2] ||
2514
	    rdev->irq.pflip[2]) {
2515
		DRM_DEBUG("evergreen_irq_set: vblank 2\n");
2516
		crtc3 |= VBLANK_INT_MASK;
2517
	}
2518
	if (rdev->irq.crtc_vblank_int[3] ||
2519
	    rdev->irq.pflip[3]) {
2520
		DRM_DEBUG("evergreen_irq_set: vblank 3\n");
2521
		crtc4 |= VBLANK_INT_MASK;
2522
	}
2523
	if (rdev->irq.crtc_vblank_int[4] ||
2524
	    rdev->irq.pflip[4]) {
2525
		DRM_DEBUG("evergreen_irq_set: vblank 4\n");
2526
		crtc5 |= VBLANK_INT_MASK;
2527
	}
2528
	if (rdev->irq.crtc_vblank_int[5] ||
2529
	    rdev->irq.pflip[5]) {
2530
		DRM_DEBUG("evergreen_irq_set: vblank 5\n");
2531
		crtc6 |= VBLANK_INT_MASK;
2532
	}
2533
	if (rdev->irq.hpd[0]) {
2534
		DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2535
		hpd1 |= DC_HPDx_INT_EN;
2536
	}
2537
	if (rdev->irq.hpd[1]) {
2538
		DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2539
		hpd2 |= DC_HPDx_INT_EN;
2540
	}
2541
	if (rdev->irq.hpd[2]) {
2542
		DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2543
		hpd3 |= DC_HPDx_INT_EN;
2544
	}
2545
	if (rdev->irq.hpd[3]) {
2546
		DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2547
		hpd4 |= DC_HPDx_INT_EN;
2548
	}
2549
	if (rdev->irq.hpd[4]) {
2550
		DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2551
		hpd5 |= DC_HPDx_INT_EN;
2552
	}
2553
	if (rdev->irq.hpd[5]) {
2554
		DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2555
		hpd6 |= DC_HPDx_INT_EN;
2556
	}
2557
	if (rdev->irq.gui_idle) {
2558
		DRM_DEBUG("gui idle\n");
2559
		grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
2560
	}
2561
 
2562
	WREG32(CP_INT_CNTL, cp_int_cntl);
2563
	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
2564
 
2565
	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
2566
	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
2567
	if (rdev->num_crtc >= 4) {
2568
		WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
2569
		WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
2570
	}
2571
	if (rdev->num_crtc >= 6) {
2572
		WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
2573
		WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
2574
	}
2575
 
2576
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
2577
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
2578
	if (rdev->num_crtc >= 4) {
2579
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
2580
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
2581
	}
2582
	if (rdev->num_crtc >= 6) {
2583
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
2584
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
2585
	}
2586
 
2587
	WREG32(DC_HPD1_INT_CONTROL, hpd1);
2588
	WREG32(DC_HPD2_INT_CONTROL, hpd2);
2589
	WREG32(DC_HPD3_INT_CONTROL, hpd3);
2590
	WREG32(DC_HPD4_INT_CONTROL, hpd4);
2591
	WREG32(DC_HPD5_INT_CONTROL, hpd5);
2592
	WREG32(DC_HPD6_INT_CONTROL, hpd6);
2593
 
2594
	return 0;
2595
}
2596
 
2597
static inline void evergreen_irq_ack(struct radeon_device *rdev)
2598
{
2599
	u32 tmp;
2600
 
2601
	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
2602
	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2603
	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
2604
	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
2605
	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
2606
	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
2607
	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
2608
	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
2609
	if (rdev->num_crtc >= 4) {
2610
		rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
2611
		rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
2612
	}
2613
	if (rdev->num_crtc >= 6) {
2614
		rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
2615
		rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
2616
	}
2617
 
2618
	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
2619
		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2620
	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
2621
		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2622
	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
2623
		WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
2624
	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
2625
		WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
2626
	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
2627
		WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
2628
	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
2629
		WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
2630
 
2631
	if (rdev->num_crtc >= 4) {
2632
		if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
2633
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2634
		if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
2635
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2636
		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
2637
			WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
2638
		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
2639
			WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
2640
		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
2641
			WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
2642
		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
2643
			WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
2644
	}
2645
 
2646
	if (rdev->num_crtc >= 6) {
2647
		if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
2648
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2649
		if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
2650
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2651
		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
2652
			WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
2653
		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
2654
			WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
2655
		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
2656
			WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
2657
		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
2658
			WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
2659
	}
2660
 
2661
	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2662
		tmp = RREG32(DC_HPD1_INT_CONTROL);
2663
		tmp |= DC_HPDx_INT_ACK;
2664
		WREG32(DC_HPD1_INT_CONTROL, tmp);
2665
	}
2666
	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2667
		tmp = RREG32(DC_HPD2_INT_CONTROL);
2668
		tmp |= DC_HPDx_INT_ACK;
2669
		WREG32(DC_HPD2_INT_CONTROL, tmp);
2670
	}
2671
	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2672
		tmp = RREG32(DC_HPD3_INT_CONTROL);
2673
		tmp |= DC_HPDx_INT_ACK;
2674
		WREG32(DC_HPD3_INT_CONTROL, tmp);
2675
	}
2676
	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2677
		tmp = RREG32(DC_HPD4_INT_CONTROL);
2678
		tmp |= DC_HPDx_INT_ACK;
2679
		WREG32(DC_HPD4_INT_CONTROL, tmp);
2680
	}
2681
	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2682
		tmp = RREG32(DC_HPD5_INT_CONTROL);
2683
		tmp |= DC_HPDx_INT_ACK;
2684
		WREG32(DC_HPD5_INT_CONTROL, tmp);
2685
	}
2686
	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2687
		tmp = RREG32(DC_HPD5_INT_CONTROL);
2688
		tmp |= DC_HPDx_INT_ACK;
2689
		WREG32(DC_HPD6_INT_CONTROL, tmp);
2690
	}
2691
}
2692
static inline u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
2693
{
2694
	u32 wptr, tmp;
2695
 
2696
	if (rdev->wb.enabled)
2697
		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
2698
	else
2699
		wptr = RREG32(IH_RB_WPTR);
2700
 
2701
	if (wptr & RB_OVERFLOW) {
2702
		/* When a ring buffer overflow happen start parsing interrupt
2703
		 * from the last not overwritten vector (wptr + 16). Hopefully
2704
		 * this should allow us to catchup.
2705
		 */
2706
		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
2707
			wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
2708
		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
2709
		tmp = RREG32(IH_RB_CNTL);
2710
		tmp |= IH_WPTR_OVERFLOW_CLEAR;
2711
		WREG32(IH_RB_CNTL, tmp);
2712
	}
2713
	return (wptr & rdev->ih.ptr_mask);
2714
}
2715
 
2716
int evergreen_irq_process(struct radeon_device *rdev)
2717
{
2718
	u32 wptr;
2719
	u32 rptr;
2720
	u32 src_id, src_data;
2721
	u32 ring_index;
2722
	unsigned long flags;
2723
	bool queue_hotplug = false;
2724
 
2725
	if (!rdev->ih.enabled || rdev->shutdown)
2726
		return IRQ_NONE;
2727
 
2728
	wptr = evergreen_get_ih_wptr(rdev);
2729
	rptr = rdev->ih.rptr;
2730
	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
2731
 
2732
	spin_lock_irqsave(&rdev->ih.lock, flags);
2733
	if (rptr == wptr) {
2734
		spin_unlock_irqrestore(&rdev->ih.lock, flags);
2735
		return IRQ_NONE;
2736
	}
2737
restart_ih:
2738
	/* display interrupts */
2739
	evergreen_irq_ack(rdev);
2740
 
2741
	rdev->ih.wptr = wptr;
2742
	while (rptr != wptr) {
2743
		/* wptr/rptr are in bytes! */
2744
		ring_index = rptr / 4;
2745
		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
2746
		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
2747
 
2748
		switch (src_id) {
2749
		case 1: /* D1 vblank/vline */
2750
			switch (src_data) {
2751
			case 0: /* D1 vblank */
2752
				if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
2753
					if (rdev->irq.crtc_vblank_int[0]) {
2754
				//		drm_handle_vblank(rdev->ddev, 0);
2755
						rdev->pm.vblank_sync = true;
2756
				//		wake_up(&rdev->irq.vblank_queue);
2757
					}
2758
				//	if (rdev->irq.pflip[0])
2759
				//		radeon_crtc_handle_flip(rdev, 0);
2760
					rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
2761
					DRM_DEBUG("IH: D1 vblank\n");
2762
				}
2763
				break;
2764
			case 1: /* D1 vline */
2765
				if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
2766
					rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
2767
					DRM_DEBUG("IH: D1 vline\n");
2768
				}
2769
				break;
2770
			default:
2771
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2772
				break;
2773
			}
2774
			break;
2775
		case 2: /* D2 vblank/vline */
2776
			switch (src_data) {
2777
			case 0: /* D2 vblank */
2778
				if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
2779
					if (rdev->irq.crtc_vblank_int[1]) {
2780
				//		drm_handle_vblank(rdev->ddev, 1);
2781
						rdev->pm.vblank_sync = true;
2782
				//		wake_up(&rdev->irq.vblank_queue);
2783
					}
2784
			//		if (rdev->irq.pflip[1])
2785
			//			radeon_crtc_handle_flip(rdev, 1);
2786
					rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
2787
					DRM_DEBUG("IH: D2 vblank\n");
2788
				}
2789
				break;
2790
			case 1: /* D2 vline */
2791
				if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
2792
					rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
2793
					DRM_DEBUG("IH: D2 vline\n");
2794
				}
2795
				break;
2796
			default:
2797
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2798
				break;
2799
			}
2800
			break;
2801
		case 3: /* D3 vblank/vline */
2802
			switch (src_data) {
2803
			case 0: /* D3 vblank */
2804
				if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
2805
					if (rdev->irq.crtc_vblank_int[2]) {
2806
				//		drm_handle_vblank(rdev->ddev, 2);
2807
						rdev->pm.vblank_sync = true;
2808
				//		wake_up(&rdev->irq.vblank_queue);
2809
					}
2810
				//	if (rdev->irq.pflip[2])
2811
				//		radeon_crtc_handle_flip(rdev, 2);
2812
					rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
2813
					DRM_DEBUG("IH: D3 vblank\n");
2814
				}
2815
				break;
2816
			case 1: /* D3 vline */
2817
				if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
2818
					rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
2819
					DRM_DEBUG("IH: D3 vline\n");
2820
				}
2821
				break;
2822
			default:
2823
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2824
				break;
2825
			}
2826
			break;
2827
		case 4: /* D4 vblank/vline */
2828
			switch (src_data) {
2829
			case 0: /* D4 vblank */
2830
				if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
2831
					if (rdev->irq.crtc_vblank_int[3]) {
2832
					//	drm_handle_vblank(rdev->ddev, 3);
2833
						rdev->pm.vblank_sync = true;
2834
					//	wake_up(&rdev->irq.vblank_queue);
2835
					}
2836
		//			if (rdev->irq.pflip[3])
2837
		//				radeon_crtc_handle_flip(rdev, 3);
2838
					rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
2839
					DRM_DEBUG("IH: D4 vblank\n");
2840
				}
2841
				break;
2842
			case 1: /* D4 vline */
2843
				if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
2844
					rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
2845
					DRM_DEBUG("IH: D4 vline\n");
2846
				}
2847
				break;
2848
			default:
2849
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2850
				break;
2851
			}
2852
			break;
2853
		case 5: /* D5 vblank/vline */
2854
			switch (src_data) {
2855
			case 0: /* D5 vblank */
2856
				if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
2857
					if (rdev->irq.crtc_vblank_int[4]) {
2858
//						drm_handle_vblank(rdev->ddev, 4);
2859
						rdev->pm.vblank_sync = true;
2860
//						wake_up(&rdev->irq.vblank_queue);
2861
					}
2862
//					if (rdev->irq.pflip[4])
2863
//						radeon_crtc_handle_flip(rdev, 4);
2864
					rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
2865
					DRM_DEBUG("IH: D5 vblank\n");
2866
				}
2867
				break;
2868
			case 1: /* D5 vline */
2869
				if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
2870
					rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
2871
					DRM_DEBUG("IH: D5 vline\n");
2872
				}
2873
				break;
2874
			default:
2875
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2876
				break;
2877
			}
2878
			break;
2879
		case 6: /* D6 vblank/vline */
2880
			switch (src_data) {
2881
			case 0: /* D6 vblank */
2882
				if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
2883
					if (rdev->irq.crtc_vblank_int[5]) {
2884
				//		drm_handle_vblank(rdev->ddev, 5);
2885
						rdev->pm.vblank_sync = true;
2886
				//		wake_up(&rdev->irq.vblank_queue);
2887
					}
2888
			//		if (rdev->irq.pflip[5])
2889
			//			radeon_crtc_handle_flip(rdev, 5);
2890
					rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
2891
					DRM_DEBUG("IH: D6 vblank\n");
2892
				}
2893
				break;
2894
			case 1: /* D6 vline */
2895
				if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
2896
					rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
2897
					DRM_DEBUG("IH: D6 vline\n");
2898
				}
2899
				break;
2900
			default:
2901
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2902
				break;
2903
			}
2904
			break;
2905
		case 42: /* HPD hotplug */
2906
			switch (src_data) {
2907
			case 0:
2908
				if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2909
					rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
2910
					queue_hotplug = true;
2911
					DRM_DEBUG("IH: HPD1\n");
2912
				}
2913
				break;
2914
			case 1:
2915
				if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2916
					rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
2917
					queue_hotplug = true;
2918
					DRM_DEBUG("IH: HPD2\n");
2919
				}
2920
				break;
2921
			case 2:
2922
				if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2923
					rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
2924
					queue_hotplug = true;
2925
					DRM_DEBUG("IH: HPD3\n");
2926
				}
2927
				break;
2928
			case 3:
2929
				if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2930
					rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
2931
					queue_hotplug = true;
2932
					DRM_DEBUG("IH: HPD4\n");
2933
				}
2934
				break;
2935
			case 4:
2936
				if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2937
					rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
2938
					queue_hotplug = true;
2939
					DRM_DEBUG("IH: HPD5\n");
2940
				}
2941
				break;
2942
			case 5:
2943
				if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2944
					rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
2945
					queue_hotplug = true;
2946
					DRM_DEBUG("IH: HPD6\n");
2947
				}
2948
				break;
2949
			default:
2950
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2951
				break;
2952
			}
2953
			break;
2954
		case 176: /* CP_INT in ring buffer */
2955
		case 177: /* CP_INT in IB1 */
2956
		case 178: /* CP_INT in IB2 */
2957
			DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2958
			radeon_fence_process(rdev);
2959
			break;
2960
		case 181: /* CP EOP event */
2961
			DRM_DEBUG("IH: CP EOP\n");
2962
			radeon_fence_process(rdev);
2963
			break;
2964
		case 233: /* GUI IDLE */
2965
			DRM_DEBUG("IH: GUI idle\n");
2966
			rdev->pm.gui_idle = true;
2967
//			wake_up(&rdev->irq.idle_queue);
2968
			break;
2969
		default:
2970
			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2971
			break;
2972
		}
2973
 
2974
		/* wptr/rptr are in bytes! */
2975
		rptr += 16;
2976
		rptr &= rdev->ih.ptr_mask;
2977
	}
2978
	/* make sure wptr hasn't changed while processing */
2979
	wptr = evergreen_get_ih_wptr(rdev);
2980
	if (wptr != rdev->ih.wptr)
2981
		goto restart_ih;
2982
//	if (queue_hotplug)
2983
//		schedule_work(&rdev->hotplug_work);
2984
	rdev->ih.rptr = rptr;
2985
	WREG32(IH_RB_RPTR, rdev->ih.rptr);
2986
	spin_unlock_irqrestore(&rdev->ih.lock, flags);
2987
	return IRQ_HANDLED;
2988
}
2989
 
1430 serge 2990
static int evergreen_startup(struct radeon_device *rdev)
2991
{
2992
	int r;
2993
 
1990 serge 2994
	/* enable pcie gen2 link */
2995
	if (!ASIC_IS_DCE5(rdev))
2996
		evergreen_pcie_gen2_enable(rdev);
2997
 
2998
	if (ASIC_IS_DCE5(rdev)) {
2999
		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
3000
			r = ni_init_microcode(rdev);
3001
			if (r) {
3002
				DRM_ERROR("Failed to load firmware!\n");
3003
				return r;
3004
			}
3005
		}
3006
		r = ni_mc_load_microcode(rdev);
3007
		if (r) {
3008
			DRM_ERROR("Failed to load MC firmware!\n");
3009
			return r;
3010
		}
3011
	} else {
1430 serge 3012
	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3013
		r = r600_init_microcode(rdev);
3014
		if (r) {
3015
			DRM_ERROR("Failed to load firmware!\n");
3016
			return r;
3017
		}
3018
	}
1990 serge 3019
	}
1963 serge 3020
 
1430 serge 3021
	evergreen_mc_program(rdev);
3022
	if (rdev->flags & RADEON_IS_AGP) {
1963 serge 3023
		evergreen_agp_enable(rdev);
1430 serge 3024
	} else {
3025
		r = evergreen_pcie_gart_enable(rdev);
3026
		if (r)
3027
			return r;
3028
	}
3029
	evergreen_gpu_init(rdev);
2005 serge 3030
 
1963 serge 3031
	r = evergreen_blit_init(rdev);
3032
	if (r) {
3033
		evergreen_blit_fini(rdev);
3034
		rdev->asic->copy = NULL;
3035
		dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3036
	}
1430 serge 3037
 
1963 serge 3038
	/* allocate wb buffer */
3039
	r = radeon_wb_init(rdev);
3040
	if (r)
3041
		return r;
3042
 
3043
	/* Enable IRQ */
2005 serge 3044
	r = r600_irq_init(rdev);
3045
	if (r) {
3046
		DRM_ERROR("radeon: IH init failed (%d).\n", r);
3047
//		radeon_irq_kms_fini(rdev);
3048
		return r;
3049
	}
3050
	evergreen_irq_set(rdev);
1963 serge 3051
 
1430 serge 3052
    r = radeon_ring_init(rdev, rdev->cp.ring_size);
3053
	if (r)
3054
		return r;
3055
	r = evergreen_cp_load_microcode(rdev);
3056
	if (r)
3057
		return r;
1963 serge 3058
	r = evergreen_cp_resume(rdev);
1430 serge 3059
	if (r)
3060
		return r;
1963 serge 3061
 
1430 serge 3062
	return 0;
3063
}
3064
 
3065
 
3066
 
3067
 
3068
 
2005 serge 3069
int evergreen_copy_blit(struct radeon_device *rdev,
3070
			uint64_t src_offset, uint64_t dst_offset,
3071
			unsigned num_pages, struct radeon_fence *fence)
3072
{
3073
	int r;
1430 serge 3074
 
2005 serge 3075
	mutex_lock(&rdev->r600_blit.mutex);
3076
	rdev->r600_blit.vb_ib = NULL;
3077
	r = evergreen_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
3078
	if (r) {
3079
		if (rdev->r600_blit.vb_ib)
3080
			radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
3081
		mutex_unlock(&rdev->r600_blit.mutex);
3082
		return r;
3083
	}
3084
	evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
3085
	evergreen_blit_done_copy(rdev, fence);
3086
	mutex_unlock(&rdev->r600_blit.mutex);
3087
	return 0;
3088
}
1430 serge 3089
 
3090
/* Plan is to move initialization in that function and use
3091
 * helper function so that radeon_device_init pretty much
3092
 * do nothing more than calling asic specific function. This
3093
 * should also allow to remove a bunch of callback function
3094
 * like vram_info.
3095
 */
3096
int evergreen_init(struct radeon_device *rdev)
3097
{
3098
	int r;
3099
 
3100
	/* This don't do much */
3101
	r = radeon_gem_init(rdev);
3102
	if (r)
3103
		return r;
3104
	/* Read BIOS */
3105
	if (!radeon_get_bios(rdev)) {
3106
		if (ASIC_IS_AVIVO(rdev))
3107
			return -EINVAL;
3108
	}
3109
	/* Must be an ATOMBIOS */
3110
	if (!rdev->is_atom_bios) {
1986 serge 3111
		dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
1430 serge 3112
		return -EINVAL;
3113
	}
3114
	r = radeon_atombios_init(rdev);
3115
	if (r)
3116
		return r;
1986 serge 3117
	/* reset the asic, the gfx blocks are often in a bad state
3118
	 * after the driver is unloaded or after a resume
3119
	 */
3120
	if (radeon_asic_reset(rdev))
3121
		dev_warn(rdev->dev, "GPU reset failed !\n");
1430 serge 3122
	/* Post card if necessary */
1986 serge 3123
	if (!radeon_card_posted(rdev)) {
1430 serge 3124
		if (!rdev->bios) {
3125
			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3126
			return -EINVAL;
3127
		}
3128
		DRM_INFO("GPU not posted. posting now...\n");
3129
		atom_asic_init(rdev->mode_info.atom_context);
3130
	}
3131
	/* Initialize scratch registers */
3132
	r600_scratch_init(rdev);
3133
	/* Initialize surface registers */
3134
	radeon_surface_init(rdev);
3135
	/* Initialize clocks */
3136
	radeon_get_clock_info(rdev->ddev);
3137
	/* Fence driver */
2005 serge 3138
	r = radeon_fence_driver_init(rdev);
3139
	if (r)
3140
		return r;
1430 serge 3141
    /* initialize AGP */
3142
	if (rdev->flags & RADEON_IS_AGP) {
3143
		r = radeon_agp_init(rdev);
3144
		if (r)
3145
			radeon_agp_disable(rdev);
3146
	}
3147
	/* initialize memory controller */
3148
	r = evergreen_mc_init(rdev);
3149
	if (r)
3150
		return r;
3151
	/* Memory manager */
3152
	r = radeon_bo_init(rdev);
3153
	if (r)
3154
		return r;
1963 serge 3155
 
2005 serge 3156
	r = radeon_irq_kms_init(rdev);
3157
	if (r)
3158
		return r;
1430 serge 3159
 
3160
	rdev->cp.ring_obj = NULL;
3161
	r600_ring_init(rdev, 1024 * 1024);
3162
 
2005 serge 3163
	rdev->ih.ring_obj = NULL;
3164
	r600_ih_ring_init(rdev, 64 * 1024);
1430 serge 3165
 
3166
	r = r600_pcie_gart_init(rdev);
3167
	if (r)
3168
		return r;
1963 serge 3169
 
3170
	rdev->accel_working = true;
1430 serge 3171
	r = evergreen_startup(rdev);
3172
	if (r) {
1963 serge 3173
		dev_err(rdev->dev, "disabling GPU acceleration\n");
1430 serge 3174
		rdev->accel_working = false;
3175
	}
3176
	if (rdev->accel_working) {
2005 serge 3177
		r = radeon_ib_pool_init(rdev);
3178
		if (r) {
3179
			DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
3180
			rdev->accel_working = false;
3181
		}
3182
		r = r600_ib_test(rdev);
3183
		if (r) {
3184
			DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3185
			rdev->accel_working = false;
3186
		}
1430 serge 3187
	}
3188
	return 0;
3189
}
3190
 
1986 serge 3191
 
3192
static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
1430 serge 3193
{
1986 serge 3194
	u32 link_width_cntl, speed_cntl;
3195
 
3196
	if (radeon_pcie_gen2 == 0)
3197
		return;
3198
 
3199
	if (rdev->flags & RADEON_IS_IGP)
3200
		return;
3201
 
3202
	if (!(rdev->flags & RADEON_IS_PCIE))
3203
		return;
3204
 
3205
	/* x2 cards have a special sequence */
3206
	if (ASIC_IS_X2(rdev))
3207
		return;
3208
 
3209
	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3210
	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
3211
	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3212
 
3213
		link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3214
		link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3215
		WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3216
 
3217
		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3218
		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3219
		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3220
 
3221
		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3222
		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3223
		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3224
 
3225
		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3226
		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3227
		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3228
 
3229
		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3230
		speed_cntl |= LC_GEN2_EN_STRAP;
3231
		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3232
 
3233
	} else {
3234
		link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3235
		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
3236
		if (1)
3237
			link_width_cntl |= LC_UPCONFIGURE_DIS;
3238
		else
3239
			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3240
		WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3241
	}
1430 serge 3242
}