Subversion Repositories Kolibri OS

Rev

Rev 2160 | Rev 2997 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
1430 serge 1
/*
2
 * Copyright 2010 Advanced Micro Devices, Inc.
3
 *
4
 * Permission is hereby granted, free of charge, to any person obtaining a
5
 * copy of this software and associated documentation files (the "Software"),
6
 * to deal in the Software without restriction, including without limitation
7
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
 * and/or sell copies of the Software, and to permit persons to whom the
9
 * Software is furnished to do so, subject to the following conditions:
10
 *
11
 * The above copyright notice and this permission notice shall be included in
12
 * all copies or substantial portions of the Software.
13
 *
14
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20
 * OTHER DEALINGS IN THE SOFTWARE.
21
 *
22
 * Authors: Alex Deucher
23
 */
24
#include 
25
//#include 
1963 serge 26
#include 
1430 serge 27
#include "drmP.h"
28
#include "radeon.h"
1963 serge 29
#include "radeon_asic.h"
1430 serge 30
#include "radeon_drm.h"
1963 serge 31
#include "evergreend.h"
1430 serge 32
#include "atom.h"
33
#include "avivod.h"
34
#include "evergreen_reg.h"
1986 serge 35
#include "evergreen_blit_shaders.h"
1430 serge 36
 
1963 serge 37
#define EVERGREEN_PFP_UCODE_SIZE 1120
38
#define EVERGREEN_PM4_UCODE_SIZE 1376
39
 
1430 serge 40
static void evergreen_gpu_init(struct radeon_device *rdev);
41
void evergreen_fini(struct radeon_device *rdev);
1986 serge 42
static void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
1430 serge 43
 
1990 serge 44
 
45
 
46
u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
47
{
48
	struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
49
	u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
50
 
51
	/* Lock the graphics update lock */
52
	tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
53
	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
54
 
55
	/* update the scanout addresses */
56
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
57
	       upper_32_bits(crtc_base));
58
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
59
	       (u32)crtc_base);
60
 
61
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
62
	       upper_32_bits(crtc_base));
63
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
64
	       (u32)crtc_base);
65
 
66
	/* Wait for update_pending to go high. */
67
	while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING));
68
	DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
69
 
70
	/* Unlock the lock, so double-buffering can take place inside vblank */
71
	tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
72
	WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
73
 
74
	/* Return current update_pending status: */
75
	return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
76
}
77
 
78
/* get temperature in millidegrees */
79
int evergreen_get_temp(struct radeon_device *rdev)
80
{
81
	u32 temp, toffset;
82
	int actual_temp = 0;
83
 
84
	if (rdev->family == CHIP_JUNIPER) {
85
		toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
86
			TOFFSET_SHIFT;
87
		temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
88
			TS0_ADC_DOUT_SHIFT;
89
 
90
		if (toffset & 0x100)
91
			actual_temp = temp / 2 - (0x200 - toffset);
92
		else
93
			actual_temp = temp / 2 + toffset;
94
 
95
		actual_temp = actual_temp * 1000;
96
 
97
	} else {
98
		temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
99
			ASIC_T_SHIFT;
100
 
101
		if (temp & 0x400)
102
			actual_temp = -256;
103
		else if (temp & 0x200)
104
			actual_temp = 255;
105
		else if (temp & 0x100) {
106
			actual_temp = temp & 0x1ff;
107
			actual_temp |= ~0x1ff;
108
		} else
109
			actual_temp = temp & 0xff;
110
 
111
		actual_temp = (actual_temp * 1000) / 2;
112
	}
113
 
114
	return actual_temp;
115
}
116
 
117
int sumo_get_temp(struct radeon_device *rdev)
118
{
119
	u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
120
	int actual_temp = temp - 49;
121
 
122
	return actual_temp * 1000;
123
}
124
 
125
void evergreen_pm_misc(struct radeon_device *rdev)
126
{
127
	int req_ps_idx = rdev->pm.requested_power_state_index;
128
	int req_cm_idx = rdev->pm.requested_clock_mode_index;
129
	struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
130
	struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
131
 
132
	if (voltage->type == VOLTAGE_SW) {
133
		/* 0xff01 is a flag rather then an actual voltage */
134
		if (voltage->voltage == 0xff01)
135
			return;
136
		if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
137
			radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
138
			rdev->pm.current_vddc = voltage->voltage;
139
			DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
140
		}
141
		/* 0xff01 is a flag rather then an actual voltage */
142
		if (voltage->vddci == 0xff01)
143
			return;
144
		if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
145
			radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
146
			rdev->pm.current_vddci = voltage->vddci;
147
			DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
148
		}
149
	}
150
}
151
 
152
void evergreen_pm_prepare(struct radeon_device *rdev)
153
{
154
	struct drm_device *ddev = rdev->ddev;
155
	struct drm_crtc *crtc;
156
	struct radeon_crtc *radeon_crtc;
157
	u32 tmp;
158
 
159
	/* disable any active CRTCs */
160
	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
161
		radeon_crtc = to_radeon_crtc(crtc);
162
		if (radeon_crtc->enabled) {
163
			tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
164
			tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
165
			WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
166
		}
167
	}
168
}
169
 
170
void evergreen_pm_finish(struct radeon_device *rdev)
171
{
172
	struct drm_device *ddev = rdev->ddev;
173
	struct drm_crtc *crtc;
174
	struct radeon_crtc *radeon_crtc;
175
	u32 tmp;
176
 
177
	/* enable any active CRTCs */
178
	list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
179
		radeon_crtc = to_radeon_crtc(crtc);
180
		if (radeon_crtc->enabled) {
181
			tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
182
			tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
183
			WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
184
		}
185
	}
186
}
187
 
1430 serge 188
bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
189
{
190
	bool connected = false;
1963 serge 191
 
192
	switch (hpd) {
193
	case RADEON_HPD_1:
194
		if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
195
			connected = true;
196
		break;
197
	case RADEON_HPD_2:
198
		if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
199
			connected = true;
200
		break;
201
	case RADEON_HPD_3:
202
		if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
203
			connected = true;
204
		break;
205
	case RADEON_HPD_4:
206
		if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
207
			connected = true;
208
		break;
209
	case RADEON_HPD_5:
210
		if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
211
			connected = true;
212
		break;
213
	case RADEON_HPD_6:
214
		if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
215
			connected = true;
216
			break;
217
	default:
218
		break;
219
	}
220
 
1430 serge 221
	return connected;
222
}
223
 
224
void evergreen_hpd_set_polarity(struct radeon_device *rdev,
225
				enum radeon_hpd_id hpd)
226
{
1963 serge 227
	u32 tmp;
228
	bool connected = evergreen_hpd_sense(rdev, hpd);
229
 
230
	switch (hpd) {
231
	case RADEON_HPD_1:
232
		tmp = RREG32(DC_HPD1_INT_CONTROL);
233
		if (connected)
234
			tmp &= ~DC_HPDx_INT_POLARITY;
235
		else
236
			tmp |= DC_HPDx_INT_POLARITY;
237
		WREG32(DC_HPD1_INT_CONTROL, tmp);
238
		break;
239
	case RADEON_HPD_2:
240
		tmp = RREG32(DC_HPD2_INT_CONTROL);
241
		if (connected)
242
			tmp &= ~DC_HPDx_INT_POLARITY;
243
		else
244
			tmp |= DC_HPDx_INT_POLARITY;
245
		WREG32(DC_HPD2_INT_CONTROL, tmp);
246
		break;
247
	case RADEON_HPD_3:
248
		tmp = RREG32(DC_HPD3_INT_CONTROL);
249
		if (connected)
250
			tmp &= ~DC_HPDx_INT_POLARITY;
251
		else
252
			tmp |= DC_HPDx_INT_POLARITY;
253
		WREG32(DC_HPD3_INT_CONTROL, tmp);
254
		break;
255
	case RADEON_HPD_4:
256
		tmp = RREG32(DC_HPD4_INT_CONTROL);
257
		if (connected)
258
			tmp &= ~DC_HPDx_INT_POLARITY;
259
		else
260
			tmp |= DC_HPDx_INT_POLARITY;
261
		WREG32(DC_HPD4_INT_CONTROL, tmp);
262
		break;
263
	case RADEON_HPD_5:
264
		tmp = RREG32(DC_HPD5_INT_CONTROL);
265
		if (connected)
266
			tmp &= ~DC_HPDx_INT_POLARITY;
267
		else
268
			tmp |= DC_HPDx_INT_POLARITY;
269
		WREG32(DC_HPD5_INT_CONTROL, tmp);
270
			break;
271
	case RADEON_HPD_6:
272
		tmp = RREG32(DC_HPD6_INT_CONTROL);
273
		if (connected)
274
			tmp &= ~DC_HPDx_INT_POLARITY;
275
		else
276
			tmp |= DC_HPDx_INT_POLARITY;
277
		WREG32(DC_HPD6_INT_CONTROL, tmp);
278
		break;
279
	default:
280
		break;
281
	}
1430 serge 282
}
283
 
284
void evergreen_hpd_init(struct radeon_device *rdev)
285
{
1963 serge 286
	struct drm_device *dev = rdev->ddev;
287
	struct drm_connector *connector;
288
	u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
289
		DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
290
 
291
	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
292
		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
293
		switch (radeon_connector->hpd.hpd) {
294
		case RADEON_HPD_1:
295
			WREG32(DC_HPD1_CONTROL, tmp);
296
			rdev->irq.hpd[0] = true;
297
			break;
298
		case RADEON_HPD_2:
299
			WREG32(DC_HPD2_CONTROL, tmp);
300
			rdev->irq.hpd[1] = true;
301
			break;
302
		case RADEON_HPD_3:
303
			WREG32(DC_HPD3_CONTROL, tmp);
304
			rdev->irq.hpd[2] = true;
305
			break;
306
		case RADEON_HPD_4:
307
			WREG32(DC_HPD4_CONTROL, tmp);
308
			rdev->irq.hpd[3] = true;
309
			break;
310
		case RADEON_HPD_5:
311
			WREG32(DC_HPD5_CONTROL, tmp);
312
			rdev->irq.hpd[4] = true;
313
			break;
314
		case RADEON_HPD_6:
315
			WREG32(DC_HPD6_CONTROL, tmp);
316
			rdev->irq.hpd[5] = true;
317
			break;
318
		default:
319
			break;
320
		}
321
	}
2005 serge 322
	if (rdev->irq.installed)
323
		evergreen_irq_set(rdev);
1430 serge 324
}
325
 
1963 serge 326
void evergreen_hpd_fini(struct radeon_device *rdev)
327
{
328
	struct drm_device *dev = rdev->ddev;
329
	struct drm_connector *connector;
1430 serge 330
 
1963 serge 331
	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
332
		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
333
		switch (radeon_connector->hpd.hpd) {
334
		case RADEON_HPD_1:
335
			WREG32(DC_HPD1_CONTROL, 0);
336
			rdev->irq.hpd[0] = false;
337
			break;
338
		case RADEON_HPD_2:
339
			WREG32(DC_HPD2_CONTROL, 0);
340
			rdev->irq.hpd[1] = false;
341
			break;
342
		case RADEON_HPD_3:
343
			WREG32(DC_HPD3_CONTROL, 0);
344
			rdev->irq.hpd[2] = false;
345
			break;
346
		case RADEON_HPD_4:
347
			WREG32(DC_HPD4_CONTROL, 0);
348
			rdev->irq.hpd[3] = false;
349
			break;
350
		case RADEON_HPD_5:
351
			WREG32(DC_HPD5_CONTROL, 0);
352
			rdev->irq.hpd[4] = false;
353
			break;
354
		case RADEON_HPD_6:
355
			WREG32(DC_HPD6_CONTROL, 0);
356
			rdev->irq.hpd[5] = false;
357
			break;
358
		default:
359
			break;
360
		}
361
	}
1430 serge 362
}
363
 
1986 serge 364
/* watermark setup */
1963 serge 365
 
1986 serge 366
static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
367
					struct radeon_crtc *radeon_crtc,
368
					struct drm_display_mode *mode,
369
					struct drm_display_mode *other_mode)
370
{
371
	u32 tmp;
372
	/*
373
	 * Line Buffer Setup
374
	 * There are 3 line buffers, each one shared by 2 display controllers.
375
	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
376
	 * the display controllers.  The paritioning is done via one of four
377
	 * preset allocations specified in bits 2:0:
378
	 * first display controller
379
	 *  0 - first half of lb (3840 * 2)
380
	 *  1 - first 3/4 of lb (5760 * 2)
381
	 *  2 - whole lb (7680 * 2), other crtc must be disabled
382
	 *  3 - first 1/4 of lb (1920 * 2)
383
	 * second display controller
384
	 *  4 - second half of lb (3840 * 2)
385
	 *  5 - second 3/4 of lb (5760 * 2)
386
	 *  6 - whole lb (7680 * 2), other crtc must be disabled
387
	 *  7 - last 1/4 of lb (1920 * 2)
388
	 */
389
	/* this can get tricky if we have two large displays on a paired group
390
	 * of crtcs.  Ideally for multiple large displays we'd assign them to
391
	 * non-linked crtcs for maximum line buffer allocation.
392
	 */
393
	if (radeon_crtc->base.enabled && mode) {
394
		if (other_mode)
395
			tmp = 0; /* 1/2 */
396
		else
397
			tmp = 2; /* whole */
398
	} else
399
		tmp = 0;
1963 serge 400
 
1986 serge 401
	/* second controller of the pair uses second half of the lb */
402
	if (radeon_crtc->crtc_id % 2)
403
		tmp += 4;
404
	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
405
 
406
	if (radeon_crtc->base.enabled && mode) {
407
		switch (tmp) {
408
		case 0:
409
		case 4:
410
		default:
411
			if (ASIC_IS_DCE5(rdev))
412
				return 4096 * 2;
413
			else
414
				return 3840 * 2;
415
		case 1:
416
		case 5:
417
			if (ASIC_IS_DCE5(rdev))
418
				return 6144 * 2;
419
			else
420
				return 5760 * 2;
421
		case 2:
422
		case 6:
423
			if (ASIC_IS_DCE5(rdev))
424
				return 8192 * 2;
425
			else
426
				return 7680 * 2;
427
		case 3:
428
		case 7:
429
			if (ASIC_IS_DCE5(rdev))
430
				return 2048 * 2;
431
			else
432
				return 1920 * 2;
433
		}
434
	}
435
 
436
	/* controller not enabled, so no lb used */
437
	return 0;
438
}
439
 
440
static u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
441
{
442
	u32 tmp = RREG32(MC_SHARED_CHMAP);
443
 
444
	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
445
	case 0:
446
	default:
447
		return 1;
448
	case 1:
449
		return 2;
450
	case 2:
451
		return 4;
452
	case 3:
453
		return 8;
454
	}
455
}
456
 
457
struct evergreen_wm_params {
458
	u32 dram_channels; /* number of dram channels */
459
	u32 yclk;          /* bandwidth per dram data pin in kHz */
460
	u32 sclk;          /* engine clock in kHz */
461
	u32 disp_clk;      /* display clock in kHz */
462
	u32 src_width;     /* viewport width */
463
	u32 active_time;   /* active display time in ns */
464
	u32 blank_time;    /* blank time in ns */
465
	bool interlaced;    /* mode is interlaced */
466
	fixed20_12 vsc;    /* vertical scale ratio */
467
	u32 num_heads;     /* number of active crtcs */
468
	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
469
	u32 lb_size;       /* line buffer allocated to pipe */
470
	u32 vtaps;         /* vertical scaler taps */
471
};
472
 
473
static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
474
{
475
	/* Calculate DRAM Bandwidth and the part allocated to display. */
476
	fixed20_12 dram_efficiency; /* 0.7 */
477
	fixed20_12 yclk, dram_channels, bandwidth;
478
	fixed20_12 a;
479
 
480
	a.full = dfixed_const(1000);
481
	yclk.full = dfixed_const(wm->yclk);
482
	yclk.full = dfixed_div(yclk, a);
483
	dram_channels.full = dfixed_const(wm->dram_channels * 4);
484
	a.full = dfixed_const(10);
485
	dram_efficiency.full = dfixed_const(7);
486
	dram_efficiency.full = dfixed_div(dram_efficiency, a);
487
	bandwidth.full = dfixed_mul(dram_channels, yclk);
488
	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
489
 
490
	return dfixed_trunc(bandwidth);
491
}
492
 
493
static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
494
{
495
	/* Calculate DRAM Bandwidth and the part allocated to display. */
496
	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
497
	fixed20_12 yclk, dram_channels, bandwidth;
498
	fixed20_12 a;
499
 
500
	a.full = dfixed_const(1000);
501
	yclk.full = dfixed_const(wm->yclk);
502
	yclk.full = dfixed_div(yclk, a);
503
	dram_channels.full = dfixed_const(wm->dram_channels * 4);
504
	a.full = dfixed_const(10);
505
	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
506
	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
507
	bandwidth.full = dfixed_mul(dram_channels, yclk);
508
	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
509
 
510
	return dfixed_trunc(bandwidth);
511
}
512
 
513
static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
514
{
515
	/* Calculate the display Data return Bandwidth */
516
	fixed20_12 return_efficiency; /* 0.8 */
517
	fixed20_12 sclk, bandwidth;
518
	fixed20_12 a;
519
 
520
	a.full = dfixed_const(1000);
521
	sclk.full = dfixed_const(wm->sclk);
522
	sclk.full = dfixed_div(sclk, a);
523
	a.full = dfixed_const(10);
524
	return_efficiency.full = dfixed_const(8);
525
	return_efficiency.full = dfixed_div(return_efficiency, a);
526
	a.full = dfixed_const(32);
527
	bandwidth.full = dfixed_mul(a, sclk);
528
	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
529
 
530
	return dfixed_trunc(bandwidth);
531
}
532
 
533
static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
534
{
535
	/* Calculate the DMIF Request Bandwidth */
536
	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
537
	fixed20_12 disp_clk, bandwidth;
538
	fixed20_12 a;
539
 
540
	a.full = dfixed_const(1000);
541
	disp_clk.full = dfixed_const(wm->disp_clk);
542
	disp_clk.full = dfixed_div(disp_clk, a);
543
	a.full = dfixed_const(10);
544
	disp_clk_request_efficiency.full = dfixed_const(8);
545
	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
546
	a.full = dfixed_const(32);
547
	bandwidth.full = dfixed_mul(a, disp_clk);
548
	bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
549
 
550
	return dfixed_trunc(bandwidth);
551
}
552
 
553
static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
554
{
555
	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
556
	u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
557
	u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
558
	u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
559
 
560
	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
561
}
562
 
563
static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
564
{
565
	/* Calculate the display mode Average Bandwidth
566
	 * DisplayMode should contain the source and destination dimensions,
567
	 * timing, etc.
568
	 */
569
	fixed20_12 bpp;
570
	fixed20_12 line_time;
571
	fixed20_12 src_width;
572
	fixed20_12 bandwidth;
573
	fixed20_12 a;
574
 
575
	a.full = dfixed_const(1000);
576
	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
577
	line_time.full = dfixed_div(line_time, a);
578
	bpp.full = dfixed_const(wm->bytes_per_pixel);
579
	src_width.full = dfixed_const(wm->src_width);
580
	bandwidth.full = dfixed_mul(src_width, bpp);
581
	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
582
	bandwidth.full = dfixed_div(bandwidth, line_time);
583
 
584
	return dfixed_trunc(bandwidth);
585
}
586
 
587
static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
588
{
589
	/* First calcualte the latency in ns */
590
	u32 mc_latency = 2000; /* 2000 ns. */
591
	u32 available_bandwidth = evergreen_available_bandwidth(wm);
592
	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
593
	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
594
	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
595
	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
596
		(wm->num_heads * cursor_line_pair_return_time);
597
	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
598
	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
599
	fixed20_12 a, b, c;
600
 
601
	if (wm->num_heads == 0)
602
		return 0;
603
 
604
	a.full = dfixed_const(2);
605
	b.full = dfixed_const(1);
606
	if ((wm->vsc.full > a.full) ||
607
	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
608
	    (wm->vtaps >= 5) ||
609
	    ((wm->vsc.full >= a.full) && wm->interlaced))
610
		max_src_lines_per_dst_line = 4;
611
	else
612
		max_src_lines_per_dst_line = 2;
613
 
614
	a.full = dfixed_const(available_bandwidth);
615
	b.full = dfixed_const(wm->num_heads);
616
	a.full = dfixed_div(a, b);
617
 
618
	b.full = dfixed_const(1000);
619
	c.full = dfixed_const(wm->disp_clk);
620
	b.full = dfixed_div(c, b);
621
	c.full = dfixed_const(wm->bytes_per_pixel);
622
	b.full = dfixed_mul(b, c);
623
 
624
	lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
625
 
626
	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
627
	b.full = dfixed_const(1000);
628
	c.full = dfixed_const(lb_fill_bw);
629
	b.full = dfixed_div(c, b);
630
	a.full = dfixed_div(a, b);
631
	line_fill_time = dfixed_trunc(a);
632
 
633
	if (line_fill_time < wm->active_time)
634
		return latency;
635
	else
636
		return latency + (line_fill_time - wm->active_time);
637
 
638
}
639
 
640
static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
641
{
642
	if (evergreen_average_bandwidth(wm) <=
643
	    (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
644
		return true;
645
	else
646
		return false;
647
};
648
 
649
static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
650
{
651
	if (evergreen_average_bandwidth(wm) <=
652
	    (evergreen_available_bandwidth(wm) / wm->num_heads))
653
		return true;
654
	else
655
		return false;
656
};
657
 
658
static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
659
{
660
	u32 lb_partitions = wm->lb_size / wm->src_width;
661
	u32 line_time = wm->active_time + wm->blank_time;
662
	u32 latency_tolerant_lines;
663
	u32 latency_hiding;
664
	fixed20_12 a;
665
 
666
	a.full = dfixed_const(1);
667
	if (wm->vsc.full > a.full)
668
		latency_tolerant_lines = 1;
669
	else {
670
		if (lb_partitions <= (wm->vtaps + 1))
671
			latency_tolerant_lines = 1;
672
		else
673
			latency_tolerant_lines = 2;
674
	}
675
 
676
	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
677
 
678
	if (evergreen_latency_watermark(wm) <= latency_hiding)
679
		return true;
680
	else
681
		return false;
682
}
683
 
684
static void evergreen_program_watermarks(struct radeon_device *rdev,
685
					 struct radeon_crtc *radeon_crtc,
686
					 u32 lb_size, u32 num_heads)
687
{
688
	struct drm_display_mode *mode = &radeon_crtc->base.mode;
689
	struct evergreen_wm_params wm;
690
	u32 pixel_period;
691
	u32 line_time = 0;
692
	u32 latency_watermark_a = 0, latency_watermark_b = 0;
693
	u32 priority_a_mark = 0, priority_b_mark = 0;
694
	u32 priority_a_cnt = PRIORITY_OFF;
695
	u32 priority_b_cnt = PRIORITY_OFF;
696
	u32 pipe_offset = radeon_crtc->crtc_id * 16;
697
	u32 tmp, arb_control3;
698
	fixed20_12 a, b, c;
699
 
700
	if (radeon_crtc->base.enabled && num_heads && mode) {
701
		pixel_period = 1000000 / (u32)mode->clock;
702
		line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
703
		priority_a_cnt = 0;
704
		priority_b_cnt = 0;
705
 
706
		wm.yclk = rdev->pm.current_mclk * 10;
707
		wm.sclk = rdev->pm.current_sclk * 10;
708
		wm.disp_clk = mode->clock;
709
		wm.src_width = mode->crtc_hdisplay;
710
		wm.active_time = mode->crtc_hdisplay * pixel_period;
711
		wm.blank_time = line_time - wm.active_time;
712
		wm.interlaced = false;
713
		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
714
			wm.interlaced = true;
715
		wm.vsc = radeon_crtc->vsc;
716
		wm.vtaps = 1;
717
		if (radeon_crtc->rmx_type != RMX_OFF)
718
			wm.vtaps = 2;
719
		wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
720
		wm.lb_size = lb_size;
721
		wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
722
		wm.num_heads = num_heads;
723
 
724
		/* set for high clocks */
725
		latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535);
726
		/* set for low clocks */
727
		/* wm.yclk = low clk; wm.sclk = low clk */
728
		latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535);
729
 
730
		/* possibly force display priority to high */
731
		/* should really do this at mode validation time... */
732
		if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
733
		    !evergreen_average_bandwidth_vs_available_bandwidth(&wm) ||
734
		    !evergreen_check_latency_hiding(&wm) ||
735
		    (rdev->disp_priority == 2)) {
2160 serge 736
			DRM_DEBUG_KMS("force priority to high\n");
1986 serge 737
			priority_a_cnt |= PRIORITY_ALWAYS_ON;
738
			priority_b_cnt |= PRIORITY_ALWAYS_ON;
739
		}
740
 
741
		a.full = dfixed_const(1000);
742
		b.full = dfixed_const(mode->clock);
743
		b.full = dfixed_div(b, a);
744
		c.full = dfixed_const(latency_watermark_a);
745
		c.full = dfixed_mul(c, b);
746
		c.full = dfixed_mul(c, radeon_crtc->hsc);
747
		c.full = dfixed_div(c, a);
748
		a.full = dfixed_const(16);
749
		c.full = dfixed_div(c, a);
750
		priority_a_mark = dfixed_trunc(c);
751
		priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
752
 
753
		a.full = dfixed_const(1000);
754
		b.full = dfixed_const(mode->clock);
755
		b.full = dfixed_div(b, a);
756
		c.full = dfixed_const(latency_watermark_b);
757
		c.full = dfixed_mul(c, b);
758
		c.full = dfixed_mul(c, radeon_crtc->hsc);
759
		c.full = dfixed_div(c, a);
760
		a.full = dfixed_const(16);
761
		c.full = dfixed_div(c, a);
762
		priority_b_mark = dfixed_trunc(c);
763
		priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
764
	}
765
 
766
	/* select wm A */
767
	arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
768
	tmp = arb_control3;
769
	tmp &= ~LATENCY_WATERMARK_MASK(3);
770
	tmp |= LATENCY_WATERMARK_MASK(1);
771
	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
772
	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
773
	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
774
		LATENCY_HIGH_WATERMARK(line_time)));
775
	/* select wm B */
776
	tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
777
	tmp &= ~LATENCY_WATERMARK_MASK(3);
778
	tmp |= LATENCY_WATERMARK_MASK(2);
779
	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
780
	WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
781
	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
782
		LATENCY_HIGH_WATERMARK(line_time)));
783
	/* restore original selection */
784
	WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
785
 
786
	/* write the priority marks */
787
	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
788
	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
789
 
790
}
791
 
1963 serge 792
void evergreen_bandwidth_update(struct radeon_device *rdev)
1430 serge 793
{
1986 serge 794
	struct drm_display_mode *mode0 = NULL;
795
	struct drm_display_mode *mode1 = NULL;
796
	u32 num_heads = 0, lb_size;
797
	int i;
798
 
799
	radeon_update_display_priority(rdev);
800
 
801
	for (i = 0; i < rdev->num_crtc; i++) {
802
		if (rdev->mode_info.crtcs[i]->base.enabled)
803
			num_heads++;
804
	}
805
	for (i = 0; i < rdev->num_crtc; i += 2) {
806
		mode0 = &rdev->mode_info.crtcs[i]->base.mode;
807
		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
808
		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
809
		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
810
		lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
811
		evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
812
	}
1430 serge 813
}
814
 
1963 serge 815
int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
1430 serge 816
{
817
	unsigned i;
818
	u32 tmp;
819
 
820
	for (i = 0; i < rdev->usec_timeout; i++) {
821
		/* read MC_STATUS */
822
		tmp = RREG32(SRBM_STATUS) & 0x1F00;
823
		if (!tmp)
824
			return 0;
825
		udelay(1);
826
	}
827
	return -1;
828
}
829
 
830
/*
831
 * GART
832
 */
1963 serge 833
void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
834
{
835
	unsigned i;
836
	u32 tmp;
837
 
838
	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
839
 
840
	WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
841
	for (i = 0; i < rdev->usec_timeout; i++) {
842
		/* read MC_STATUS */
843
		tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
844
		tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
845
		if (tmp == 2) {
846
			printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
847
			return;
848
		}
849
		if (tmp) {
850
			return;
851
		}
852
		udelay(1);
853
	}
854
}
855
 
1430 serge 856
int evergreen_pcie_gart_enable(struct radeon_device *rdev)
857
{
858
	u32 tmp;
1963 serge 859
	int r;
1430 serge 860
 
861
	if (rdev->gart.table.vram.robj == NULL) {
862
		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
863
		return -EINVAL;
864
	}
865
	r = radeon_gart_table_vram_pin(rdev);
866
	if (r)
867
		return r;
868
	radeon_gart_restore(rdev);
869
	/* Setup L2 cache */
870
	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
871
				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
872
				EFFECTIVE_L2_QUEUE_SIZE(7));
873
	WREG32(VM_L2_CNTL2, 0);
874
	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
875
	/* Setup TLB control */
876
	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
877
		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
878
		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
879
		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
1963 serge 880
	if (rdev->flags & RADEON_IS_IGP) {
881
		WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
882
		WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
883
		WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
884
	} else {
1430 serge 885
	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
886
	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
887
	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
1963 serge 888
	}
1430 serge 889
	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
890
	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
891
	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
892
	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
893
	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
894
	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
895
	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
896
	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
897
				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
898
	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
899
			(u32)(rdev->dummy_page.addr >> 12));
1963 serge 900
	WREG32(VM_CONTEXT1_CNTL, 0);
1430 serge 901
 
1963 serge 902
	evergreen_pcie_gart_tlb_flush(rdev);
1430 serge 903
	rdev->gart.ready = true;
904
	return 0;
905
}
906
 
907
void evergreen_pcie_gart_disable(struct radeon_device *rdev)
908
{
909
	u32 tmp;
1963 serge 910
	int r;
1430 serge 911
 
912
	/* Disable all tables */
1963 serge 913
	WREG32(VM_CONTEXT0_CNTL, 0);
914
	WREG32(VM_CONTEXT1_CNTL, 0);
1430 serge 915
 
916
	/* Setup L2 cache */
917
	WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
918
				EFFECTIVE_L2_QUEUE_SIZE(7));
919
	WREG32(VM_L2_CNTL2, 0);
920
	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
921
	/* Setup TLB control */
922
	tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
923
	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
924
	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
925
	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
926
	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
927
	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
928
	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
929
	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
930
	if (rdev->gart.table.vram.robj) {
931
		r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
932
		if (likely(r == 0)) {
933
			radeon_bo_kunmap(rdev->gart.table.vram.robj);
934
			radeon_bo_unpin(rdev->gart.table.vram.robj);
935
			radeon_bo_unreserve(rdev->gart.table.vram.robj);
936
		}
937
	}
938
}
939
 
940
void evergreen_pcie_gart_fini(struct radeon_device *rdev)
941
{
942
	evergreen_pcie_gart_disable(rdev);
943
	radeon_gart_table_vram_free(rdev);
944
	radeon_gart_fini(rdev);
945
}
946
 
947
 
948
void evergreen_agp_enable(struct radeon_device *rdev)
949
{
950
	u32 tmp;
951
 
952
	/* Setup L2 cache */
953
	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
954
				ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
955
				EFFECTIVE_L2_QUEUE_SIZE(7));
956
	WREG32(VM_L2_CNTL2, 0);
957
	WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
958
	/* Setup TLB control */
959
	tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
960
		SYSTEM_ACCESS_MODE_NOT_IN_SYS |
961
		SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
962
		EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
963
	WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
964
	WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
965
	WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
966
	WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
967
	WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
968
	WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
969
	WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
1963 serge 970
	WREG32(VM_CONTEXT0_CNTL, 0);
971
	WREG32(VM_CONTEXT1_CNTL, 0);
1430 serge 972
}
973
 
1963 serge 974
void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
1430 serge 975
{
976
	save->vga_control[0] = RREG32(D1VGA_CONTROL);
977
	save->vga_control[1] = RREG32(D2VGA_CONTROL);
978
	save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
979
	save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
980
	save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
981
	save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
2005 serge 982
	if (rdev->num_crtc >= 4) {
983
		save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL);
984
		save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL);
1430 serge 985
	save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
986
	save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
2005 serge 987
	}
988
	if (rdev->num_crtc >= 6) {
989
		save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL);
990
		save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL);
1430 serge 991
	save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
992
	save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
1963 serge 993
	}
1430 serge 994
 
995
	/* Stop all video */
996
	WREG32(VGA_RENDER_CONTROL, 0);
997
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
998
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
2005 serge 999
	if (rdev->num_crtc >= 4) {
1430 serge 1000
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
1001
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
2005 serge 1002
	}
1003
	if (rdev->num_crtc >= 6) {
1430 serge 1004
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1005
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1963 serge 1006
	}
1430 serge 1007
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1008
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005 serge 1009
	if (rdev->num_crtc >= 4) {
1430 serge 1010
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1011
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005 serge 1012
	}
1013
	if (rdev->num_crtc >= 6) {
1430 serge 1014
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1015
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963 serge 1016
	}
1430 serge 1017
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1018
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005 serge 1019
	if (rdev->num_crtc >= 4) {
1430 serge 1020
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1021
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005 serge 1022
	}
1023
	if (rdev->num_crtc >= 6) {
1430 serge 1024
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1025
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963 serge 1026
	}
1430 serge 1027
 
1028
	WREG32(D1VGA_CONTROL, 0);
1029
	WREG32(D2VGA_CONTROL, 0);
2005 serge 1030
	if (rdev->num_crtc >= 4) {
1430 serge 1031
	WREG32(EVERGREEN_D3VGA_CONTROL, 0);
1032
	WREG32(EVERGREEN_D4VGA_CONTROL, 0);
2005 serge 1033
	}
1034
	if (rdev->num_crtc >= 6) {
1430 serge 1035
	WREG32(EVERGREEN_D5VGA_CONTROL, 0);
1036
	WREG32(EVERGREEN_D6VGA_CONTROL, 0);
2005 serge 1037
	}
1430 serge 1038
}
1039
 
1963 serge 1040
void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
1430 serge 1041
{
1042
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
1043
	       upper_32_bits(rdev->mc.vram_start));
1044
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET,
1045
	       upper_32_bits(rdev->mc.vram_start));
1046
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1047
	       (u32)rdev->mc.vram_start);
1048
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET,
1049
	       (u32)rdev->mc.vram_start);
1050
 
1051
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
1052
	       upper_32_bits(rdev->mc.vram_start));
1053
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET,
1054
	       upper_32_bits(rdev->mc.vram_start));
1055
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1056
	       (u32)rdev->mc.vram_start);
1057
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET,
1058
	       (u32)rdev->mc.vram_start);
1059
 
2005 serge 1060
	if (rdev->num_crtc >= 4) {
1430 serge 1061
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
1062
	       upper_32_bits(rdev->mc.vram_start));
1063
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET,
1064
	       upper_32_bits(rdev->mc.vram_start));
1065
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1066
	       (u32)rdev->mc.vram_start);
1067
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET,
1068
	       (u32)rdev->mc.vram_start);
1069
 
1070
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
1071
	       upper_32_bits(rdev->mc.vram_start));
1072
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET,
1073
	       upper_32_bits(rdev->mc.vram_start));
1074
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1075
	       (u32)rdev->mc.vram_start);
1076
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET,
1077
	       (u32)rdev->mc.vram_start);
2005 serge 1078
	}
1079
	if (rdev->num_crtc >= 6) {
1430 serge 1080
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
1081
	       upper_32_bits(rdev->mc.vram_start));
1082
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET,
1083
	       upper_32_bits(rdev->mc.vram_start));
1084
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1085
	       (u32)rdev->mc.vram_start);
1086
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET,
1087
	       (u32)rdev->mc.vram_start);
1088
 
1089
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
1090
	       upper_32_bits(rdev->mc.vram_start));
1091
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET,
1092
	       upper_32_bits(rdev->mc.vram_start));
1093
	WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1094
	       (u32)rdev->mc.vram_start);
1095
	WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET,
1096
	       (u32)rdev->mc.vram_start);
1963 serge 1097
	}
1430 serge 1098
 
1099
	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
1100
	WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
1101
	/* Unlock host access */
1102
	WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
1103
	mdelay(1);
1104
	/* Restore video state */
1105
	WREG32(D1VGA_CONTROL, save->vga_control[0]);
1106
	WREG32(D2VGA_CONTROL, save->vga_control[1]);
2005 serge 1107
	if (rdev->num_crtc >= 4) {
1430 serge 1108
	WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]);
1109
	WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]);
2005 serge 1110
	}
1111
	if (rdev->num_crtc >= 6) {
1430 serge 1112
	WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]);
1113
	WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]);
2005 serge 1114
	}
1430 serge 1115
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1);
1116
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1);
2005 serge 1117
	if (rdev->num_crtc >= 4) {
1430 serge 1118
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1);
1119
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1);
2005 serge 1120
	}
1121
	if (rdev->num_crtc >= 6) {
1430 serge 1122
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1);
1123
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1);
1963 serge 1124
	}
1430 serge 1125
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]);
1126
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]);
2005 serge 1127
	if (rdev->num_crtc >= 4) {
1430 serge 1128
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]);
1129
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]);
2005 serge 1130
	}
1131
	if (rdev->num_crtc >= 6) {
1430 serge 1132
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]);
1133
	WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]);
1963 serge 1134
	}
1430 serge 1135
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
1136
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005 serge 1137
	if (rdev->num_crtc >= 4) {
1430 serge 1138
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
1139
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005 serge 1140
	}
1141
	if (rdev->num_crtc >= 6) {
1430 serge 1142
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
1143
	WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
1963 serge 1144
	}
1430 serge 1145
	WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
1146
}
1147
 
1963 serge 1148
void evergreen_mc_program(struct radeon_device *rdev)
1430 serge 1149
{
1150
	struct evergreen_mc_save save;
1151
	u32 tmp;
1152
	int i, j;
1153
 
1154
	/* Initialize HDP */
1155
	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1156
		WREG32((0x2c14 + j), 0x00000000);
1157
		WREG32((0x2c18 + j), 0x00000000);
1158
		WREG32((0x2c1c + j), 0x00000000);
1159
		WREG32((0x2c20 + j), 0x00000000);
1160
		WREG32((0x2c24 + j), 0x00000000);
1161
	}
1162
	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1163
 
1164
	evergreen_mc_stop(rdev, &save);
1165
	if (evergreen_mc_wait_for_idle(rdev)) {
1166
		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1167
	}
1168
	/* Lockout access through VGA aperture*/
1169
	WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1170
	/* Update configuration */
1171
	if (rdev->flags & RADEON_IS_AGP) {
1172
		if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1173
			/* VRAM before AGP */
1174
			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1175
				rdev->mc.vram_start >> 12);
1176
			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1177
				rdev->mc.gtt_end >> 12);
1178
		} else {
1179
			/* VRAM after AGP */
1180
			WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1181
				rdev->mc.gtt_start >> 12);
1182
			WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1183
				rdev->mc.vram_end >> 12);
1184
		}
1185
	} else {
1186
		WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1187
			rdev->mc.vram_start >> 12);
1188
		WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1189
			rdev->mc.vram_end >> 12);
1190
	}
1191
	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0);
1963 serge 1192
	if (rdev->flags & RADEON_IS_IGP) {
1193
		tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
1194
		tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
1195
		tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
1196
		WREG32(MC_FUS_VM_FB_OFFSET, tmp);
1197
	}
1430 serge 1198
	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1199
	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1200
	WREG32(MC_VM_FB_LOCATION, tmp);
1201
	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1963 serge 1202
	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
1203
	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1430 serge 1204
	if (rdev->flags & RADEON_IS_AGP) {
1205
		WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
1206
		WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
1207
		WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1208
	} else {
1209
		WREG32(MC_VM_AGP_BASE, 0);
1210
		WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1211
		WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1212
	}
1213
	if (evergreen_mc_wait_for_idle(rdev)) {
1214
		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1215
	}
1216
	evergreen_mc_resume(rdev, &save);
1217
	/* we need to own VRAM, so turn off the VGA renderer here
1218
	 * to stop it overwriting our objects */
1219
	rv515_vga_render_disable(rdev);
1220
}
1221
 
1222
/*
1223
 * CP.
1224
 */
1986 serge 1225
void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1226
{
1227
	/* set to DX10/11 mode */
1228
	radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0));
1229
	radeon_ring_write(rdev, 1);
1230
	/* FIXME: implement */
1231
	radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1232
	radeon_ring_write(rdev,
1233
#ifdef __BIG_ENDIAN
1234
			  (2 << 0) |
1235
#endif
1236
			  (ib->gpu_addr & 0xFFFFFFFC));
1237
	radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF);
1238
	radeon_ring_write(rdev, ib->length_dw);
1239
}
1963 serge 1240
 
1986 serge 1241
 
1963 serge 1242
static int evergreen_cp_load_microcode(struct radeon_device *rdev)
1430 serge 1243
{
1963 serge 1244
	const __be32 *fw_data;
1245
	int i;
1246
 
1247
	if (!rdev->me_fw || !rdev->pfp_fw)
1248
		return -EINVAL;
1249
 
1250
	r700_cp_stop(rdev);
1251
	WREG32(CP_RB_CNTL,
1252
#ifdef __BIG_ENDIAN
1253
	       BUF_SWAP_32BIT |
1254
#endif
1255
	       RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
1256
 
1257
	fw_data = (const __be32 *)rdev->pfp_fw->data;
1258
	WREG32(CP_PFP_UCODE_ADDR, 0);
1259
	for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
1260
		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1261
	WREG32(CP_PFP_UCODE_ADDR, 0);
1262
 
1263
	fw_data = (const __be32 *)rdev->me_fw->data;
1264
	WREG32(CP_ME_RAM_WADDR, 0);
1265
	for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
1266
		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1267
 
1268
	WREG32(CP_PFP_UCODE_ADDR, 0);
1269
	WREG32(CP_ME_RAM_WADDR, 0);
1270
	WREG32(CP_ME_RAM_RADDR, 0);
1271
	return 0;
1430 serge 1272
}
1273
 
1963 serge 1274
static int evergreen_cp_start(struct radeon_device *rdev)
1275
{
1276
	int r, i;
1277
	uint32_t cp_me;
1430 serge 1278
 
1963 serge 1279
	r = radeon_ring_lock(rdev, 7);
1280
	if (r) {
1281
		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1282
		return r;
1283
	}
1284
	radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5));
1285
	radeon_ring_write(rdev, 0x1);
1286
	radeon_ring_write(rdev, 0x0);
1287
	radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1);
1288
	radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1289
	radeon_ring_write(rdev, 0);
1290
	radeon_ring_write(rdev, 0);
1291
	radeon_ring_unlock_commit(rdev);
1292
 
1293
	cp_me = 0xff;
1294
	WREG32(CP_ME_CNTL, cp_me);
1295
 
1296
	r = radeon_ring_lock(rdev, evergreen_default_size + 19);
1297
	if (r) {
1298
		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1299
		return r;
1300
	}
1301
 
1302
	/* setup clear context state */
1303
	radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1304
	radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1305
 
1306
	for (i = 0; i < evergreen_default_size; i++)
1307
		radeon_ring_write(rdev, evergreen_default_state[i]);
1308
 
1309
	radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1310
	radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE);
1311
 
1312
	/* set clear context state */
1313
	radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0));
1314
	radeon_ring_write(rdev, 0);
1315
 
1316
	/* SQ_VTX_BASE_VTX_LOC */
1317
	radeon_ring_write(rdev, 0xc0026f00);
1318
	radeon_ring_write(rdev, 0x00000000);
1319
	radeon_ring_write(rdev, 0x00000000);
1320
	radeon_ring_write(rdev, 0x00000000);
1321
 
1322
	/* Clear consts */
1323
	radeon_ring_write(rdev, 0xc0036f00);
1324
	radeon_ring_write(rdev, 0x00000bc4);
1325
	radeon_ring_write(rdev, 0xffffffff);
1326
	radeon_ring_write(rdev, 0xffffffff);
1327
	radeon_ring_write(rdev, 0xffffffff);
1328
 
1329
	radeon_ring_write(rdev, 0xc0026900);
1330
	radeon_ring_write(rdev, 0x00000316);
1331
	radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1332
	radeon_ring_write(rdev, 0x00000010); /*  */
1333
 
1334
	radeon_ring_unlock_commit(rdev);
1335
 
1336
	return 0;
1337
}
1338
 
1339
int evergreen_cp_resume(struct radeon_device *rdev)
1430 serge 1340
{
1963 serge 1341
	u32 tmp;
1342
	u32 rb_bufsz;
1343
	int r;
1430 serge 1344
 
1963 serge 1345
	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1346
	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1347
				 SOFT_RESET_PA |
1348
				 SOFT_RESET_SH |
1349
				 SOFT_RESET_VGT |
2160 serge 1350
				 SOFT_RESET_SPI |
1963 serge 1351
				 SOFT_RESET_SX));
1352
	RREG32(GRBM_SOFT_RESET);
1353
	mdelay(15);
1354
	WREG32(GRBM_SOFT_RESET, 0);
1355
	RREG32(GRBM_SOFT_RESET);
1356
 
1357
	/* Set ring buffer size */
1358
	rb_bufsz = drm_order(rdev->cp.ring_size / 8);
1359
	tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
1360
#ifdef __BIG_ENDIAN
1361
	tmp |= BUF_SWAP_32BIT;
1362
#endif
1363
	WREG32(CP_RB_CNTL, tmp);
1364
	WREG32(CP_SEM_WAIT_TIMER, 0x4);
1365
 
1366
	/* Set the write pointer delay */
1367
	WREG32(CP_RB_WPTR_DELAY, 0);
1368
 
1369
	/* Initialize the ring buffer's read and write pointers */
1370
	WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
1371
	WREG32(CP_RB_RPTR_WR, 0);
1372
	WREG32(CP_RB_WPTR, 0);
1373
 
1374
	/* set the wb address wether it's enabled or not */
1375
	WREG32(CP_RB_RPTR_ADDR,
1376
	       ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
1377
	WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
1378
	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1379
 
1380
	if (rdev->wb.enabled)
1381
		WREG32(SCRATCH_UMSK, 0xff);
1382
	else {
1383
		tmp |= RB_NO_UPDATE;
1384
		WREG32(SCRATCH_UMSK, 0);
1385
	}
1386
 
1387
	mdelay(1);
1388
	WREG32(CP_RB_CNTL, tmp);
1389
 
1390
	WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8);
1391
	WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
1392
 
1393
	rdev->cp.rptr = RREG32(CP_RB_RPTR);
1394
	rdev->cp.wptr = RREG32(CP_RB_WPTR);
1395
 
1396
	evergreen_cp_start(rdev);
1397
	rdev->cp.ready = true;
1398
	r = radeon_ring_test(rdev);
1399
	if (r) {
1400
		rdev->cp.ready = false;
1401
		return r;
1402
	}
1430 serge 1403
	return 0;
1404
}
1405
 
1406
/*
1407
 * Core functions
1408
 */
1963 serge 1409
static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev,
1410
						  u32 num_tile_pipes,
1430 serge 1411
						  u32 num_backends,
1412
						  u32 backend_disable_mask)
1413
{
1414
	u32 backend_map = 0;
1963 serge 1415
	u32 enabled_backends_mask = 0;
1416
	u32 enabled_backends_count = 0;
1417
	u32 cur_pipe;
1418
	u32 swizzle_pipe[EVERGREEN_MAX_PIPES];
1419
	u32 cur_backend = 0;
1420
	u32 i;
1421
	bool force_no_swizzle;
1430 serge 1422
 
1963 serge 1423
	if (num_tile_pipes > EVERGREEN_MAX_PIPES)
1424
		num_tile_pipes = EVERGREEN_MAX_PIPES;
1425
	if (num_tile_pipes < 1)
1426
		num_tile_pipes = 1;
1427
	if (num_backends > EVERGREEN_MAX_BACKENDS)
1428
		num_backends = EVERGREEN_MAX_BACKENDS;
1429
	if (num_backends < 1)
1430
		num_backends = 1;
1431
 
1432
	for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
1433
		if (((backend_disable_mask >> i) & 1) == 0) {
1434
			enabled_backends_mask |= (1 << i);
1435
			++enabled_backends_count;
1436
		}
1437
		if (enabled_backends_count == num_backends)
1438
			break;
1439
	}
1440
 
1441
	if (enabled_backends_count == 0) {
1442
		enabled_backends_mask = 1;
1443
		enabled_backends_count = 1;
1444
	}
1445
 
1446
	if (enabled_backends_count != num_backends)
1447
		num_backends = enabled_backends_count;
1448
 
1449
	memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * EVERGREEN_MAX_PIPES);
1450
	switch (rdev->family) {
1451
	case CHIP_CEDAR:
1452
	case CHIP_REDWOOD:
1453
	case CHIP_PALM:
1454
	case CHIP_SUMO:
1455
	case CHIP_SUMO2:
1456
	case CHIP_TURKS:
1457
	case CHIP_CAICOS:
1458
		force_no_swizzle = false;
1459
		break;
1460
	case CHIP_CYPRESS:
1461
	case CHIP_HEMLOCK:
1462
	case CHIP_JUNIPER:
1463
	case CHIP_BARTS:
1464
	default:
1465
		force_no_swizzle = true;
1466
		break;
1467
	}
1468
	if (force_no_swizzle) {
1469
		bool last_backend_enabled = false;
1470
 
1471
		force_no_swizzle = false;
1472
		for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) {
1473
			if (((enabled_backends_mask >> i) & 1) == 1) {
1474
				if (last_backend_enabled)
1475
					force_no_swizzle = true;
1476
				last_backend_enabled = true;
1477
			} else
1478
				last_backend_enabled = false;
1479
		}
1480
	}
1481
 
1482
	switch (num_tile_pipes) {
1483
	case 1:
1484
	case 3:
1485
	case 5:
1486
	case 7:
1487
		DRM_ERROR("odd number of pipes!\n");
1488
		break;
1489
	case 2:
1490
		swizzle_pipe[0] = 0;
1491
		swizzle_pipe[1] = 1;
1492
		break;
1493
	case 4:
1494
		if (force_no_swizzle) {
1495
			swizzle_pipe[0] = 0;
1496
			swizzle_pipe[1] = 1;
1497
			swizzle_pipe[2] = 2;
1498
			swizzle_pipe[3] = 3;
1499
		} else {
1500
			swizzle_pipe[0] = 0;
1501
			swizzle_pipe[1] = 2;
1502
			swizzle_pipe[2] = 1;
1503
			swizzle_pipe[3] = 3;
1504
		}
1505
		break;
1506
	case 6:
1507
		if (force_no_swizzle) {
1508
			swizzle_pipe[0] = 0;
1509
			swizzle_pipe[1] = 1;
1510
			swizzle_pipe[2] = 2;
1511
			swizzle_pipe[3] = 3;
1512
			swizzle_pipe[4] = 4;
1513
			swizzle_pipe[5] = 5;
1514
		} else {
1515
			swizzle_pipe[0] = 0;
1516
			swizzle_pipe[1] = 2;
1517
			swizzle_pipe[2] = 4;
1518
			swizzle_pipe[3] = 1;
1519
			swizzle_pipe[4] = 3;
1520
			swizzle_pipe[5] = 5;
1521
		}
1522
		break;
1523
	case 8:
1524
		if (force_no_swizzle) {
1525
			swizzle_pipe[0] = 0;
1526
			swizzle_pipe[1] = 1;
1527
			swizzle_pipe[2] = 2;
1528
			swizzle_pipe[3] = 3;
1529
			swizzle_pipe[4] = 4;
1530
			swizzle_pipe[5] = 5;
1531
			swizzle_pipe[6] = 6;
1532
			swizzle_pipe[7] = 7;
1533
		} else {
1534
			swizzle_pipe[0] = 0;
1535
			swizzle_pipe[1] = 2;
1536
			swizzle_pipe[2] = 4;
1537
			swizzle_pipe[3] = 6;
1538
			swizzle_pipe[4] = 1;
1539
			swizzle_pipe[5] = 3;
1540
			swizzle_pipe[6] = 5;
1541
			swizzle_pipe[7] = 7;
1542
		}
1543
		break;
1544
	}
1545
 
1546
	for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) {
1547
		while (((1 << cur_backend) & enabled_backends_mask) == 0)
1548
			cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
1549
 
1550
		backend_map |= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4)));
1551
 
1552
		cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS;
1553
	}
1554
 
1430 serge 1555
	return backend_map;
1556
}
1557
 
1986 serge 1558
static void evergreen_program_channel_remap(struct radeon_device *rdev)
1559
{
1560
	u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp;
1561
 
1562
	tmp = RREG32(MC_SHARED_CHMAP);
1563
	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1564
	case 0:
1565
	case 1:
1566
	case 2:
1567
	case 3:
1568
	default:
1569
		/* default mapping */
1570
		mc_shared_chremap = 0x00fac688;
1571
		break;
1572
	}
1573
 
1574
	switch (rdev->family) {
1575
	case CHIP_HEMLOCK:
1576
	case CHIP_CYPRESS:
1577
	case CHIP_BARTS:
1578
		tcp_chan_steer_lo = 0x54763210;
1579
		tcp_chan_steer_hi = 0x0000ba98;
1580
		break;
1581
	case CHIP_JUNIPER:
1582
	case CHIP_REDWOOD:
1583
	case CHIP_CEDAR:
1584
	case CHIP_PALM:
1585
	case CHIP_SUMO:
1586
	case CHIP_SUMO2:
1587
	case CHIP_TURKS:
1588
	case CHIP_CAICOS:
1589
	default:
1590
		tcp_chan_steer_lo = 0x76543210;
1591
		tcp_chan_steer_hi = 0x0000ba98;
1592
		break;
1593
	}
1594
 
1595
	WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo);
1596
	WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi);
1597
	WREG32(MC_SHARED_CHREMAP, mc_shared_chremap);
1598
}
1599
 
1430 serge 1600
static void evergreen_gpu_init(struct radeon_device *rdev)
1601
{
1963 serge 1602
	u32 cc_rb_backend_disable = 0;
1603
	u32 cc_gc_shader_pipe_config;
1604
	u32 gb_addr_config = 0;
1605
	u32 mc_shared_chmap, mc_arb_ramcfg;
1606
	u32 gb_backend_map;
1607
	u32 grbm_gfx_index;
1608
	u32 sx_debug_1;
1609
	u32 smx_dc_ctl0;
1610
	u32 sq_config;
1611
	u32 sq_lds_resource_mgmt;
1612
	u32 sq_gpr_resource_mgmt_1;
1613
	u32 sq_gpr_resource_mgmt_2;
1614
	u32 sq_gpr_resource_mgmt_3;
1615
	u32 sq_thread_resource_mgmt;
1616
	u32 sq_thread_resource_mgmt_2;
1617
	u32 sq_stack_resource_mgmt_1;
1618
	u32 sq_stack_resource_mgmt_2;
1619
	u32 sq_stack_resource_mgmt_3;
1620
	u32 vgt_cache_invalidation;
1621
	u32 hdp_host_path_cntl, tmp;
1622
	int i, j, num_shader_engines, ps_thread_count;
1623
 
1624
	switch (rdev->family) {
1625
	case CHIP_CYPRESS:
1626
	case CHIP_HEMLOCK:
1627
		rdev->config.evergreen.num_ses = 2;
1628
		rdev->config.evergreen.max_pipes = 4;
1629
		rdev->config.evergreen.max_tile_pipes = 8;
1630
		rdev->config.evergreen.max_simds = 10;
1631
		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1632
		rdev->config.evergreen.max_gprs = 256;
1633
		rdev->config.evergreen.max_threads = 248;
1634
		rdev->config.evergreen.max_gs_threads = 32;
1635
		rdev->config.evergreen.max_stack_entries = 512;
1636
		rdev->config.evergreen.sx_num_of_sets = 4;
1637
		rdev->config.evergreen.sx_max_export_size = 256;
1638
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1639
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1640
		rdev->config.evergreen.max_hw_contexts = 8;
1641
		rdev->config.evergreen.sq_num_cf_insts = 2;
1642
 
1643
		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1644
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1645
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1646
		break;
1647
	case CHIP_JUNIPER:
1648
		rdev->config.evergreen.num_ses = 1;
1649
		rdev->config.evergreen.max_pipes = 4;
1650
		rdev->config.evergreen.max_tile_pipes = 4;
1651
		rdev->config.evergreen.max_simds = 10;
1652
		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1653
		rdev->config.evergreen.max_gprs = 256;
1654
		rdev->config.evergreen.max_threads = 248;
1655
		rdev->config.evergreen.max_gs_threads = 32;
1656
		rdev->config.evergreen.max_stack_entries = 512;
1657
		rdev->config.evergreen.sx_num_of_sets = 4;
1658
		rdev->config.evergreen.sx_max_export_size = 256;
1659
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1660
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1661
		rdev->config.evergreen.max_hw_contexts = 8;
1662
		rdev->config.evergreen.sq_num_cf_insts = 2;
1663
 
1664
		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1665
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1666
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1667
		break;
1668
	case CHIP_REDWOOD:
1669
		rdev->config.evergreen.num_ses = 1;
1670
		rdev->config.evergreen.max_pipes = 4;
1671
		rdev->config.evergreen.max_tile_pipes = 4;
1672
		rdev->config.evergreen.max_simds = 5;
1673
		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1674
		rdev->config.evergreen.max_gprs = 256;
1675
		rdev->config.evergreen.max_threads = 248;
1676
		rdev->config.evergreen.max_gs_threads = 32;
1677
		rdev->config.evergreen.max_stack_entries = 256;
1678
		rdev->config.evergreen.sx_num_of_sets = 4;
1679
		rdev->config.evergreen.sx_max_export_size = 256;
1680
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1681
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1682
		rdev->config.evergreen.max_hw_contexts = 8;
1683
		rdev->config.evergreen.sq_num_cf_insts = 2;
1684
 
1685
		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1686
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1687
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1688
		break;
1689
	case CHIP_CEDAR:
1690
	default:
1691
		rdev->config.evergreen.num_ses = 1;
1692
		rdev->config.evergreen.max_pipes = 2;
1693
		rdev->config.evergreen.max_tile_pipes = 2;
1694
		rdev->config.evergreen.max_simds = 2;
1695
		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1696
		rdev->config.evergreen.max_gprs = 256;
1697
		rdev->config.evergreen.max_threads = 192;
1698
		rdev->config.evergreen.max_gs_threads = 16;
1699
		rdev->config.evergreen.max_stack_entries = 256;
1700
		rdev->config.evergreen.sx_num_of_sets = 4;
1701
		rdev->config.evergreen.sx_max_export_size = 128;
1702
		rdev->config.evergreen.sx_max_export_pos_size = 32;
1703
		rdev->config.evergreen.sx_max_export_smx_size = 96;
1704
		rdev->config.evergreen.max_hw_contexts = 4;
1705
		rdev->config.evergreen.sq_num_cf_insts = 1;
1706
 
1707
		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1708
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1709
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1710
		break;
1711
	case CHIP_PALM:
1712
		rdev->config.evergreen.num_ses = 1;
1713
		rdev->config.evergreen.max_pipes = 2;
1714
		rdev->config.evergreen.max_tile_pipes = 2;
1715
		rdev->config.evergreen.max_simds = 2;
1716
		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1717
		rdev->config.evergreen.max_gprs = 256;
1718
		rdev->config.evergreen.max_threads = 192;
1719
		rdev->config.evergreen.max_gs_threads = 16;
1720
		rdev->config.evergreen.max_stack_entries = 256;
1721
		rdev->config.evergreen.sx_num_of_sets = 4;
1722
		rdev->config.evergreen.sx_max_export_size = 128;
1723
		rdev->config.evergreen.sx_max_export_pos_size = 32;
1724
		rdev->config.evergreen.sx_max_export_smx_size = 96;
1725
		rdev->config.evergreen.max_hw_contexts = 4;
1726
		rdev->config.evergreen.sq_num_cf_insts = 1;
1727
 
1728
		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1729
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1730
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1731
		break;
1732
	case CHIP_SUMO:
1733
		rdev->config.evergreen.num_ses = 1;
1734
		rdev->config.evergreen.max_pipes = 4;
1735
		rdev->config.evergreen.max_tile_pipes = 2;
1736
		if (rdev->pdev->device == 0x9648)
1737
			rdev->config.evergreen.max_simds = 3;
1738
		else if ((rdev->pdev->device == 0x9647) ||
1739
			 (rdev->pdev->device == 0x964a))
1740
			rdev->config.evergreen.max_simds = 4;
1741
		else
1742
			rdev->config.evergreen.max_simds = 5;
1743
		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1744
		rdev->config.evergreen.max_gprs = 256;
1745
		rdev->config.evergreen.max_threads = 248;
1746
		rdev->config.evergreen.max_gs_threads = 32;
1747
		rdev->config.evergreen.max_stack_entries = 256;
1748
		rdev->config.evergreen.sx_num_of_sets = 4;
1749
		rdev->config.evergreen.sx_max_export_size = 256;
1750
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1751
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1752
		rdev->config.evergreen.max_hw_contexts = 8;
1753
		rdev->config.evergreen.sq_num_cf_insts = 2;
1754
 
1755
		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1756
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1757
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1758
		break;
1759
	case CHIP_SUMO2:
1760
		rdev->config.evergreen.num_ses = 1;
1761
		rdev->config.evergreen.max_pipes = 4;
1762
		rdev->config.evergreen.max_tile_pipes = 4;
1763
		rdev->config.evergreen.max_simds = 2;
1764
		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1765
		rdev->config.evergreen.max_gprs = 256;
1766
		rdev->config.evergreen.max_threads = 248;
1767
		rdev->config.evergreen.max_gs_threads = 32;
1768
		rdev->config.evergreen.max_stack_entries = 512;
1769
		rdev->config.evergreen.sx_num_of_sets = 4;
1770
		rdev->config.evergreen.sx_max_export_size = 256;
1771
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1772
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1773
		rdev->config.evergreen.max_hw_contexts = 8;
1774
		rdev->config.evergreen.sq_num_cf_insts = 2;
1775
 
1776
		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1777
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1778
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1779
		break;
1780
	case CHIP_BARTS:
1781
		rdev->config.evergreen.num_ses = 2;
1782
		rdev->config.evergreen.max_pipes = 4;
1783
		rdev->config.evergreen.max_tile_pipes = 8;
1784
		rdev->config.evergreen.max_simds = 7;
1785
		rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
1786
		rdev->config.evergreen.max_gprs = 256;
1787
		rdev->config.evergreen.max_threads = 248;
1788
		rdev->config.evergreen.max_gs_threads = 32;
1789
		rdev->config.evergreen.max_stack_entries = 512;
1790
		rdev->config.evergreen.sx_num_of_sets = 4;
1791
		rdev->config.evergreen.sx_max_export_size = 256;
1792
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1793
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1794
		rdev->config.evergreen.max_hw_contexts = 8;
1795
		rdev->config.evergreen.sq_num_cf_insts = 2;
1796
 
1797
		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1798
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1799
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1800
		break;
1801
	case CHIP_TURKS:
1802
		rdev->config.evergreen.num_ses = 1;
1803
		rdev->config.evergreen.max_pipes = 4;
1804
		rdev->config.evergreen.max_tile_pipes = 4;
1805
		rdev->config.evergreen.max_simds = 6;
1806
		rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
1807
		rdev->config.evergreen.max_gprs = 256;
1808
		rdev->config.evergreen.max_threads = 248;
1809
		rdev->config.evergreen.max_gs_threads = 32;
1810
		rdev->config.evergreen.max_stack_entries = 256;
1811
		rdev->config.evergreen.sx_num_of_sets = 4;
1812
		rdev->config.evergreen.sx_max_export_size = 256;
1813
		rdev->config.evergreen.sx_max_export_pos_size = 64;
1814
		rdev->config.evergreen.sx_max_export_smx_size = 192;
1815
		rdev->config.evergreen.max_hw_contexts = 8;
1816
		rdev->config.evergreen.sq_num_cf_insts = 2;
1817
 
1818
		rdev->config.evergreen.sc_prim_fifo_size = 0x100;
1819
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1820
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1821
		break;
1822
	case CHIP_CAICOS:
1823
		rdev->config.evergreen.num_ses = 1;
1824
		rdev->config.evergreen.max_pipes = 4;
1825
		rdev->config.evergreen.max_tile_pipes = 2;
1826
		rdev->config.evergreen.max_simds = 2;
1827
		rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
1828
		rdev->config.evergreen.max_gprs = 256;
1829
		rdev->config.evergreen.max_threads = 192;
1830
		rdev->config.evergreen.max_gs_threads = 16;
1831
		rdev->config.evergreen.max_stack_entries = 256;
1832
		rdev->config.evergreen.sx_num_of_sets = 4;
1833
		rdev->config.evergreen.sx_max_export_size = 128;
1834
		rdev->config.evergreen.sx_max_export_pos_size = 32;
1835
		rdev->config.evergreen.sx_max_export_smx_size = 96;
1836
		rdev->config.evergreen.max_hw_contexts = 4;
1837
		rdev->config.evergreen.sq_num_cf_insts = 1;
1838
 
1839
		rdev->config.evergreen.sc_prim_fifo_size = 0x40;
1840
		rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
1841
		rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
1842
		break;
1843
	}
1844
 
1845
	/* Initialize HDP */
1846
	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1847
		WREG32((0x2c14 + j), 0x00000000);
1848
		WREG32((0x2c18 + j), 0x00000000);
1849
		WREG32((0x2c1c + j), 0x00000000);
1850
		WREG32((0x2c20 + j), 0x00000000);
1851
		WREG32((0x2c24 + j), 0x00000000);
1852
	}
1853
 
1854
	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1855
 
1856
	cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & ~2;
1857
 
1858
	cc_gc_shader_pipe_config |=
1859
		INACTIVE_QD_PIPES((EVERGREEN_MAX_PIPES_MASK << rdev->config.evergreen.max_pipes)
1860
				  & EVERGREEN_MAX_PIPES_MASK);
1861
	cc_gc_shader_pipe_config |=
1862
		INACTIVE_SIMDS((EVERGREEN_MAX_SIMDS_MASK << rdev->config.evergreen.max_simds)
1863
			       & EVERGREEN_MAX_SIMDS_MASK);
1864
 
1865
	cc_rb_backend_disable =
1866
		BACKEND_DISABLE((EVERGREEN_MAX_BACKENDS_MASK << rdev->config.evergreen.max_backends)
1867
				& EVERGREEN_MAX_BACKENDS_MASK);
1868
 
1869
 
1870
	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1871
	if (rdev->flags & RADEON_IS_IGP)
1872
		mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
1873
	else
1874
	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1875
 
1876
	switch (rdev->config.evergreen.max_tile_pipes) {
1877
	case 1:
1878
	default:
1879
		gb_addr_config |= NUM_PIPES(0);
1880
		break;
1881
	case 2:
1882
		gb_addr_config |= NUM_PIPES(1);
1883
		break;
1884
	case 4:
1885
		gb_addr_config |= NUM_PIPES(2);
1886
		break;
1887
	case 8:
1888
		gb_addr_config |= NUM_PIPES(3);
1889
		break;
1890
	}
1891
 
1892
	gb_addr_config |= PIPE_INTERLEAVE_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1893
	gb_addr_config |= BANK_INTERLEAVE_SIZE(0);
1894
	gb_addr_config |= NUM_SHADER_ENGINES(rdev->config.evergreen.num_ses - 1);
1895
	gb_addr_config |= SHADER_ENGINE_TILE_SIZE(1);
1896
	gb_addr_config |= NUM_GPUS(0); /* Hemlock? */
1897
	gb_addr_config |= MULTI_GPU_TILE_SIZE(2);
1898
 
1899
	if (((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) > 2)
1900
		gb_addr_config |= ROW_SIZE(2);
1901
	else
1902
		gb_addr_config |= ROW_SIZE((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT);
1903
 
1904
	if (rdev->ddev->pdev->device == 0x689e) {
1905
		u32 efuse_straps_4;
1906
		u32 efuse_straps_3;
1907
		u8 efuse_box_bit_131_124;
1908
 
1909
		WREG32(RCU_IND_INDEX, 0x204);
1910
		efuse_straps_4 = RREG32(RCU_IND_DATA);
1911
		WREG32(RCU_IND_INDEX, 0x203);
1912
		efuse_straps_3 = RREG32(RCU_IND_DATA);
1913
		efuse_box_bit_131_124 = (u8)(((efuse_straps_4 & 0xf) << 4) | ((efuse_straps_3 & 0xf0000000) >> 28));
1914
 
1915
		switch(efuse_box_bit_131_124) {
1916
		case 0x00:
1917
			gb_backend_map = 0x76543210;
1918
			break;
1919
		case 0x55:
1920
			gb_backend_map = 0x77553311;
1921
			break;
1922
		case 0x56:
1923
			gb_backend_map = 0x77553300;
1924
			break;
1925
		case 0x59:
1926
			gb_backend_map = 0x77552211;
1927
			break;
1928
		case 0x66:
1929
			gb_backend_map = 0x77443300;
1930
			break;
1931
		case 0x99:
1932
			gb_backend_map = 0x66552211;
1933
			break;
1934
		case 0x5a:
1935
			gb_backend_map = 0x77552200;
1936
			break;
1937
		case 0xaa:
1938
			gb_backend_map = 0x66442200;
1939
			break;
1940
		case 0x95:
1941
			gb_backend_map = 0x66553311;
1942
			break;
1943
		default:
1944
			DRM_ERROR("bad backend map, using default\n");
1945
			gb_backend_map =
1946
				evergreen_get_tile_pipe_to_backend_map(rdev,
1947
								       rdev->config.evergreen.max_tile_pipes,
1948
								       rdev->config.evergreen.max_backends,
1949
								       ((EVERGREEN_MAX_BACKENDS_MASK <<
1950
								   rdev->config.evergreen.max_backends) &
1951
									EVERGREEN_MAX_BACKENDS_MASK));
1952
			break;
1953
		}
1954
	} else if (rdev->ddev->pdev->device == 0x68b9) {
1955
		u32 efuse_straps_3;
1956
		u8 efuse_box_bit_127_124;
1957
 
1958
		WREG32(RCU_IND_INDEX, 0x203);
1959
		efuse_straps_3 = RREG32(RCU_IND_DATA);
1960
		efuse_box_bit_127_124 = (u8)((efuse_straps_3 & 0xF0000000) >> 28);
1961
 
1962
		switch(efuse_box_bit_127_124) {
1963
		case 0x0:
1964
			gb_backend_map = 0x00003210;
1965
			break;
1966
		case 0x5:
1967
		case 0x6:
1968
		case 0x9:
1969
		case 0xa:
1970
			gb_backend_map = 0x00003311;
1971
			break;
1972
		default:
1973
			DRM_ERROR("bad backend map, using default\n");
1974
			gb_backend_map =
1975
				evergreen_get_tile_pipe_to_backend_map(rdev,
1976
								       rdev->config.evergreen.max_tile_pipes,
1977
								       rdev->config.evergreen.max_backends,
1978
								       ((EVERGREEN_MAX_BACKENDS_MASK <<
1979
								   rdev->config.evergreen.max_backends) &
1980
									EVERGREEN_MAX_BACKENDS_MASK));
1981
			break;
1982
		}
1983
	} else {
1984
		switch (rdev->family) {
1985
		case CHIP_CYPRESS:
1986
		case CHIP_HEMLOCK:
1987
		case CHIP_BARTS:
1988
			gb_backend_map = 0x66442200;
1989
			break;
1990
		case CHIP_JUNIPER:
2005 serge 1991
			gb_backend_map = 0x00002200;
1963 serge 1992
			break;
1993
		default:
1994
			gb_backend_map =
1995
				evergreen_get_tile_pipe_to_backend_map(rdev,
1996
								       rdev->config.evergreen.max_tile_pipes,
1997
								       rdev->config.evergreen.max_backends,
1998
								       ((EVERGREEN_MAX_BACKENDS_MASK <<
1999
									 rdev->config.evergreen.max_backends) &
2000
									EVERGREEN_MAX_BACKENDS_MASK));
2001
		}
2002
	}
2003
 
2004
	/* setup tiling info dword.  gb_addr_config is not adequate since it does
2005
	 * not have bank info, so create a custom tiling dword.
2006
	 * bits 3:0   num_pipes
2007
	 * bits 7:4   num_banks
2008
	 * bits 11:8  group_size
2009
	 * bits 15:12 row_size
2010
	 */
2011
	rdev->config.evergreen.tile_config = 0;
2012
	switch (rdev->config.evergreen.max_tile_pipes) {
2013
	case 1:
2014
	default:
2015
		rdev->config.evergreen.tile_config |= (0 << 0);
2016
		break;
2017
	case 2:
2018
		rdev->config.evergreen.tile_config |= (1 << 0);
2019
		break;
2020
	case 4:
2021
		rdev->config.evergreen.tile_config |= (2 << 0);
2022
		break;
2023
	case 8:
2024
		rdev->config.evergreen.tile_config |= (3 << 0);
2025
		break;
2026
	}
1986 serge 2027
	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1963 serge 2028
	if (rdev->flags & RADEON_IS_IGP)
1986 serge 2029
		rdev->config.evergreen.tile_config |= 1 << 4;
1963 serge 2030
	else
2031
		rdev->config.evergreen.tile_config |=
2032
			((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
2033
	rdev->config.evergreen.tile_config |=
2034
		((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT) << 8;
2035
	rdev->config.evergreen.tile_config |=
2036
		((gb_addr_config & 0x30000000) >> 28) << 12;
2037
 
2160 serge 2038
	rdev->config.evergreen.backend_map = gb_backend_map;
1963 serge 2039
	WREG32(GB_BACKEND_MAP, gb_backend_map);
2040
	WREG32(GB_ADDR_CONFIG, gb_addr_config);
2041
	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2042
	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2043
 
2044
	evergreen_program_channel_remap(rdev);
2045
 
2046
	num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1;
2047
	grbm_gfx_index = INSTANCE_BROADCAST_WRITES;
2048
 
2049
	for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
2050
		u32 rb = cc_rb_backend_disable | (0xf0 << 16);
2051
		u32 sp = cc_gc_shader_pipe_config;
2052
		u32 gfx = grbm_gfx_index | SE_INDEX(i);
2053
 
2054
		if (i == num_shader_engines) {
2055
			rb |= BACKEND_DISABLE(EVERGREEN_MAX_BACKENDS_MASK);
2056
			sp |= INACTIVE_SIMDS(EVERGREEN_MAX_SIMDS_MASK);
2057
		}
2058
 
2059
		WREG32(GRBM_GFX_INDEX, gfx);
2060
		WREG32(RLC_GFX_INDEX, gfx);
2061
 
2062
		WREG32(CC_RB_BACKEND_DISABLE, rb);
2063
		WREG32(CC_SYS_RB_BACKEND_DISABLE, rb);
2064
		WREG32(GC_USER_RB_BACKEND_DISABLE, rb);
2065
		WREG32(CC_GC_SHADER_PIPE_CONFIG, sp);
2066
        }
2067
 
2068
	grbm_gfx_index |= SE_BROADCAST_WRITES;
2069
	WREG32(GRBM_GFX_INDEX, grbm_gfx_index);
2070
	WREG32(RLC_GFX_INDEX, grbm_gfx_index);
2071
 
2072
	WREG32(CGTS_SYS_TCC_DISABLE, 0);
2073
	WREG32(CGTS_TCC_DISABLE, 0);
2074
	WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
2075
	WREG32(CGTS_USER_TCC_DISABLE, 0);
2076
 
2077
	/* set HW defaults for 3D engine */
2078
	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2079
				     ROQ_IB2_START(0x2b)));
2080
 
2081
	WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
2082
 
2083
	WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
2084
			     SYNC_GRADIENT |
2085
			     SYNC_WALKER |
2086
			     SYNC_ALIGNER));
2087
 
2088
	sx_debug_1 = RREG32(SX_DEBUG_1);
2089
	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
2090
	WREG32(SX_DEBUG_1, sx_debug_1);
2091
 
2092
 
2093
	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
2094
	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
2095
	smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
2096
	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
2097
 
2098
	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
2099
					POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
2100
					SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
2101
 
2102
	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
2103
				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
2104
				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
2105
 
2106
	WREG32(VGT_NUM_INSTANCES, 1);
2107
	WREG32(SPI_CONFIG_CNTL, 0);
2108
	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2109
	WREG32(CP_PERFMON_CNTL, 0);
2110
 
2111
	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
2112
				  FETCH_FIFO_HIWATER(0x4) |
2113
				  DONE_FIFO_HIWATER(0xe0) |
2114
				  ALU_UPDATE_FIFO_HIWATER(0x8)));
2115
 
2116
	sq_config = RREG32(SQ_CONFIG);
2117
	sq_config &= ~(PS_PRIO(3) |
2118
		       VS_PRIO(3) |
2119
		       GS_PRIO(3) |
2120
		       ES_PRIO(3));
2121
	sq_config |= (VC_ENABLE |
2122
		      EXPORT_SRC_C |
2123
		      PS_PRIO(0) |
2124
		      VS_PRIO(1) |
2125
		      GS_PRIO(2) |
2126
		      ES_PRIO(3));
2127
 
2128
	switch (rdev->family) {
2129
	case CHIP_CEDAR:
2130
	case CHIP_PALM:
2131
	case CHIP_SUMO:
2132
	case CHIP_SUMO2:
2133
	case CHIP_CAICOS:
2134
		/* no vertex cache */
2135
		sq_config &= ~VC_ENABLE;
2136
		break;
2137
	default:
2138
		break;
2139
	}
2140
 
2141
	sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
2142
 
2143
	sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
2144
	sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
2145
	sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
2146
	sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2147
	sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
2148
	sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2149
	sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
2150
 
2151
	switch (rdev->family) {
2152
	case CHIP_CEDAR:
2153
	case CHIP_PALM:
2154
	case CHIP_SUMO:
2155
	case CHIP_SUMO2:
2156
		ps_thread_count = 96;
2157
		break;
2158
	default:
2159
		ps_thread_count = 128;
2160
		break;
2161
	}
2162
 
2163
	sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
2164
	sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2165
	sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2166
	sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2167
	sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2168
	sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
2169
 
2170
	sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2171
	sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2172
	sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2173
	sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2174
	sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2175
	sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
2176
 
2177
	WREG32(SQ_CONFIG, sq_config);
2178
	WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
2179
	WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
2180
	WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
2181
	WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
2182
	WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
2183
	WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
2184
	WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
2185
	WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
2186
	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
2187
	WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
2188
 
2189
	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2190
					  FORCE_EOV_MAX_REZ_CNT(255)));
2191
 
2192
	switch (rdev->family) {
2193
	case CHIP_CEDAR:
2194
	case CHIP_PALM:
2195
	case CHIP_SUMO:
2196
	case CHIP_SUMO2:
2197
	case CHIP_CAICOS:
2198
		vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
2199
		break;
2200
	default:
2201
		vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
2202
		break;
2203
	}
2204
	vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
2205
	WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
2206
 
2207
	WREG32(VGT_GS_VERTEX_REUSE, 16);
2208
	WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
2209
	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2210
 
2211
	WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
2212
	WREG32(VGT_OUT_DEALLOC_CNTL, 16);
2213
 
2214
	WREG32(CB_PERF_CTR0_SEL_0, 0);
2215
	WREG32(CB_PERF_CTR0_SEL_1, 0);
2216
	WREG32(CB_PERF_CTR1_SEL_0, 0);
2217
	WREG32(CB_PERF_CTR1_SEL_1, 0);
2218
	WREG32(CB_PERF_CTR2_SEL_0, 0);
2219
	WREG32(CB_PERF_CTR2_SEL_1, 0);
2220
	WREG32(CB_PERF_CTR3_SEL_0, 0);
2221
	WREG32(CB_PERF_CTR3_SEL_1, 0);
2222
 
2223
	/* clear render buffer base addresses */
2224
	WREG32(CB_COLOR0_BASE, 0);
2225
	WREG32(CB_COLOR1_BASE, 0);
2226
	WREG32(CB_COLOR2_BASE, 0);
2227
	WREG32(CB_COLOR3_BASE, 0);
2228
	WREG32(CB_COLOR4_BASE, 0);
2229
	WREG32(CB_COLOR5_BASE, 0);
2230
	WREG32(CB_COLOR6_BASE, 0);
2231
	WREG32(CB_COLOR7_BASE, 0);
2232
	WREG32(CB_COLOR8_BASE, 0);
2233
	WREG32(CB_COLOR9_BASE, 0);
2234
	WREG32(CB_COLOR10_BASE, 0);
2235
	WREG32(CB_COLOR11_BASE, 0);
2236
 
2237
	/* set the shader const cache sizes to 0 */
2238
	for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
2239
		WREG32(i, 0);
2240
	for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
2241
		WREG32(i, 0);
2242
 
2243
	tmp = RREG32(HDP_MISC_CNTL);
2244
	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2245
	WREG32(HDP_MISC_CNTL, tmp);
2246
 
2247
	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2248
	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2249
 
2250
	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2251
 
2252
	udelay(50);
2253
 
1430 serge 2254
}
2255
 
2256
int evergreen_mc_init(struct radeon_device *rdev)
2257
{
2258
	u32 tmp;
2259
	int chansize, numchan;
2260
 
2261
	/* Get VRAM informations */
2262
	rdev->mc.vram_is_ddr = true;
2004 serge 2263
	if (rdev->flags & RADEON_IS_IGP)
2264
		tmp = RREG32(FUS_MC_ARB_RAMCFG);
2265
	else
1430 serge 2266
	tmp = RREG32(MC_ARB_RAMCFG);
2267
	if (tmp & CHANSIZE_OVERRIDE) {
2268
		chansize = 16;
2269
	} else if (tmp & CHANSIZE_MASK) {
2270
		chansize = 64;
2271
	} else {
2272
		chansize = 32;
2273
	}
2274
	tmp = RREG32(MC_SHARED_CHMAP);
2275
	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2276
	case 0:
2277
	default:
2278
		numchan = 1;
2279
		break;
2280
	case 1:
2281
		numchan = 2;
2282
		break;
2283
	case 2:
2284
		numchan = 4;
2285
		break;
2286
	case 3:
2287
		numchan = 8;
2288
		break;
2289
	}
2290
	rdev->mc.vram_width = numchan * chansize;
2291
	/* Could aper size report 0 ? */
1963 serge 2292
	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
2293
	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1430 serge 2294
	/* Setup GPU memory space */
1963 serge 2295
	if (rdev->flags & RADEON_IS_IGP) {
2296
		/* size in bytes on fusion */
2297
		rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
2298
		rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
2299
	} else {
1430 serge 2300
	/* size in MB on evergreen */
2301
	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
2302
	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
1963 serge 2303
	}
1430 serge 2304
	rdev->mc.visible_vram_size = rdev->mc.aper_size;
1963 serge 2305
	r700_vram_gtt_location(rdev, &rdev->mc);
2306
	radeon_update_bandwidth_info(rdev);
2307
 
1430 serge 2308
	return 0;
2309
}
2310
 
1963 serge 2311
bool evergreen_gpu_is_lockup(struct radeon_device *rdev)
1430 serge 2312
{
1986 serge 2313
	u32 srbm_status;
2314
	u32 grbm_status;
2315
	u32 grbm_status_se0, grbm_status_se1;
2316
	struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup;
2317
	int r;
2318
 
2319
	srbm_status = RREG32(SRBM_STATUS);
2320
	grbm_status = RREG32(GRBM_STATUS);
2321
	grbm_status_se0 = RREG32(GRBM_STATUS_SE0);
2322
	grbm_status_se1 = RREG32(GRBM_STATUS_SE1);
2323
	if (!(grbm_status & GUI_ACTIVE)) {
2324
		r100_gpu_lockup_update(lockup, &rdev->cp);
1963 serge 2325
	return false;
1986 serge 2326
	}
2327
	/* force CP activities */
2328
	r = radeon_ring_lock(rdev, 2);
2329
	if (!r) {
2330
		/* PACKET2 NOP */
2331
		radeon_ring_write(rdev, 0x80000000);
2332
		radeon_ring_write(rdev, 0x80000000);
2333
		radeon_ring_unlock_commit(rdev);
2334
	}
2335
	rdev->cp.rptr = RREG32(CP_RB_RPTR);
2336
	return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp);
1963 serge 2337
}
2338
 
2339
static int evergreen_gpu_soft_reset(struct radeon_device *rdev)
2340
{
2341
	struct evergreen_mc_save save;
2342
	u32 grbm_reset = 0;
2343
 
2344
	if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE))
2345
		return 0;
2346
 
2347
	dev_info(rdev->dev, "GPU softreset \n");
2348
	dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
2349
		RREG32(GRBM_STATUS));
2350
	dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
2351
		RREG32(GRBM_STATUS_SE0));
2352
	dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
2353
		RREG32(GRBM_STATUS_SE1));
2354
	dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
2355
		RREG32(SRBM_STATUS));
2356
	evergreen_mc_stop(rdev, &save);
2357
	if (evergreen_mc_wait_for_idle(rdev)) {
2358
		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2359
	}
2360
	/* Disable CP parsing/prefetching */
2361
	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
2362
 
2363
	/* reset all the gfx blocks */
2364
	grbm_reset = (SOFT_RESET_CP |
2365
		      SOFT_RESET_CB |
2366
		      SOFT_RESET_DB |
2367
		      SOFT_RESET_PA |
2368
		      SOFT_RESET_SC |
2369
		      SOFT_RESET_SPI |
2370
		      SOFT_RESET_SH |
2371
		      SOFT_RESET_SX |
2372
		      SOFT_RESET_TC |
2373
		      SOFT_RESET_TA |
2374
		      SOFT_RESET_VC |
2375
		      SOFT_RESET_VGT);
2376
 
2377
	dev_info(rdev->dev, "  GRBM_SOFT_RESET=0x%08X\n", grbm_reset);
2378
	WREG32(GRBM_SOFT_RESET, grbm_reset);
2379
	(void)RREG32(GRBM_SOFT_RESET);
2380
	udelay(50);
2381
	WREG32(GRBM_SOFT_RESET, 0);
2382
	(void)RREG32(GRBM_SOFT_RESET);
2383
	/* Wait a little for things to settle down */
2384
	udelay(50);
2385
	dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
2386
		RREG32(GRBM_STATUS));
2387
	dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
2388
		RREG32(GRBM_STATUS_SE0));
2389
	dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
2390
		RREG32(GRBM_STATUS_SE1));
2391
	dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
2392
		RREG32(SRBM_STATUS));
2393
	evergreen_mc_resume(rdev, &save);
1430 serge 2394
	return 0;
2395
}
2396
 
1963 serge 2397
int evergreen_asic_reset(struct radeon_device *rdev)
2398
{
2399
	return evergreen_gpu_soft_reset(rdev);
2400
}
2401
 
2402
/* Interrupts */
2403
 
2404
u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
2405
{
2406
	switch (crtc) {
2407
	case 0:
2408
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC0_REGISTER_OFFSET);
2409
	case 1:
2410
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC1_REGISTER_OFFSET);
2411
	case 2:
2412
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC2_REGISTER_OFFSET);
2413
	case 3:
2414
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC3_REGISTER_OFFSET);
2415
	case 4:
2416
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC4_REGISTER_OFFSET);
2417
	case 5:
2418
		return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC5_REGISTER_OFFSET);
2419
	default:
2420
	return 0;
2421
	}
2422
}
2423
 
2424
void evergreen_disable_interrupt_state(struct radeon_device *rdev)
2425
{
2426
	u32 tmp;
2427
 
2428
	WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
2429
	WREG32(GRBM_INT_CNTL, 0);
2430
	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2431
	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005 serge 2432
	if (rdev->num_crtc >= 4) {
1963 serge 2433
	WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2434
	WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005 serge 2435
	}
2436
	if (rdev->num_crtc >= 6) {
1963 serge 2437
	WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2438
	WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2439
	}
2440
 
2441
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
2442
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
2005 serge 2443
	if (rdev->num_crtc >= 4) {
1963 serge 2444
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
2445
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
2005 serge 2446
	}
2447
	if (rdev->num_crtc >= 6) {
1963 serge 2448
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
2449
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
2450
	}
2451
 
2452
	WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
2453
	WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
2454
 
2455
	tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2456
	WREG32(DC_HPD1_INT_CONTROL, tmp);
2457
	tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2458
	WREG32(DC_HPD2_INT_CONTROL, tmp);
2459
	tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2460
	WREG32(DC_HPD3_INT_CONTROL, tmp);
2461
	tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2462
	WREG32(DC_HPD4_INT_CONTROL, tmp);
2463
	tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2464
	WREG32(DC_HPD5_INT_CONTROL, tmp);
2465
	tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
2466
	WREG32(DC_HPD6_INT_CONTROL, tmp);
2467
 
2468
}
2005 serge 2469
 
2470
int evergreen_irq_set(struct radeon_device *rdev)
2471
{
2472
	u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
2473
	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
2474
	u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
2475
	u32 grbm_int_cntl = 0;
2476
	u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
2477
 
2478
	if (!rdev->irq.installed) {
2479
		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
2480
		return -EINVAL;
2481
	}
2482
	/* don't enable anything if the ih is disabled */
2483
	if (!rdev->ih.enabled) {
2484
		r600_disable_interrupts(rdev);
2485
		/* force the active interrupt state to all disabled */
2486
		evergreen_disable_interrupt_state(rdev);
2487
		return 0;
2488
	}
2489
 
2490
	hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
2491
	hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
2492
	hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
2493
	hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
2494
	hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
2495
	hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
2496
 
2497
	if (rdev->irq.sw_int) {
2498
		DRM_DEBUG("evergreen_irq_set: sw int\n");
2499
		cp_int_cntl |= RB_INT_ENABLE;
2500
		cp_int_cntl |= TIME_STAMP_INT_ENABLE;
2501
	}
2502
	if (rdev->irq.crtc_vblank_int[0] ||
2503
	    rdev->irq.pflip[0]) {
2504
		DRM_DEBUG("evergreen_irq_set: vblank 0\n");
2505
		crtc1 |= VBLANK_INT_MASK;
2506
	}
2507
	if (rdev->irq.crtc_vblank_int[1] ||
2508
	    rdev->irq.pflip[1]) {
2509
		DRM_DEBUG("evergreen_irq_set: vblank 1\n");
2510
		crtc2 |= VBLANK_INT_MASK;
2511
	}
2512
	if (rdev->irq.crtc_vblank_int[2] ||
2513
	    rdev->irq.pflip[2]) {
2514
		DRM_DEBUG("evergreen_irq_set: vblank 2\n");
2515
		crtc3 |= VBLANK_INT_MASK;
2516
	}
2517
	if (rdev->irq.crtc_vblank_int[3] ||
2518
	    rdev->irq.pflip[3]) {
2519
		DRM_DEBUG("evergreen_irq_set: vblank 3\n");
2520
		crtc4 |= VBLANK_INT_MASK;
2521
	}
2522
	if (rdev->irq.crtc_vblank_int[4] ||
2523
	    rdev->irq.pflip[4]) {
2524
		DRM_DEBUG("evergreen_irq_set: vblank 4\n");
2525
		crtc5 |= VBLANK_INT_MASK;
2526
	}
2527
	if (rdev->irq.crtc_vblank_int[5] ||
2528
	    rdev->irq.pflip[5]) {
2529
		DRM_DEBUG("evergreen_irq_set: vblank 5\n");
2530
		crtc6 |= VBLANK_INT_MASK;
2531
	}
2532
	if (rdev->irq.hpd[0]) {
2533
		DRM_DEBUG("evergreen_irq_set: hpd 1\n");
2534
		hpd1 |= DC_HPDx_INT_EN;
2535
	}
2536
	if (rdev->irq.hpd[1]) {
2537
		DRM_DEBUG("evergreen_irq_set: hpd 2\n");
2538
		hpd2 |= DC_HPDx_INT_EN;
2539
	}
2540
	if (rdev->irq.hpd[2]) {
2541
		DRM_DEBUG("evergreen_irq_set: hpd 3\n");
2542
		hpd3 |= DC_HPDx_INT_EN;
2543
	}
2544
	if (rdev->irq.hpd[3]) {
2545
		DRM_DEBUG("evergreen_irq_set: hpd 4\n");
2546
		hpd4 |= DC_HPDx_INT_EN;
2547
	}
2548
	if (rdev->irq.hpd[4]) {
2549
		DRM_DEBUG("evergreen_irq_set: hpd 5\n");
2550
		hpd5 |= DC_HPDx_INT_EN;
2551
	}
2552
	if (rdev->irq.hpd[5]) {
2553
		DRM_DEBUG("evergreen_irq_set: hpd 6\n");
2554
		hpd6 |= DC_HPDx_INT_EN;
2555
	}
2556
	if (rdev->irq.gui_idle) {
2557
		DRM_DEBUG("gui idle\n");
2558
		grbm_int_cntl |= GUI_IDLE_INT_ENABLE;
2559
	}
2560
 
2561
	WREG32(CP_INT_CNTL, cp_int_cntl);
2562
	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
2563
 
2564
	WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
2565
	WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
2566
	if (rdev->num_crtc >= 4) {
2567
		WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
2568
		WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
2569
	}
2570
	if (rdev->num_crtc >= 6) {
2571
		WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
2572
		WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
2573
	}
2574
 
2575
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
2576
	WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
2577
	if (rdev->num_crtc >= 4) {
2578
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
2579
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
2580
	}
2581
	if (rdev->num_crtc >= 6) {
2582
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
2583
		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
2584
	}
2585
 
2586
	WREG32(DC_HPD1_INT_CONTROL, hpd1);
2587
	WREG32(DC_HPD2_INT_CONTROL, hpd2);
2588
	WREG32(DC_HPD3_INT_CONTROL, hpd3);
2589
	WREG32(DC_HPD4_INT_CONTROL, hpd4);
2590
	WREG32(DC_HPD5_INT_CONTROL, hpd5);
2591
	WREG32(DC_HPD6_INT_CONTROL, hpd6);
2592
 
2593
	return 0;
2594
}
2595
 
2596
static inline void evergreen_irq_ack(struct radeon_device *rdev)
2597
{
2598
	u32 tmp;
2599
 
2600
	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
2601
	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
2602
	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
2603
	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
2604
	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
2605
	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
2606
	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
2607
	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
2608
	if (rdev->num_crtc >= 4) {
2609
		rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
2610
		rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
2611
	}
2612
	if (rdev->num_crtc >= 6) {
2613
		rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
2614
		rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
2615
	}
2616
 
2617
	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
2618
		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2619
	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
2620
		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2621
	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
2622
		WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
2623
	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
2624
		WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
2625
	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
2626
		WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
2627
	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
2628
		WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
2629
 
2630
	if (rdev->num_crtc >= 4) {
2631
		if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
2632
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2633
		if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
2634
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2635
		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
2636
			WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
2637
		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
2638
			WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
2639
		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
2640
			WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
2641
		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
2642
			WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
2643
	}
2644
 
2645
	if (rdev->num_crtc >= 6) {
2646
		if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
2647
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2648
		if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
2649
			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
2650
		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
2651
			WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
2652
		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
2653
			WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
2654
		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
2655
			WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
2656
		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
2657
			WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
2658
	}
2659
 
2660
	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2661
		tmp = RREG32(DC_HPD1_INT_CONTROL);
2662
		tmp |= DC_HPDx_INT_ACK;
2663
		WREG32(DC_HPD1_INT_CONTROL, tmp);
2664
	}
2665
	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2666
		tmp = RREG32(DC_HPD2_INT_CONTROL);
2667
		tmp |= DC_HPDx_INT_ACK;
2668
		WREG32(DC_HPD2_INT_CONTROL, tmp);
2669
	}
2670
	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2671
		tmp = RREG32(DC_HPD3_INT_CONTROL);
2672
		tmp |= DC_HPDx_INT_ACK;
2673
		WREG32(DC_HPD3_INT_CONTROL, tmp);
2674
	}
2675
	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2676
		tmp = RREG32(DC_HPD4_INT_CONTROL);
2677
		tmp |= DC_HPDx_INT_ACK;
2678
		WREG32(DC_HPD4_INT_CONTROL, tmp);
2679
	}
2680
	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2681
		tmp = RREG32(DC_HPD5_INT_CONTROL);
2682
		tmp |= DC_HPDx_INT_ACK;
2683
		WREG32(DC_HPD5_INT_CONTROL, tmp);
2684
	}
2685
	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2686
		tmp = RREG32(DC_HPD5_INT_CONTROL);
2687
		tmp |= DC_HPDx_INT_ACK;
2688
		WREG32(DC_HPD6_INT_CONTROL, tmp);
2689
	}
2690
}
2691
static inline u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
2692
{
2693
	u32 wptr, tmp;
2694
 
2695
	if (rdev->wb.enabled)
2696
		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
2697
	else
2698
		wptr = RREG32(IH_RB_WPTR);
2699
 
2700
	if (wptr & RB_OVERFLOW) {
2701
		/* When a ring buffer overflow happen start parsing interrupt
2702
		 * from the last not overwritten vector (wptr + 16). Hopefully
2703
		 * this should allow us to catchup.
2704
		 */
2705
		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
2706
			wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
2707
		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
2708
		tmp = RREG32(IH_RB_CNTL);
2709
		tmp |= IH_WPTR_OVERFLOW_CLEAR;
2710
		WREG32(IH_RB_CNTL, tmp);
2711
	}
2712
	return (wptr & rdev->ih.ptr_mask);
2713
}
2714
 
2715
int evergreen_irq_process(struct radeon_device *rdev)
2716
{
2717
	u32 wptr;
2718
	u32 rptr;
2719
	u32 src_id, src_data;
2720
	u32 ring_index;
2721
	unsigned long flags;
2722
	bool queue_hotplug = false;
2723
 
2724
	if (!rdev->ih.enabled || rdev->shutdown)
2725
		return IRQ_NONE;
2726
 
2727
	wptr = evergreen_get_ih_wptr(rdev);
2728
	rptr = rdev->ih.rptr;
2729
	DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
2730
 
2731
	spin_lock_irqsave(&rdev->ih.lock, flags);
2732
	if (rptr == wptr) {
2733
		spin_unlock_irqrestore(&rdev->ih.lock, flags);
2734
		return IRQ_NONE;
2735
	}
2736
restart_ih:
2175 serge 2737
	/* Order reading of wptr vs. reading of IH ring data */
2738
	rmb();
2739
 
2005 serge 2740
	/* display interrupts */
2741
	evergreen_irq_ack(rdev);
2742
 
2743
	rdev->ih.wptr = wptr;
2744
	while (rptr != wptr) {
2745
		/* wptr/rptr are in bytes! */
2746
		ring_index = rptr / 4;
2747
		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
2748
		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
2749
 
2750
		switch (src_id) {
2751
		case 1: /* D1 vblank/vline */
2752
			switch (src_data) {
2753
			case 0: /* D1 vblank */
2754
				if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
2755
					if (rdev->irq.crtc_vblank_int[0]) {
2756
				//		drm_handle_vblank(rdev->ddev, 0);
2757
						rdev->pm.vblank_sync = true;
2758
				//		wake_up(&rdev->irq.vblank_queue);
2759
					}
2760
				//	if (rdev->irq.pflip[0])
2761
				//		radeon_crtc_handle_flip(rdev, 0);
2762
					rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
2763
					DRM_DEBUG("IH: D1 vblank\n");
2764
				}
2765
				break;
2766
			case 1: /* D1 vline */
2767
				if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
2768
					rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
2769
					DRM_DEBUG("IH: D1 vline\n");
2770
				}
2771
				break;
2772
			default:
2773
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2774
				break;
2775
			}
2776
			break;
2777
		case 2: /* D2 vblank/vline */
2778
			switch (src_data) {
2779
			case 0: /* D2 vblank */
2780
				if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
2781
					if (rdev->irq.crtc_vblank_int[1]) {
2782
				//		drm_handle_vblank(rdev->ddev, 1);
2783
						rdev->pm.vblank_sync = true;
2784
				//		wake_up(&rdev->irq.vblank_queue);
2785
					}
2786
			//		if (rdev->irq.pflip[1])
2787
			//			radeon_crtc_handle_flip(rdev, 1);
2788
					rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
2789
					DRM_DEBUG("IH: D2 vblank\n");
2790
				}
2791
				break;
2792
			case 1: /* D2 vline */
2793
				if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
2794
					rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
2795
					DRM_DEBUG("IH: D2 vline\n");
2796
				}
2797
				break;
2798
			default:
2799
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2800
				break;
2801
			}
2802
			break;
2803
		case 3: /* D3 vblank/vline */
2804
			switch (src_data) {
2805
			case 0: /* D3 vblank */
2806
				if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
2807
					if (rdev->irq.crtc_vblank_int[2]) {
2808
				//		drm_handle_vblank(rdev->ddev, 2);
2809
						rdev->pm.vblank_sync = true;
2810
				//		wake_up(&rdev->irq.vblank_queue);
2811
					}
2812
				//	if (rdev->irq.pflip[2])
2813
				//		radeon_crtc_handle_flip(rdev, 2);
2814
					rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
2815
					DRM_DEBUG("IH: D3 vblank\n");
2816
				}
2817
				break;
2818
			case 1: /* D3 vline */
2819
				if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
2820
					rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
2821
					DRM_DEBUG("IH: D3 vline\n");
2822
				}
2823
				break;
2824
			default:
2825
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2826
				break;
2827
			}
2828
			break;
2829
		case 4: /* D4 vblank/vline */
2830
			switch (src_data) {
2831
			case 0: /* D4 vblank */
2832
				if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
2833
					if (rdev->irq.crtc_vblank_int[3]) {
2834
					//	drm_handle_vblank(rdev->ddev, 3);
2835
						rdev->pm.vblank_sync = true;
2836
					//	wake_up(&rdev->irq.vblank_queue);
2837
					}
2838
		//			if (rdev->irq.pflip[3])
2839
		//				radeon_crtc_handle_flip(rdev, 3);
2840
					rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
2841
					DRM_DEBUG("IH: D4 vblank\n");
2842
				}
2843
				break;
2844
			case 1: /* D4 vline */
2845
				if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
2846
					rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
2847
					DRM_DEBUG("IH: D4 vline\n");
2848
				}
2849
				break;
2850
			default:
2851
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2852
				break;
2853
			}
2854
			break;
2855
		case 5: /* D5 vblank/vline */
2856
			switch (src_data) {
2857
			case 0: /* D5 vblank */
2858
				if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
2859
					if (rdev->irq.crtc_vblank_int[4]) {
2860
//						drm_handle_vblank(rdev->ddev, 4);
2861
						rdev->pm.vblank_sync = true;
2862
//						wake_up(&rdev->irq.vblank_queue);
2863
					}
2864
//					if (rdev->irq.pflip[4])
2865
//						radeon_crtc_handle_flip(rdev, 4);
2866
					rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
2867
					DRM_DEBUG("IH: D5 vblank\n");
2868
				}
2869
				break;
2870
			case 1: /* D5 vline */
2871
				if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
2872
					rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
2873
					DRM_DEBUG("IH: D5 vline\n");
2874
				}
2875
				break;
2876
			default:
2877
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2878
				break;
2879
			}
2880
			break;
2881
		case 6: /* D6 vblank/vline */
2882
			switch (src_data) {
2883
			case 0: /* D6 vblank */
2884
				if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
2885
					if (rdev->irq.crtc_vblank_int[5]) {
2886
				//		drm_handle_vblank(rdev->ddev, 5);
2887
						rdev->pm.vblank_sync = true;
2888
				//		wake_up(&rdev->irq.vblank_queue);
2889
					}
2890
			//		if (rdev->irq.pflip[5])
2891
			//			radeon_crtc_handle_flip(rdev, 5);
2892
					rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
2893
					DRM_DEBUG("IH: D6 vblank\n");
2894
				}
2895
				break;
2896
			case 1: /* D6 vline */
2897
				if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
2898
					rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
2899
					DRM_DEBUG("IH: D6 vline\n");
2900
				}
2901
				break;
2902
			default:
2903
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2904
				break;
2905
			}
2906
			break;
2907
		case 42: /* HPD hotplug */
2908
			switch (src_data) {
2909
			case 0:
2910
				if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
2911
					rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
2912
					queue_hotplug = true;
2913
					DRM_DEBUG("IH: HPD1\n");
2914
				}
2915
				break;
2916
			case 1:
2917
				if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
2918
					rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
2919
					queue_hotplug = true;
2920
					DRM_DEBUG("IH: HPD2\n");
2921
				}
2922
				break;
2923
			case 2:
2924
				if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
2925
					rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
2926
					queue_hotplug = true;
2927
					DRM_DEBUG("IH: HPD3\n");
2928
				}
2929
				break;
2930
			case 3:
2931
				if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
2932
					rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
2933
					queue_hotplug = true;
2934
					DRM_DEBUG("IH: HPD4\n");
2935
				}
2936
				break;
2937
			case 4:
2938
				if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
2939
					rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
2940
					queue_hotplug = true;
2941
					DRM_DEBUG("IH: HPD5\n");
2942
				}
2943
				break;
2944
			case 5:
2945
				if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
2946
					rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
2947
					queue_hotplug = true;
2948
					DRM_DEBUG("IH: HPD6\n");
2949
				}
2950
				break;
2951
			default:
2952
				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2953
				break;
2954
			}
2955
			break;
2956
		case 176: /* CP_INT in ring buffer */
2957
		case 177: /* CP_INT in IB1 */
2958
		case 178: /* CP_INT in IB2 */
2959
			DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
2960
			radeon_fence_process(rdev);
2961
			break;
2962
		case 181: /* CP EOP event */
2963
			DRM_DEBUG("IH: CP EOP\n");
2964
			radeon_fence_process(rdev);
2965
			break;
2966
		case 233: /* GUI IDLE */
2967
			DRM_DEBUG("IH: GUI idle\n");
2968
			rdev->pm.gui_idle = true;
2969
//			wake_up(&rdev->irq.idle_queue);
2970
			break;
2971
		default:
2972
			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
2973
			break;
2974
		}
2975
 
2976
		/* wptr/rptr are in bytes! */
2977
		rptr += 16;
2978
		rptr &= rdev->ih.ptr_mask;
2979
	}
2980
	/* make sure wptr hasn't changed while processing */
2981
	wptr = evergreen_get_ih_wptr(rdev);
2982
	if (wptr != rdev->ih.wptr)
2983
		goto restart_ih;
2984
//	if (queue_hotplug)
2985
//		schedule_work(&rdev->hotplug_work);
2986
	rdev->ih.rptr = rptr;
2987
	WREG32(IH_RB_RPTR, rdev->ih.rptr);
2988
	spin_unlock_irqrestore(&rdev->ih.lock, flags);
2989
	return IRQ_HANDLED;
2990
}
2991
 
1430 serge 2992
static int evergreen_startup(struct radeon_device *rdev)
2993
{
2994
	int r;
2995
 
1990 serge 2996
	/* enable pcie gen2 link */
2997
	if (!ASIC_IS_DCE5(rdev))
2998
		evergreen_pcie_gen2_enable(rdev);
2999
 
3000
	if (ASIC_IS_DCE5(rdev)) {
3001
		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
3002
			r = ni_init_microcode(rdev);
3003
			if (r) {
3004
				DRM_ERROR("Failed to load firmware!\n");
3005
				return r;
3006
			}
3007
		}
3008
		r = ni_mc_load_microcode(rdev);
3009
		if (r) {
3010
			DRM_ERROR("Failed to load MC firmware!\n");
3011
			return r;
3012
		}
3013
	} else {
1430 serge 3014
	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
3015
		r = r600_init_microcode(rdev);
3016
		if (r) {
3017
			DRM_ERROR("Failed to load firmware!\n");
3018
			return r;
3019
		}
3020
	}
1990 serge 3021
	}
1963 serge 3022
 
1430 serge 3023
	evergreen_mc_program(rdev);
3024
	if (rdev->flags & RADEON_IS_AGP) {
1963 serge 3025
		evergreen_agp_enable(rdev);
1430 serge 3026
	} else {
3027
		r = evergreen_pcie_gart_enable(rdev);
3028
		if (r)
3029
			return r;
3030
	}
3031
	evergreen_gpu_init(rdev);
2005 serge 3032
 
1963 serge 3033
	r = evergreen_blit_init(rdev);
3034
	if (r) {
3035
		evergreen_blit_fini(rdev);
3036
		rdev->asic->copy = NULL;
3037
		dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
3038
	}
1430 serge 3039
 
1963 serge 3040
	/* allocate wb buffer */
3041
	r = radeon_wb_init(rdev);
3042
	if (r)
3043
		return r;
3044
 
3045
	/* Enable IRQ */
2005 serge 3046
	r = r600_irq_init(rdev);
3047
	if (r) {
3048
		DRM_ERROR("radeon: IH init failed (%d).\n", r);
3049
//		radeon_irq_kms_fini(rdev);
3050
		return r;
3051
	}
3052
	evergreen_irq_set(rdev);
1963 serge 3053
 
1430 serge 3054
    r = radeon_ring_init(rdev, rdev->cp.ring_size);
3055
	if (r)
3056
		return r;
3057
	r = evergreen_cp_load_microcode(rdev);
3058
	if (r)
3059
		return r;
1963 serge 3060
	r = evergreen_cp_resume(rdev);
1430 serge 3061
	if (r)
3062
		return r;
1963 serge 3063
 
1430 serge 3064
	return 0;
3065
}
3066
 
3067
 
3068
 
3069
 
3070
 
2005 serge 3071
int evergreen_copy_blit(struct radeon_device *rdev,
3072
			uint64_t src_offset, uint64_t dst_offset,
3073
			unsigned num_pages, struct radeon_fence *fence)
3074
{
3075
	int r;
1430 serge 3076
 
2005 serge 3077
	mutex_lock(&rdev->r600_blit.mutex);
3078
	rdev->r600_blit.vb_ib = NULL;
3079
	r = evergreen_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE);
3080
	if (r) {
3081
		if (rdev->r600_blit.vb_ib)
3082
			radeon_ib_free(rdev, &rdev->r600_blit.vb_ib);
3083
		mutex_unlock(&rdev->r600_blit.mutex);
3084
		return r;
3085
	}
3086
	evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE);
3087
	evergreen_blit_done_copy(rdev, fence);
3088
	mutex_unlock(&rdev->r600_blit.mutex);
3089
	return 0;
3090
}
1430 serge 3091
 
3092
/* Plan is to move initialization in that function and use
3093
 * helper function so that radeon_device_init pretty much
3094
 * do nothing more than calling asic specific function. This
3095
 * should also allow to remove a bunch of callback function
3096
 * like vram_info.
3097
 */
3098
int evergreen_init(struct radeon_device *rdev)
3099
{
3100
	int r;
3101
 
3102
	/* This don't do much */
3103
	r = radeon_gem_init(rdev);
3104
	if (r)
3105
		return r;
3106
	/* Read BIOS */
3107
	if (!radeon_get_bios(rdev)) {
3108
		if (ASIC_IS_AVIVO(rdev))
3109
			return -EINVAL;
3110
	}
3111
	/* Must be an ATOMBIOS */
3112
	if (!rdev->is_atom_bios) {
1986 serge 3113
		dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
1430 serge 3114
		return -EINVAL;
3115
	}
3116
	r = radeon_atombios_init(rdev);
3117
	if (r)
3118
		return r;
1986 serge 3119
	/* reset the asic, the gfx blocks are often in a bad state
3120
	 * after the driver is unloaded or after a resume
3121
	 */
3122
	if (radeon_asic_reset(rdev))
3123
		dev_warn(rdev->dev, "GPU reset failed !\n");
1430 serge 3124
	/* Post card if necessary */
1986 serge 3125
	if (!radeon_card_posted(rdev)) {
1430 serge 3126
		if (!rdev->bios) {
3127
			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3128
			return -EINVAL;
3129
		}
3130
		DRM_INFO("GPU not posted. posting now...\n");
3131
		atom_asic_init(rdev->mode_info.atom_context);
3132
	}
3133
	/* Initialize scratch registers */
3134
	r600_scratch_init(rdev);
3135
	/* Initialize surface registers */
3136
	radeon_surface_init(rdev);
3137
	/* Initialize clocks */
3138
	radeon_get_clock_info(rdev->ddev);
3139
	/* Fence driver */
2005 serge 3140
	r = radeon_fence_driver_init(rdev);
3141
	if (r)
3142
		return r;
1430 serge 3143
    /* initialize AGP */
3144
	if (rdev->flags & RADEON_IS_AGP) {
3145
		r = radeon_agp_init(rdev);
3146
		if (r)
3147
			radeon_agp_disable(rdev);
3148
	}
3149
	/* initialize memory controller */
3150
	r = evergreen_mc_init(rdev);
3151
	if (r)
3152
		return r;
3153
	/* Memory manager */
3154
	r = radeon_bo_init(rdev);
3155
	if (r)
3156
		return r;
1963 serge 3157
 
2005 serge 3158
	r = radeon_irq_kms_init(rdev);
3159
	if (r)
3160
		return r;
1430 serge 3161
 
3162
	rdev->cp.ring_obj = NULL;
3163
	r600_ring_init(rdev, 1024 * 1024);
3164
 
2005 serge 3165
	rdev->ih.ring_obj = NULL;
3166
	r600_ih_ring_init(rdev, 64 * 1024);
1430 serge 3167
 
3168
	r = r600_pcie_gart_init(rdev);
3169
	if (r)
3170
		return r;
1963 serge 3171
 
3172
	rdev->accel_working = true;
1430 serge 3173
	r = evergreen_startup(rdev);
3174
	if (r) {
1963 serge 3175
		dev_err(rdev->dev, "disabling GPU acceleration\n");
1430 serge 3176
		rdev->accel_working = false;
3177
	}
3178
	if (rdev->accel_working) {
2005 serge 3179
		r = radeon_ib_pool_init(rdev);
3180
		if (r) {
3181
			DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r);
3182
			rdev->accel_working = false;
3183
		}
3184
		r = r600_ib_test(rdev);
3185
		if (r) {
3186
			DRM_ERROR("radeon: failed testing IB (%d).\n", r);
3187
			rdev->accel_working = false;
3188
		}
1430 serge 3189
	}
3190
	return 0;
3191
}
3192
 
1986 serge 3193
 
3194
static void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
1430 serge 3195
{
1986 serge 3196
	u32 link_width_cntl, speed_cntl;
3197
 
3198
	if (radeon_pcie_gen2 == 0)
3199
		return;
3200
 
3201
	if (rdev->flags & RADEON_IS_IGP)
3202
		return;
3203
 
3204
	if (!(rdev->flags & RADEON_IS_PCIE))
3205
		return;
3206
 
3207
	/* x2 cards have a special sequence */
3208
	if (ASIC_IS_X2(rdev))
3209
		return;
3210
 
3211
	speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3212
	if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
3213
	    (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3214
 
3215
		link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3216
		link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3217
		WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3218
 
3219
		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3220
		speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
3221
		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3222
 
3223
		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3224
		speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3225
		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3226
 
3227
		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3228
		speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3229
		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3230
 
3231
		speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
3232
		speed_cntl |= LC_GEN2_EN_STRAP;
3233
		WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
3234
 
3235
	} else {
3236
		link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
3237
		/* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
3238
		if (1)
3239
			link_width_cntl |= LC_UPCONFIGURE_DIS;
3240
		else
3241
			link_width_cntl &= ~LC_UPCONFIGURE_DIS;
3242
		WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
3243
	}
1430 serge 3244
}